1	//===------- LegalizeVectorTypes.cpp - Legalization of vector types -------===//
2	//
3	// Part of the LLVM Project, under the Apache License v2.0 with LLVM Exceptions.
4	// See https://llvm.org/LICENSE.txt for license information.
5	// SPDX-License-Identifier: Apache-2.0 WITH LLVM-exception
6	//
7	//===----------------------------------------------------------------------===//
8	//
9	// This file performs vector type splitting and scalarization for LegalizeTypes.
10	// Scalarization is the act of changing a computation in an illegal one-element
11	// vector type to be a computation in its scalar element type. For example,
12	// implementing <1 x f32> arithmetic in a scalar f32 register. This is needed
13	// as a base case when scalarizing vector arithmetic like <4 x f32>, which
14	// eventually decomposes to scalars if the target doesn't support v4f32 or v2f32
15	// types.
16	// Splitting is the act of changing a computation in an invalid vector type to
17	// be a computation in two vectors of half the size. For example, implementing
18	// <128 x f32> operations in terms of two <64 x f32> operations.
19	//
20	//===----------------------------------------------------------------------===//
21
22	#include "LegalizeTypes.h"
23	#include "llvm/ADT/SmallBitVector.h"
24	#include "llvm/Analysis/MemoryLocation.h"
25	#include "llvm/Analysis/VectorUtils.h"
26	#include "llvm/IR/DataLayout.h"
27	#include "llvm/Support/ErrorHandling.h"
28	#include "llvm/Support/TypeSize.h"
29	#include "llvm/Support/raw_ostream.h"
30	#include <numeric>
31
32	using namespace llvm;
33
34	#define DEBUG_TYPE "legalize-types"
35
36	//===----------------------------------------------------------------------===//
37	// Result Vector Scalarization: <1 x ty> -> ty.
38	//===----------------------------------------------------------------------===//
39
40	void DAGTypeLegalizer::ScalarizeVectorResult(SDNode *N, unsigned ResNo) {
41	LLVM_DEBUG(dbgs() << "Scalarize node result " << ResNo << ": ";
42	N->dump(&DAG));
43	SDValue R = SDValue();
44
45	switch (N->getOpcode()) {
46	default:
47	#ifndef NDEBUG
48	dbgs() << "ScalarizeVectorResult #" << ResNo << ": ";
49	N->dump(G: &DAG);
50	dbgs() << "\n";
51	#endif
52	report_fatal_error(reason: "Do not know how to scalarize the result of this "
53	"operator!\n");
54
55	case ISD::MERGE_VALUES: R = ScalarizeVecRes_MERGE_VALUES(N, ResNo);break;
56	case ISD::BITCAST: R = ScalarizeVecRes_BITCAST(N); break;
57	case ISD::BUILD_VECTOR: R = ScalarizeVecRes_BUILD_VECTOR(N); break;
58	case ISD::EXTRACT_SUBVECTOR: R = ScalarizeVecRes_EXTRACT_SUBVECTOR(N); break;
59	case ISD::FP_ROUND: R = ScalarizeVecRes_FP_ROUND(N); break;
60	case ISD::FPOWI: R = ScalarizeVecRes_ExpOp(N); break;
61	case ISD::INSERT_VECTOR_ELT: R = ScalarizeVecRes_INSERT_VECTOR_ELT(N); break;
62	case ISD::LOAD: R = ScalarizeVecRes_LOAD(N: cast<LoadSDNode>(Val: N));break;
63	case ISD::SCALAR_TO_VECTOR: R = ScalarizeVecRes_SCALAR_TO_VECTOR(N); break;
64	case ISD::SIGN_EXTEND_INREG: R = ScalarizeVecRes_InregOp(N); break;
65	case ISD::VSELECT: R = ScalarizeVecRes_VSELECT(N); break;
66	case ISD::SELECT: R = ScalarizeVecRes_SELECT(N); break;
67	case ISD::SELECT_CC: R = ScalarizeVecRes_SELECT_CC(N); break;
68	case ISD::SETCC: R = ScalarizeVecRes_SETCC(N); break;
69	case ISD::UNDEF: R = ScalarizeVecRes_UNDEF(N); break;
70	case ISD::VECTOR_SHUFFLE: R = ScalarizeVecRes_VECTOR_SHUFFLE(N); break;
71	case ISD::IS_FPCLASS: R = ScalarizeVecRes_IS_FPCLASS(N); break;
72	case ISD::ANY_EXTEND_VECTOR_INREG:
73	case ISD::SIGN_EXTEND_VECTOR_INREG:
74	case ISD::ZERO_EXTEND_VECTOR_INREG:
75	R = ScalarizeVecRes_VecInregOp(N);
76	break;
77	case ISD::ABS:
78	case ISD::ANY_EXTEND:
79	case ISD::BITREVERSE:
80	case ISD::BSWAP:
81	case ISD::CTLZ:
82	case ISD::CTLZ_ZERO_UNDEF:
83	case ISD::CTPOP:
84	case ISD::CTTZ:
85	case ISD::CTTZ_ZERO_UNDEF:
86	case ISD::FABS:
87	case ISD::FCEIL:
88	case ISD::FCOS:
89	case ISD::FEXP:
90	case ISD::FEXP2:
91	case ISD::FEXP10:
92	case ISD::FFLOOR:
93	case ISD::FLOG:
94	case ISD::FLOG10:
95	case ISD::FLOG2:
96	case ISD::FNEARBYINT:
97	case ISD::FNEG:
98	case ISD::FREEZE:
99	case ISD::ARITH_FENCE:
100	case ISD::FP_EXTEND:
101	case ISD::FP_TO_SINT:
102	case ISD::FP_TO_UINT:
103	case ISD::FRINT:
104	case ISD::LRINT:
105	case ISD::LLRINT:
106	case ISD::FROUND:
107	case ISD::FROUNDEVEN:
108	case ISD::FSIN:
109	case ISD::FSQRT:
110	case ISD::FTRUNC:
111	case ISD::SIGN_EXTEND:
112	case ISD::SINT_TO_FP:
113	case ISD::TRUNCATE:
114	case ISD::UINT_TO_FP:
115	case ISD::ZERO_EXTEND:
116	case ISD::FCANONICALIZE:
117	R = ScalarizeVecRes_UnaryOp(N);
118	break;
119	case ISD::FFREXP:
120	R = ScalarizeVecRes_FFREXP(N, ResNo);
121	break;
122	case ISD::ADD:
123	case ISD::AND:
124	case ISD::FADD:
125	case ISD::FCOPYSIGN:
126	case ISD::FDIV:
127	case ISD::FMUL:
128	case ISD::FMINNUM:
129	case ISD::FMAXNUM:
130	case ISD::FMINNUM_IEEE:
131	case ISD::FMAXNUM_IEEE:
132	case ISD::FMINIMUM:
133	case ISD::FMAXIMUM:
134	case ISD::FLDEXP:
135	case ISD::SMIN:
136	case ISD::SMAX:
137	case ISD::UMIN:
138	case ISD::UMAX:
139
140	case ISD::SADDSAT:
141	case ISD::UADDSAT:
142	case ISD::SSUBSAT:
143	case ISD::USUBSAT:
144	case ISD::SSHLSAT:
145	case ISD::USHLSAT:
146
147	case ISD::FPOW:
148	case ISD::FREM:
149	case ISD::FSUB:
150	case ISD::MUL:
151	case ISD::MULHS:
152	case ISD::MULHU:
153	case ISD::OR:
154	case ISD::SDIV:
155	case ISD::SREM:
156	case ISD::SUB:
157	case ISD::UDIV:
158	case ISD::UREM:
159	case ISD::XOR:
160	case ISD::SHL:
161	case ISD::SRA:
162	case ISD::SRL:
163	case ISD::ROTL:
164	case ISD::ROTR:
165	R = ScalarizeVecRes_BinOp(N);
166	break;
167	case ISD::FMA:
168	case ISD::FSHL:
169	case ISD::FSHR:
170	R = ScalarizeVecRes_TernaryOp(N);
171	break;
172
173	#define DAG_INSTRUCTION(NAME, NARG, ROUND_MODE, INTRINSIC, DAGN) \
174	case ISD::STRICT_##DAGN:
175	#include "llvm/IR/ConstrainedOps.def"
176	R = ScalarizeVecRes_StrictFPOp(N);
177	break;
178
179	case ISD::FP_TO_UINT_SAT:
180	case ISD::FP_TO_SINT_SAT:
181	R = ScalarizeVecRes_FP_TO_XINT_SAT(N);
182	break;
183
184	case ISD::UADDO:
185	case ISD::SADDO:
186	case ISD::USUBO:
187	case ISD::SSUBO:
188	case ISD::UMULO:
189	case ISD::SMULO:
190	R = ScalarizeVecRes_OverflowOp(N, ResNo);
191	break;
192	case ISD::SMULFIX:
193	case ISD::SMULFIXSAT:
194	case ISD::UMULFIX:
195	case ISD::UMULFIXSAT:
196	case ISD::SDIVFIX:
197	case ISD::SDIVFIXSAT:
198	case ISD::UDIVFIX:
199	case ISD::UDIVFIXSAT:
200	R = ScalarizeVecRes_FIX(N);
201	break;
202	}
203
204	// If R is null, the sub-method took care of registering the result.
205	if (R.getNode())
206	SetScalarizedVector(Op: SDValue(N, ResNo), Result: R);
207	}
208
209	SDValue DAGTypeLegalizer::ScalarizeVecRes_BinOp(SDNode *N) {
210	SDValue LHS = GetScalarizedVector(Op: N->getOperand(Num: 0));
211	SDValue RHS = GetScalarizedVector(Op: N->getOperand(Num: 1));
212	return DAG.getNode(Opcode: N->getOpcode(), DL: SDLoc(N),
213	VT: LHS.getValueType(), N1: LHS, N2: RHS, Flags: N->getFlags());
214	}
215
216	SDValue DAGTypeLegalizer::ScalarizeVecRes_TernaryOp(SDNode *N) {
217	SDValue Op0 = GetScalarizedVector(Op: N->getOperand(Num: 0));
218	SDValue Op1 = GetScalarizedVector(Op: N->getOperand(Num: 1));
219	SDValue Op2 = GetScalarizedVector(Op: N->getOperand(Num: 2));
220	return DAG.getNode(Opcode: N->getOpcode(), DL: SDLoc(N), VT: Op0.getValueType(), N1: Op0, N2: Op1,
221	N3: Op2, Flags: N->getFlags());
222	}
223
224	SDValue DAGTypeLegalizer::ScalarizeVecRes_FIX(SDNode *N) {
225	SDValue Op0 = GetScalarizedVector(Op: N->getOperand(Num: 0));
226	SDValue Op1 = GetScalarizedVector(Op: N->getOperand(Num: 1));
227	SDValue Op2 = N->getOperand(Num: 2);
228	return DAG.getNode(Opcode: N->getOpcode(), DL: SDLoc(N), VT: Op0.getValueType(), N1: Op0, N2: Op1,
229	N3: Op2, Flags: N->getFlags());
230	}
231
232	SDValue DAGTypeLegalizer::ScalarizeVecRes_FFREXP(SDNode *N, unsigned ResNo) {
233	assert(N->getValueType(0).getVectorNumElements() == 1 &&
234	"Unexpected vector type!");
235	SDValue Elt = GetScalarizedVector(Op: N->getOperand(Num: 0));
236
237	EVT VT0 = N->getValueType(ResNo: 0);
238	EVT VT1 = N->getValueType(ResNo: 1);
239	SDLoc dl(N);
240
241	SDNode *ScalarNode =
242	DAG.getNode(Opcode: N->getOpcode(), DL: dl,
243	ResultTys: {VT0.getScalarType(), VT1.getScalarType()}, Ops: Elt)
244	.getNode();
245
246	// Replace the other vector result not being explicitly scalarized here.
247	unsigned OtherNo = 1 - ResNo;
248	EVT OtherVT = N->getValueType(ResNo: OtherNo);
249	if (getTypeAction(VT: OtherVT) == TargetLowering::TypeScalarizeVector) {
250	SetScalarizedVector(Op: SDValue(N, OtherNo), Result: SDValue(ScalarNode, OtherNo));
251	} else {
252	SDValue OtherVal = DAG.getNode(Opcode: ISD::SCALAR_TO_VECTOR, DL: dl, VT: OtherVT,
253	Operand: SDValue(ScalarNode, OtherNo));
254	ReplaceValueWith(From: SDValue(N, OtherNo), To: OtherVal);
255	}
256
257	return SDValue(ScalarNode, ResNo);
258	}
259
260	SDValue DAGTypeLegalizer::ScalarizeVecRes_StrictFPOp(SDNode *N) {
261	EVT VT = N->getValueType(ResNo: 0).getVectorElementType();
262	unsigned NumOpers = N->getNumOperands();
263	SDValue Chain = N->getOperand(Num: 0);
264	EVT ValueVTs[] = {VT, MVT::Other};
265	SDLoc dl(N);
266
267	SmallVector<SDValue, 4> Opers(NumOpers);
268
269	// The Chain is the first operand.
270	Opers[0] = Chain;
271
272	// Now process the remaining operands.
273	for (unsigned i = 1; i < NumOpers; ++i) {
274	SDValue Oper = N->getOperand(Num: i);
275	EVT OperVT = Oper.getValueType();
276
277	if (OperVT.isVector()) {
278	if (getTypeAction(VT: OperVT) == TargetLowering::TypeScalarizeVector)
279	Oper = GetScalarizedVector(Op: Oper);
280	else
281	Oper = DAG.getNode(Opcode: ISD::EXTRACT_VECTOR_ELT, DL: dl,
282	VT: OperVT.getVectorElementType(), N1: Oper,
283	N2: DAG.getVectorIdxConstant(Val: 0, DL: dl));
284	}
285
286	Opers[i] = Oper;
287	}
288
289	SDValue Result = DAG.getNode(Opcode: N->getOpcode(), DL: dl, VTList: DAG.getVTList(ValueVTs),
290	Ops: Opers, Flags: N->getFlags());
291
292	// Legalize the chain result - switch anything that used the old chain to
293	// use the new one.
294	ReplaceValueWith(From: SDValue(N, 1), To: Result.getValue(R: 1));
295	return Result;
296	}
297
298	SDValue DAGTypeLegalizer::ScalarizeVecRes_OverflowOp(SDNode *N,
299	unsigned ResNo) {
300	SDLoc DL(N);
301	EVT ResVT = N->getValueType(ResNo: 0);
302	EVT OvVT = N->getValueType(ResNo: 1);
303
304	SDValue ScalarLHS, ScalarRHS;
305	if (getTypeAction(VT: ResVT) == TargetLowering::TypeScalarizeVector) {
306	ScalarLHS = GetScalarizedVector(Op: N->getOperand(Num: 0));
307	ScalarRHS = GetScalarizedVector(Op: N->getOperand(Num: 1));
308	} else {
309	SmallVector<SDValue, 1> ElemsLHS, ElemsRHS;
310	DAG.ExtractVectorElements(Op: N->getOperand(Num: 0), Args&: ElemsLHS);
311	DAG.ExtractVectorElements(Op: N->getOperand(Num: 1), Args&: ElemsRHS);
312	ScalarLHS = ElemsLHS[0];
313	ScalarRHS = ElemsRHS[0];
314	}
315
316	SDVTList ScalarVTs = DAG.getVTList(
317	VT1: ResVT.getVectorElementType(), VT2: OvVT.getVectorElementType());
318	SDNode *ScalarNode = DAG.getNode(
319	Opcode: N->getOpcode(), DL, VTList: ScalarVTs, N1: ScalarLHS, N2: ScalarRHS).getNode();
320	ScalarNode->setFlags(N->getFlags());
321
322	// Replace the other vector result not being explicitly scalarized here.
323	unsigned OtherNo = 1 - ResNo;
324	EVT OtherVT = N->getValueType(ResNo: OtherNo);
325	if (getTypeAction(VT: OtherVT) == TargetLowering::TypeScalarizeVector) {
326	SetScalarizedVector(Op: SDValue(N, OtherNo), Result: SDValue(ScalarNode, OtherNo));
327	} else {
328	SDValue OtherVal = DAG.getNode(
329	Opcode: ISD::SCALAR_TO_VECTOR, DL, VT: OtherVT, Operand: SDValue(ScalarNode, OtherNo));
330	ReplaceValueWith(From: SDValue(N, OtherNo), To: OtherVal);
331	}
332
333	return SDValue(ScalarNode, ResNo);
334	}
335
336	SDValue DAGTypeLegalizer::ScalarizeVecRes_MERGE_VALUES(SDNode *N,
337	unsigned ResNo) {
338	SDValue Op = DisintegrateMERGE_VALUES(N, ResNo);
339	return GetScalarizedVector(Op);
340	}
341
342	SDValue DAGTypeLegalizer::ScalarizeVecRes_BITCAST(SDNode *N) {
343	SDValue Op = N->getOperand(Num: 0);
344	if (Op.getValueType().isVector()
345	&& Op.getValueType().getVectorNumElements() == 1
346	&& !isSimpleLegalType(VT: Op.getValueType()))
347	Op = GetScalarizedVector(Op);
348	EVT NewVT = N->getValueType(ResNo: 0).getVectorElementType();
349	return DAG.getNode(Opcode: ISD::BITCAST, DL: SDLoc(N),
350	VT: NewVT, Operand: Op);
351	}
352
353	SDValue DAGTypeLegalizer::ScalarizeVecRes_BUILD_VECTOR(SDNode *N) {
354	EVT EltVT = N->getValueType(ResNo: 0).getVectorElementType();
355	SDValue InOp = N->getOperand(Num: 0);
356	// The BUILD_VECTOR operands may be of wider element types and
357	// we may need to truncate them back to the requested return type.
358	if (EltVT.isInteger())
359	return DAG.getNode(Opcode: ISD::TRUNCATE, DL: SDLoc(N), VT: EltVT, Operand: InOp);
360	return InOp;
361	}
362
363	SDValue DAGTypeLegalizer::ScalarizeVecRes_EXTRACT_SUBVECTOR(SDNode *N) {
364	return DAG.getNode(Opcode: ISD::EXTRACT_VECTOR_ELT, DL: SDLoc(N),
365	VT: N->getValueType(ResNo: 0).getVectorElementType(),
366	N1: N->getOperand(Num: 0), N2: N->getOperand(Num: 1));
367	}
368
369	SDValue DAGTypeLegalizer::ScalarizeVecRes_FP_ROUND(SDNode *N) {
370	SDLoc DL(N);
371	SDValue Op = N->getOperand(Num: 0);
372	EVT OpVT = Op.getValueType();
373	// The result needs scalarizing, but it's not a given that the source does.
374	// See similar logic in ScalarizeVecRes_UnaryOp.
375	if (getTypeAction(VT: OpVT) == TargetLowering::TypeScalarizeVector) {
376	Op = GetScalarizedVector(Op);
377	} else {
378	EVT VT = OpVT.getVectorElementType();
379	Op = DAG.getNode(Opcode: ISD::EXTRACT_VECTOR_ELT, DL, VT, N1: Op,
380	N2: DAG.getVectorIdxConstant(Val: 0, DL));
381	}
382	return DAG.getNode(Opcode: ISD::FP_ROUND, DL,
383	VT: N->getValueType(ResNo: 0).getVectorElementType(), N1: Op,
384	N2: N->getOperand(Num: 1));
385	}
386
387	SDValue DAGTypeLegalizer::ScalarizeVecRes_ExpOp(SDNode *N) {
388	SDValue Op = GetScalarizedVector(Op: N->getOperand(Num: 0));
389	return DAG.getNode(Opcode: N->getOpcode(), DL: SDLoc(N), VT: Op.getValueType(), N1: Op,
390	N2: N->getOperand(Num: 1));
391	}
392
393	SDValue DAGTypeLegalizer::ScalarizeVecRes_INSERT_VECTOR_ELT(SDNode *N) {
394	// The value to insert may have a wider type than the vector element type,
395	// so be sure to truncate it to the element type if necessary.
396	SDValue Op = N->getOperand(Num: 1);
397	EVT EltVT = N->getValueType(ResNo: 0).getVectorElementType();
398	if (Op.getValueType() != EltVT)
399	// FIXME: Can this happen for floating point types?
400	Op = DAG.getNode(Opcode: ISD::TRUNCATE, DL: SDLoc(N), VT: EltVT, Operand: Op);
401	return Op;
402	}
403
404	SDValue DAGTypeLegalizer::ScalarizeVecRes_LOAD(LoadSDNode *N) {
405	assert(N->isUnindexed() && "Indexed vector load?");
406
407	SDValue Result = DAG.getLoad(
408	AM: ISD::UNINDEXED, ExtType: N->getExtensionType(),
409	VT: N->getValueType(ResNo: 0).getVectorElementType(), dl: SDLoc(N), Chain: N->getChain(),
410	Ptr: N->getBasePtr(), Offset: DAG.getUNDEF(VT: N->getBasePtr().getValueType()),
411	PtrInfo: N->getPointerInfo(), MemVT: N->getMemoryVT().getVectorElementType(),
412	Alignment: N->getOriginalAlign(), MMOFlags: N->getMemOperand()->getFlags(), AAInfo: N->getAAInfo());
413
414	// Legalize the chain result - switch anything that used the old chain to
415	// use the new one.
416	ReplaceValueWith(From: SDValue(N, 1), To: Result.getValue(R: 1));
417	return Result;
418	}
419
420	SDValue DAGTypeLegalizer::ScalarizeVecRes_UnaryOp(SDNode *N) {
421	// Get the dest type - it doesn't always match the input type, e.g. int_to_fp.
422	EVT DestVT = N->getValueType(ResNo: 0).getVectorElementType();
423	SDValue Op = N->getOperand(Num: 0);
424	EVT OpVT = Op.getValueType();
425	SDLoc DL(N);
426	// The result needs scalarizing, but it's not a given that the source does.
427	// This is a workaround for targets where it's impossible to scalarize the
428	// result of a conversion, because the source type is legal.
429	// For instance, this happens on AArch64: v1i1 is illegal but v1i{8,16,32}
430	// are widened to v8i8, v4i16, and v2i32, which is legal, because v1i64 is
431	// legal and was not scalarized.
432	// See the similar logic in ScalarizeVecRes_SETCC
433	if (getTypeAction(VT: OpVT) == TargetLowering::TypeScalarizeVector) {
434	Op = GetScalarizedVector(Op);
435	} else {
436	EVT VT = OpVT.getVectorElementType();
437	Op = DAG.getNode(Opcode: ISD::EXTRACT_VECTOR_ELT, DL, VT, N1: Op,
438	N2: DAG.getVectorIdxConstant(Val: 0, DL));
439	}
440	return DAG.getNode(Opcode: N->getOpcode(), DL: SDLoc(N), VT: DestVT, Operand: Op, Flags: N->getFlags());
441	}
442
443	SDValue DAGTypeLegalizer::ScalarizeVecRes_InregOp(SDNode *N) {
444	EVT EltVT = N->getValueType(ResNo: 0).getVectorElementType();
445	EVT ExtVT = cast<VTSDNode>(Val: N->getOperand(Num: 1))->getVT().getVectorElementType();
446	SDValue LHS = GetScalarizedVector(Op: N->getOperand(Num: 0));
447	return DAG.getNode(Opcode: N->getOpcode(), DL: SDLoc(N), VT: EltVT,
448	N1: LHS, N2: DAG.getValueType(ExtVT));
449	}
450
451	SDValue DAGTypeLegalizer::ScalarizeVecRes_VecInregOp(SDNode *N) {
452	SDLoc DL(N);
453	SDValue Op = N->getOperand(Num: 0);
454
455	EVT OpVT = Op.getValueType();
456	EVT OpEltVT = OpVT.getVectorElementType();
457	EVT EltVT = N->getValueType(ResNo: 0).getVectorElementType();
458
459	if (getTypeAction(VT: OpVT) == TargetLowering::TypeScalarizeVector) {
460	Op = GetScalarizedVector(Op);
461	} else {
462	Op = DAG.getNode(Opcode: ISD::EXTRACT_VECTOR_ELT, DL, VT: OpEltVT, N1: Op,
463	N2: DAG.getVectorIdxConstant(Val: 0, DL));
464	}
465
466	switch (N->getOpcode()) {
467	case ISD::ANY_EXTEND_VECTOR_INREG:
468	return DAG.getNode(Opcode: ISD::ANY_EXTEND, DL, VT: EltVT, Operand: Op);
469	case ISD::SIGN_EXTEND_VECTOR_INREG:
470	return DAG.getNode(Opcode: ISD::SIGN_EXTEND, DL, VT: EltVT, Operand: Op);
471	case ISD::ZERO_EXTEND_VECTOR_INREG:
472	return DAG.getNode(Opcode: ISD::ZERO_EXTEND, DL, VT: EltVT, Operand: Op);
473	}
474
475	llvm_unreachable("Illegal extend_vector_inreg opcode");
476	}
477
478	SDValue DAGTypeLegalizer::ScalarizeVecRes_SCALAR_TO_VECTOR(SDNode *N) {
479	// If the operand is wider than the vector element type then it is implicitly
480	// truncated. Make that explicit here.
481	EVT EltVT = N->getValueType(ResNo: 0).getVectorElementType();
482	SDValue InOp = N->getOperand(Num: 0);
483	if (InOp.getValueType() != EltVT)
484	return DAG.getNode(Opcode: ISD::TRUNCATE, DL: SDLoc(N), VT: EltVT, Operand: InOp);
485	return InOp;
486	}
487
488	SDValue DAGTypeLegalizer::ScalarizeVecRes_VSELECT(SDNode *N) {
489	SDValue Cond = N->getOperand(Num: 0);
490	EVT OpVT = Cond.getValueType();
491	SDLoc DL(N);
492	// The vselect result and true/value operands needs scalarizing, but it's
493	// not a given that the Cond does. For instance, in AVX512 v1i1 is legal.
494	// See the similar logic in ScalarizeVecRes_SETCC
495	if (getTypeAction(VT: OpVT) == TargetLowering::TypeScalarizeVector) {
496	Cond = GetScalarizedVector(Op: Cond);
497	} else {
498	EVT VT = OpVT.getVectorElementType();
499	Cond = DAG.getNode(Opcode: ISD::EXTRACT_VECTOR_ELT, DL, VT, N1: Cond,
500	N2: DAG.getVectorIdxConstant(Val: 0, DL));
501	}
502
503	SDValue LHS = GetScalarizedVector(Op: N->getOperand(Num: 1));
504	TargetLowering::BooleanContent ScalarBool =
505	TLI.getBooleanContents(isVec: false, isFloat: false);
506	TargetLowering::BooleanContent VecBool = TLI.getBooleanContents(isVec: true, isFloat: false);
507
508	// If integer and float booleans have different contents then we can't
509	// reliably optimize in all cases. There is a full explanation for this in
510	// DAGCombiner::visitSELECT() where the same issue affects folding
511	// (select C, 0, 1) to (xor C, 1).
512	if (TLI.getBooleanContents(isVec: false, isFloat: false) !=
513	TLI.getBooleanContents(isVec: false, isFloat: true)) {
514	// At least try the common case where the boolean is generated by a
515	// comparison.
516	if (Cond->getOpcode() == ISD::SETCC) {
517	EVT OpVT = Cond->getOperand(Num: 0).getValueType();
518	ScalarBool = TLI.getBooleanContents(Type: OpVT.getScalarType());
519	VecBool = TLI.getBooleanContents(Type: OpVT);
520	} else
521	ScalarBool = TargetLowering::UndefinedBooleanContent;
522	}
523
524	EVT CondVT = Cond.getValueType();
525	if (ScalarBool != VecBool) {
526	switch (ScalarBool) {
527	case TargetLowering::UndefinedBooleanContent:
528	break;
529	case TargetLowering::ZeroOrOneBooleanContent:
530	assert(VecBool == TargetLowering::UndefinedBooleanContent ||
531	VecBool == TargetLowering::ZeroOrNegativeOneBooleanContent);
532	// Vector read from all ones, scalar expects a single 1 so mask.
533	Cond = DAG.getNode(Opcode: ISD::AND, DL: SDLoc(N), VT: CondVT,
534	N1: Cond, N2: DAG.getConstant(Val: 1, DL: SDLoc(N), VT: CondVT));
535	break;
536	case TargetLowering::ZeroOrNegativeOneBooleanContent:
537	assert(VecBool == TargetLowering::UndefinedBooleanContent ||
538	VecBool == TargetLowering::ZeroOrOneBooleanContent);
539	// Vector reads from a one, scalar from all ones so sign extend.
540	Cond = DAG.getNode(ISD::SIGN_EXTEND_INREG, SDLoc(N), CondVT,
541	Cond, DAG.getValueType(MVT::i1));
542	break;
543	}
544	}
545
546	// Truncate the condition if needed
547	auto BoolVT = getSetCCResultType(VT: CondVT);
548	if (BoolVT.bitsLT(VT: CondVT))
549	Cond = DAG.getNode(Opcode: ISD::TRUNCATE, DL: SDLoc(N), VT: BoolVT, Operand: Cond);
550
551	return DAG.getSelect(DL: SDLoc(N),
552	VT: LHS.getValueType(), Cond, LHS,
553	RHS: GetScalarizedVector(Op: N->getOperand(Num: 2)));
554	}
555
556	SDValue DAGTypeLegalizer::ScalarizeVecRes_SELECT(SDNode *N) {
557	SDValue LHS = GetScalarizedVector(Op: N->getOperand(Num: 1));
558	return DAG.getSelect(DL: SDLoc(N),
559	VT: LHS.getValueType(), Cond: N->getOperand(Num: 0), LHS,
560	RHS: GetScalarizedVector(Op: N->getOperand(Num: 2)));
561	}
562
563	SDValue DAGTypeLegalizer::ScalarizeVecRes_SELECT_CC(SDNode *N) {
564	SDValue LHS = GetScalarizedVector(Op: N->getOperand(Num: 2));
565	return DAG.getNode(Opcode: ISD::SELECT_CC, DL: SDLoc(N), VT: LHS.getValueType(),
566	N1: N->getOperand(Num: 0), N2: N->getOperand(Num: 1),
567	N3: LHS, N4: GetScalarizedVector(Op: N->getOperand(Num: 3)),
568	N5: N->getOperand(Num: 4));
569	}
570
571	SDValue DAGTypeLegalizer::ScalarizeVecRes_UNDEF(SDNode *N) {
572	return DAG.getUNDEF(VT: N->getValueType(ResNo: 0).getVectorElementType());
573	}
574
575	SDValue DAGTypeLegalizer::ScalarizeVecRes_VECTOR_SHUFFLE(SDNode *N) {
576	// Figure out if the scalar is the LHS or RHS and return it.
577	SDValue Arg = N->getOperand(Num: 2).getOperand(i: 0);
578	if (Arg.isUndef())
579	return DAG.getUNDEF(VT: N->getValueType(ResNo: 0).getVectorElementType());
580	unsigned Op = !cast<ConstantSDNode>(Val&: Arg)->isZero();
581	return GetScalarizedVector(Op: N->getOperand(Num: Op));
582	}
583
584	SDValue DAGTypeLegalizer::ScalarizeVecRes_FP_TO_XINT_SAT(SDNode *N) {
585	SDValue Src = N->getOperand(Num: 0);
586	EVT SrcVT = Src.getValueType();
587	SDLoc dl(N);
588
589	// Handle case where result is scalarized but operand is not
590	if (getTypeAction(VT: SrcVT) == TargetLowering::TypeScalarizeVector)
591	Src = GetScalarizedVector(Op: Src);
592	else
593	Src = DAG.getNode(
594	Opcode: ISD::EXTRACT_VECTOR_ELT, DL: dl, VT: SrcVT.getVectorElementType(), N1: Src,
595	N2: DAG.getConstant(Val: 0, DL: dl, VT: TLI.getVectorIdxTy(DL: DAG.getDataLayout())));
596
597	EVT DstVT = N->getValueType(ResNo: 0).getVectorElementType();
598	return DAG.getNode(Opcode: N->getOpcode(), DL: dl, VT: DstVT, N1: Src, N2: N->getOperand(Num: 1));
599	}
600
601	SDValue DAGTypeLegalizer::ScalarizeVecRes_SETCC(SDNode *N) {
602	assert(N->getValueType(0).isVector() &&
603	N->getOperand(0).getValueType().isVector() &&
604	"Operand types must be vectors");
605	SDValue LHS = N->getOperand(Num: 0);
606	SDValue RHS = N->getOperand(Num: 1);
607	EVT OpVT = LHS.getValueType();
608	EVT NVT = N->getValueType(ResNo: 0).getVectorElementType();
609	SDLoc DL(N);
610
611	// The result needs scalarizing, but it's not a given that the source does.
612	if (getTypeAction(VT: OpVT) == TargetLowering::TypeScalarizeVector) {
613	LHS = GetScalarizedVector(Op: LHS);
614	RHS = GetScalarizedVector(Op: RHS);
615	} else {
616	EVT VT = OpVT.getVectorElementType();
617	LHS = DAG.getNode(Opcode: ISD::EXTRACT_VECTOR_ELT, DL, VT, N1: LHS,
618	N2: DAG.getVectorIdxConstant(Val: 0, DL));
619	RHS = DAG.getNode(Opcode: ISD::EXTRACT_VECTOR_ELT, DL, VT, N1: RHS,
620	N2: DAG.getVectorIdxConstant(Val: 0, DL));
621	}
622
623	// Turn it into a scalar SETCC.
624	SDValue Res = DAG.getNode(ISD::SETCC, DL, MVT::i1, LHS, RHS,
625	N->getOperand(Num: 2));
626	// Vectors may have a different boolean contents to scalars. Promote the
627	// value appropriately.
628	ISD::NodeType ExtendCode =
629	TargetLowering::getExtendForContent(Content: TLI.getBooleanContents(Type: OpVT));
630	return DAG.getNode(Opcode: ExtendCode, DL, VT: NVT, Operand: Res);
631	}
632
633	SDValue DAGTypeLegalizer::ScalarizeVecRes_IS_FPCLASS(SDNode *N) {
634	SDLoc DL(N);
635	SDValue Arg = N->getOperand(Num: 0);
636	SDValue Test = N->getOperand(Num: 1);
637	EVT ArgVT = Arg.getValueType();
638	EVT ResultVT = N->getValueType(ResNo: 0).getVectorElementType();
639
640	if (getTypeAction(VT: ArgVT) == TargetLowering::TypeScalarizeVector) {
641	Arg = GetScalarizedVector(Op: Arg);
642	} else {
643	EVT VT = ArgVT.getVectorElementType();
644	Arg = DAG.getNode(Opcode: ISD::EXTRACT_VECTOR_ELT, DL, VT, N1: Arg,
645	N2: DAG.getVectorIdxConstant(Val: 0, DL));
646	}
647
648	SDValue Res =
649	DAG.getNode(ISD::IS_FPCLASS, DL, MVT::i1, {Arg, Test}, N->getFlags());
650	// Vectors may have a different boolean contents to scalars. Promote the
651	// value appropriately.
652	ISD::NodeType ExtendCode =
653	TargetLowering::getExtendForContent(Content: TLI.getBooleanContents(Type: ArgVT));
654	return DAG.getNode(Opcode: ExtendCode, DL, VT: ResultVT, Operand: Res);
655	}
656
657	//===----------------------------------------------------------------------===//
658	// Operand Vector Scalarization <1 x ty> -> ty.
659	//===----------------------------------------------------------------------===//
660
661	bool DAGTypeLegalizer::ScalarizeVectorOperand(SDNode *N, unsigned OpNo) {
662	LLVM_DEBUG(dbgs() << "Scalarize node operand " << OpNo << ": ";
663	N->dump(&DAG));
664	SDValue Res = SDValue();
665
666	switch (N->getOpcode()) {
667	default:
668	#ifndef NDEBUG
669	dbgs() << "ScalarizeVectorOperand Op #" << OpNo << ": ";
670	N->dump(G: &DAG);
671	dbgs() << "\n";
672	#endif
673	report_fatal_error(reason: "Do not know how to scalarize this operator's "
674	"operand!\n");
675	case ISD::BITCAST:
676	Res = ScalarizeVecOp_BITCAST(N);
677	break;
678	case ISD::ANY_EXTEND:
679	case ISD::ZERO_EXTEND:
680	case ISD::SIGN_EXTEND:
681	case ISD::TRUNCATE:
682	case ISD::FP_TO_SINT:
683	case ISD::FP_TO_UINT:
684	case ISD::SINT_TO_FP:
685	case ISD::UINT_TO_FP:
686	case ISD::LRINT:
687	case ISD::LLRINT:
688	Res = ScalarizeVecOp_UnaryOp(N);
689	break;
690	case ISD::STRICT_SINT_TO_FP:
691	case ISD::STRICT_UINT_TO_FP:
692	case ISD::STRICT_FP_TO_SINT:
693	case ISD::STRICT_FP_TO_UINT:
694	Res = ScalarizeVecOp_UnaryOp_StrictFP(N);
695	break;
696	case ISD::CONCAT_VECTORS:
697	Res = ScalarizeVecOp_CONCAT_VECTORS(N);
698	break;
699	case ISD::EXTRACT_VECTOR_ELT:
700	Res = ScalarizeVecOp_EXTRACT_VECTOR_ELT(N);
701	break;
702	case ISD::VSELECT:
703	Res = ScalarizeVecOp_VSELECT(N);
704	break;
705	case ISD::SETCC:
706	Res = ScalarizeVecOp_VSETCC(N);
707	break;
708	case ISD::STORE:
709	Res = ScalarizeVecOp_STORE(N: cast<StoreSDNode>(Val: N), OpNo);
710	break;
711	case ISD::STRICT_FP_ROUND:
712	Res = ScalarizeVecOp_STRICT_FP_ROUND(N, OpNo);
713	break;
714	case ISD::FP_ROUND:
715	Res = ScalarizeVecOp_FP_ROUND(N, OpNo);
716	break;
717	case ISD::STRICT_FP_EXTEND:
718	Res = ScalarizeVecOp_STRICT_FP_EXTEND(N);
719	break;
720	case ISD::FP_EXTEND:
721	Res = ScalarizeVecOp_FP_EXTEND(N);
722	break;
723	case ISD::VECREDUCE_FADD:
724	case ISD::VECREDUCE_FMUL:
725	case ISD::VECREDUCE_ADD:
726	case ISD::VECREDUCE_MUL:
727	case ISD::VECREDUCE_AND:
728	case ISD::VECREDUCE_OR:
729	case ISD::VECREDUCE_XOR:
730	case ISD::VECREDUCE_SMAX:
731	case ISD::VECREDUCE_SMIN:
732	case ISD::VECREDUCE_UMAX:
733	case ISD::VECREDUCE_UMIN:
734	case ISD::VECREDUCE_FMAX:
735	case ISD::VECREDUCE_FMIN:
736	case ISD::VECREDUCE_FMAXIMUM:
737	case ISD::VECREDUCE_FMINIMUM:
738	Res = ScalarizeVecOp_VECREDUCE(N);
739	break;
740	case ISD::VECREDUCE_SEQ_FADD:
741	case ISD::VECREDUCE_SEQ_FMUL:
742	Res = ScalarizeVecOp_VECREDUCE_SEQ(N);
743	break;
744	}
745
746	// If the result is null, the sub-method took care of registering results etc.
747	if (!Res.getNode()) return false;
748
749	// If the result is N, the sub-method updated N in place. Tell the legalizer
750	// core about this.
751	if (Res.getNode() == N)
752	return true;
753
754	assert(Res.getValueType() == N->getValueType(0) && N->getNumValues() == 1 &&
755	"Invalid operand expansion");
756
757	ReplaceValueWith(From: SDValue(N, 0), To: Res);
758	return false;
759	}
760
761	/// If the value to convert is a vector that needs to be scalarized, it must be
762	/// <1 x ty>. Convert the element instead.
763	SDValue DAGTypeLegalizer::ScalarizeVecOp_BITCAST(SDNode *N) {
764	SDValue Elt = GetScalarizedVector(Op: N->getOperand(Num: 0));
765	return DAG.getNode(Opcode: ISD::BITCAST, DL: SDLoc(N),
766	VT: N->getValueType(ResNo: 0), Operand: Elt);
767	}
768
769	/// If the input is a vector that needs to be scalarized, it must be <1 x ty>.
770	/// Do the operation on the element instead.
771	SDValue DAGTypeLegalizer::ScalarizeVecOp_UnaryOp(SDNode *N) {
772	assert(N->getValueType(0).getVectorNumElements() == 1 &&
773	"Unexpected vector type!");
774	SDValue Elt = GetScalarizedVector(Op: N->getOperand(Num: 0));
775	SDValue Op = DAG.getNode(Opcode: N->getOpcode(), DL: SDLoc(N),
776	VT: N->getValueType(ResNo: 0).getScalarType(), Operand: Elt);
777	// Revectorize the result so the types line up with what the uses of this
778	// expression expect.
779	return DAG.getNode(Opcode: ISD::SCALAR_TO_VECTOR, DL: SDLoc(N), VT: N->getValueType(ResNo: 0), Operand: Op);
780	}
781
782	/// If the input is a vector that needs to be scalarized, it must be <1 x ty>.
783	/// Do the strict FP operation on the element instead.
784	SDValue DAGTypeLegalizer::ScalarizeVecOp_UnaryOp_StrictFP(SDNode *N) {
785	assert(N->getValueType(0).getVectorNumElements() == 1 &&
786	"Unexpected vector type!");
787	SDValue Elt = GetScalarizedVector(Op: N->getOperand(Num: 1));
788	SDValue Res = DAG.getNode(N->getOpcode(), SDLoc(N),
789	{ N->getValueType(0).getScalarType(), MVT::Other },
790	{ N->getOperand(0), Elt });
791	// Legalize the chain result - switch anything that used the old chain to
792	// use the new one.
793	ReplaceValueWith(From: SDValue(N, 1), To: Res.getValue(R: 1));
794	// Revectorize the result so the types line up with what the uses of this
795	// expression expect.
796	Res = DAG.getNode(Opcode: ISD::SCALAR_TO_VECTOR, DL: SDLoc(N), VT: N->getValueType(ResNo: 0), Operand: Res);
797
798	// Do our own replacement and return SDValue() to tell the caller that we
799	// handled all replacements since caller can only handle a single result.
800	ReplaceValueWith(From: SDValue(N, 0), To: Res);
801	return SDValue();
802	}
803
804	/// The vectors to concatenate have length one - use a BUILD_VECTOR instead.
805	SDValue DAGTypeLegalizer::ScalarizeVecOp_CONCAT_VECTORS(SDNode *N) {
806	SmallVector<SDValue, 8> Ops(N->getNumOperands());
807	for (unsigned i = 0, e = N->getNumOperands(); i < e; ++i)
808	Ops[i] = GetScalarizedVector(Op: N->getOperand(Num: i));
809	return DAG.getBuildVector(VT: N->getValueType(ResNo: 0), DL: SDLoc(N), Ops);
810	}
811
812	/// If the input is a vector that needs to be scalarized, it must be <1 x ty>,
813	/// so just return the element, ignoring the index.
814	SDValue DAGTypeLegalizer::ScalarizeVecOp_EXTRACT_VECTOR_ELT(SDNode *N) {
815	EVT VT = N->getValueType(ResNo: 0);
816	SDValue Res = GetScalarizedVector(Op: N->getOperand(Num: 0));
817	if (Res.getValueType() != VT)
818	Res = VT.isFloatingPoint()
819	? DAG.getNode(Opcode: ISD::FP_EXTEND, DL: SDLoc(N), VT, Operand: Res)
820	: DAG.getNode(Opcode: ISD::ANY_EXTEND, DL: SDLoc(N), VT, Operand: Res);
821	return Res;
822	}
823
824	/// If the input condition is a vector that needs to be scalarized, it must be
825	/// <1 x i1>, so just convert to a normal ISD::SELECT
826	/// (still with vector output type since that was acceptable if we got here).
827	SDValue DAGTypeLegalizer::ScalarizeVecOp_VSELECT(SDNode *N) {
828	SDValue ScalarCond = GetScalarizedVector(Op: N->getOperand(Num: 0));
829	EVT VT = N->getValueType(ResNo: 0);
830
831	return DAG.getNode(Opcode: ISD::SELECT, DL: SDLoc(N), VT, N1: ScalarCond, N2: N->getOperand(Num: 1),
832	N3: N->getOperand(Num: 2));
833	}
834
835	/// If the operand is a vector that needs to be scalarized then the
836	/// result must be v1i1, so just convert to a scalar SETCC and wrap
837	/// with a scalar_to_vector since the res type is legal if we got here
838	SDValue DAGTypeLegalizer::ScalarizeVecOp_VSETCC(SDNode *N) {
839	assert(N->getValueType(0).isVector() &&
840	N->getOperand(0).getValueType().isVector() &&
841	"Operand types must be vectors");
842	assert(N->getValueType(0) == MVT::v1i1 && "Expected v1i1 type");
843
844	EVT VT = N->getValueType(ResNo: 0);
845	SDValue LHS = GetScalarizedVector(Op: N->getOperand(Num: 0));
846	SDValue RHS = GetScalarizedVector(Op: N->getOperand(Num: 1));
847
848	EVT OpVT = N->getOperand(Num: 0).getValueType();
849	EVT NVT = VT.getVectorElementType();
850	SDLoc DL(N);
851	// Turn it into a scalar SETCC.
852	SDValue Res = DAG.getNode(ISD::SETCC, DL, MVT::i1, LHS, RHS,
853	N->getOperand(Num: 2));
854
855	// Vectors may have a different boolean contents to scalars. Promote the
856	// value appropriately.
857	ISD::NodeType ExtendCode =
858	TargetLowering::getExtendForContent(Content: TLI.getBooleanContents(Type: OpVT));
859
860	Res = DAG.getNode(Opcode: ExtendCode, DL, VT: NVT, Operand: Res);
861
862	return DAG.getNode(Opcode: ISD::SCALAR_TO_VECTOR, DL, VT, Operand: Res);
863	}
864
865	/// If the value to store is a vector that needs to be scalarized, it must be
866	/// <1 x ty>. Just store the element.
867	SDValue DAGTypeLegalizer::ScalarizeVecOp_STORE(StoreSDNode *N, unsigned OpNo){
868	assert(N->isUnindexed() && "Indexed store of one-element vector?");
869	assert(OpNo == 1 && "Do not know how to scalarize this operand!");
870	SDLoc dl(N);
871
872	if (N->isTruncatingStore())
873	return DAG.getTruncStore(
874	Chain: N->getChain(), dl, Val: GetScalarizedVector(Op: N->getOperand(Num: 1)),
875	Ptr: N->getBasePtr(), PtrInfo: N->getPointerInfo(),
876	SVT: N->getMemoryVT().getVectorElementType(), Alignment: N->getOriginalAlign(),
877	MMOFlags: N->getMemOperand()->getFlags(), AAInfo: N->getAAInfo());
878
879	return DAG.getStore(Chain: N->getChain(), dl, Val: GetScalarizedVector(Op: N->getOperand(Num: 1)),
880	Ptr: N->getBasePtr(), PtrInfo: N->getPointerInfo(),
881	Alignment: N->getOriginalAlign(), MMOFlags: N->getMemOperand()->getFlags(),
882	AAInfo: N->getAAInfo());
883	}
884
885	/// If the value to round is a vector that needs to be scalarized, it must be
886	/// <1 x ty>. Convert the element instead.
887	SDValue DAGTypeLegalizer::ScalarizeVecOp_FP_ROUND(SDNode *N, unsigned OpNo) {
888	assert(OpNo == 0 && "Wrong operand for scalarization!");
889	SDValue Elt = GetScalarizedVector(Op: N->getOperand(Num: 0));
890	SDValue Res = DAG.getNode(Opcode: ISD::FP_ROUND, DL: SDLoc(N),
891	VT: N->getValueType(ResNo: 0).getVectorElementType(), N1: Elt,
892	N2: N->getOperand(Num: 1));
893	return DAG.getNode(Opcode: ISD::SCALAR_TO_VECTOR, DL: SDLoc(N), VT: N->getValueType(ResNo: 0), Operand: Res);
894	}
895
896	SDValue DAGTypeLegalizer::ScalarizeVecOp_STRICT_FP_ROUND(SDNode *N,
897	unsigned OpNo) {
898	assert(OpNo == 1 && "Wrong operand for scalarization!");
899	SDValue Elt = GetScalarizedVector(Op: N->getOperand(Num: 1));
900	SDValue Res = DAG.getNode(ISD::STRICT_FP_ROUND, SDLoc(N),
901	{ N->getValueType(0).getVectorElementType(),
902	MVT::Other },
903	{ N->getOperand(0), Elt, N->getOperand(2) });
904	// Legalize the chain result - switch anything that used the old chain to
905	// use the new one.
906	ReplaceValueWith(From: SDValue(N, 1), To: Res.getValue(R: 1));
907
908	Res = DAG.getNode(Opcode: ISD::SCALAR_TO_VECTOR, DL: SDLoc(N), VT: N->getValueType(ResNo: 0), Operand: Res);
909
910	// Do our own replacement and return SDValue() to tell the caller that we
911	// handled all replacements since caller can only handle a single result.
912	ReplaceValueWith(From: SDValue(N, 0), To: Res);
913	return SDValue();
914	}
915
916	/// If the value to extend is a vector that needs to be scalarized, it must be
917	/// <1 x ty>. Convert the element instead.
918	SDValue DAGTypeLegalizer::ScalarizeVecOp_FP_EXTEND(SDNode *N) {
919	SDValue Elt = GetScalarizedVector(Op: N->getOperand(Num: 0));
920	SDValue Res = DAG.getNode(Opcode: ISD::FP_EXTEND, DL: SDLoc(N),
921	VT: N->getValueType(ResNo: 0).getVectorElementType(), Operand: Elt);
922	return DAG.getNode(Opcode: ISD::SCALAR_TO_VECTOR, DL: SDLoc(N), VT: N->getValueType(ResNo: 0), Operand: Res);
923	}
924
925	/// If the value to extend is a vector that needs to be scalarized, it must be
926	/// <1 x ty>. Convert the element instead.
927	SDValue DAGTypeLegalizer::ScalarizeVecOp_STRICT_FP_EXTEND(SDNode *N) {
928	SDValue Elt = GetScalarizedVector(Op: N->getOperand(Num: 1));
929	SDValue Res =
930	DAG.getNode(ISD::STRICT_FP_EXTEND, SDLoc(N),
931	{N->getValueType(0).getVectorElementType(), MVT::Other},
932	{N->getOperand(0), Elt});
933	// Legalize the chain result - switch anything that used the old chain to
934	// use the new one.
935	ReplaceValueWith(From: SDValue(N, 1), To: Res.getValue(R: 1));
936
937	Res = DAG.getNode(Opcode: ISD::SCALAR_TO_VECTOR, DL: SDLoc(N), VT: N->getValueType(ResNo: 0), Operand: Res);
938
939	// Do our own replacement and return SDValue() to tell the caller that we
940	// handled all replacements since caller can only handle a single result.
941	ReplaceValueWith(From: SDValue(N, 0), To: Res);
942	return SDValue();
943	}
944
945	SDValue DAGTypeLegalizer::ScalarizeVecOp_VECREDUCE(SDNode *N) {
946	SDValue Res = GetScalarizedVector(Op: N->getOperand(Num: 0));
947	// Result type may be wider than element type.
948	if (Res.getValueType() != N->getValueType(ResNo: 0))
949	Res = DAG.getNode(Opcode: ISD::ANY_EXTEND, DL: SDLoc(N), VT: N->getValueType(ResNo: 0), Operand: Res);
950	return Res;
951	}
952
953	SDValue DAGTypeLegalizer::ScalarizeVecOp_VECREDUCE_SEQ(SDNode *N) {
954	SDValue AccOp = N->getOperand(Num: 0);
955	SDValue VecOp = N->getOperand(Num: 1);
956
957	unsigned BaseOpc = ISD::getVecReduceBaseOpcode(VecReduceOpcode: N->getOpcode());
958
959	SDValue Op = GetScalarizedVector(Op: VecOp);
960	return DAG.getNode(Opcode: BaseOpc, DL: SDLoc(N), VT: N->getValueType(ResNo: 0),
961	N1: AccOp, N2: Op, Flags: N->getFlags());
962	}
963
964	//===----------------------------------------------------------------------===//
965	// Result Vector Splitting
966	//===----------------------------------------------------------------------===//
967
968	/// This method is called when the specified result of the specified node is
969	/// found to need vector splitting. At this point, the node may also have
970	/// invalid operands or may have other results that need legalization, we just
971	/// know that (at least) one result needs vector splitting.
972	void DAGTypeLegalizer::SplitVectorResult(SDNode *N, unsigned ResNo) {
973	LLVM_DEBUG(dbgs() << "Split node result: "; N->dump(&DAG));
974	SDValue Lo, Hi;
975
976	// See if the target wants to custom expand this node.
977	if (CustomLowerNode(N, VT: N->getValueType(ResNo), LegalizeResult: true))
978	return;
979
980	switch (N->getOpcode()) {
981	default:
982	#ifndef NDEBUG
983	dbgs() << "SplitVectorResult #" << ResNo << ": ";
984	N->dump(G: &DAG);
985	dbgs() << "\n";
986	#endif
987	report_fatal_error(reason: "Do not know how to split the result of this "
988	"operator!\n");
989
990	case ISD::MERGE_VALUES: SplitRes_MERGE_VALUES(N, ResNo, Lo, Hi); break;
991	case ISD::AssertZext: SplitVecRes_AssertZext(N, Lo, Hi); break;
992	case ISD::VSELECT:
993	case ISD::SELECT:
994	case ISD::VP_MERGE:
995	case ISD::VP_SELECT: SplitRes_Select(N, Lo, Hi); break;
996	case ISD::SELECT_CC: SplitRes_SELECT_CC(N, Lo, Hi); break;
997	case ISD::UNDEF: SplitRes_UNDEF(N, Lo, Hi); break;
998	case ISD::BITCAST: SplitVecRes_BITCAST(N, Lo, Hi); break;
999	case ISD::BUILD_VECTOR: SplitVecRes_BUILD_VECTOR(N, Lo, Hi); break;
1000	case ISD::CONCAT_VECTORS: SplitVecRes_CONCAT_VECTORS(N, Lo, Hi); break;
1001	case ISD::EXTRACT_SUBVECTOR: SplitVecRes_EXTRACT_SUBVECTOR(N, Lo, Hi); break;
1002	case ISD::INSERT_SUBVECTOR: SplitVecRes_INSERT_SUBVECTOR(N, Lo, Hi); break;
1003	case ISD::FPOWI:
1004	case ISD::FLDEXP:
1005	case ISD::FCOPYSIGN: SplitVecRes_FPOp_MultiType(N, Lo, Hi); break;
1006	case ISD::IS_FPCLASS: SplitVecRes_IS_FPCLASS(N, Lo, Hi); break;
1007	case ISD::INSERT_VECTOR_ELT: SplitVecRes_INSERT_VECTOR_ELT(N, Lo, Hi); break;
1008	case ISD::SPLAT_VECTOR:
1009	case ISD::SCALAR_TO_VECTOR:
1010	SplitVecRes_ScalarOp(N, Lo, Hi);
1011	break;
1012	case ISD::STEP_VECTOR:
1013	SplitVecRes_STEP_VECTOR(N, Lo, Hi);
1014	break;
1015	case ISD::SIGN_EXTEND_INREG: SplitVecRes_InregOp(N, Lo, Hi); break;
1016	case ISD::LOAD:
1017	SplitVecRes_LOAD(LD: cast<LoadSDNode>(Val: N), Lo, Hi);
1018	break;
1019	case ISD::VP_LOAD:
1020	SplitVecRes_VP_LOAD(LD: cast<VPLoadSDNode>(Val: N), Lo, Hi);
1021	break;
1022	case ISD::EXPERIMENTAL_VP_STRIDED_LOAD:
1023	SplitVecRes_VP_STRIDED_LOAD(SLD: cast<VPStridedLoadSDNode>(Val: N), Lo, Hi);
1024	break;
1025	case ISD::MLOAD:
1026	SplitVecRes_MLOAD(MLD: cast<MaskedLoadSDNode>(Val: N), Lo, Hi);
1027	break;
1028	case ISD::MGATHER:
1029	case ISD::VP_GATHER:
1030	SplitVecRes_Gather(VPGT: cast<MemSDNode>(Val: N), Lo, Hi, /*SplitSETCC*/ true);
1031	break;
1032	case ISD::SETCC:
1033	case ISD::VP_SETCC:
1034	SplitVecRes_SETCC(N, Lo, Hi);
1035	break;
1036	case ISD::VECTOR_REVERSE:
1037	SplitVecRes_VECTOR_REVERSE(N, Lo, Hi);
1038	break;
1039	case ISD::VECTOR_SHUFFLE:
1040	SplitVecRes_VECTOR_SHUFFLE(N: cast<ShuffleVectorSDNode>(Val: N), Lo, Hi);
1041	break;
1042	case ISD::VECTOR_SPLICE:
1043	SplitVecRes_VECTOR_SPLICE(N, Lo, Hi);
1044	break;
1045	case ISD::VECTOR_DEINTERLEAVE:
1046	SplitVecRes_VECTOR_DEINTERLEAVE(N);
1047	return;
1048	case ISD::VECTOR_INTERLEAVE:
1049	SplitVecRes_VECTOR_INTERLEAVE(N);
1050	return;
1051	case ISD::VAARG:
1052	SplitVecRes_VAARG(N, Lo, Hi);
1053	break;
1054
1055	case ISD::ANY_EXTEND_VECTOR_INREG:
1056	case ISD::SIGN_EXTEND_VECTOR_INREG:
1057	case ISD::ZERO_EXTEND_VECTOR_INREG:
1058	SplitVecRes_ExtVecInRegOp(N, Lo, Hi);
1059	break;
1060
1061	case ISD::ABS:
1062	case ISD::VP_ABS:
1063	case ISD::BITREVERSE:
1064	case ISD::VP_BITREVERSE:
1065	case ISD::BSWAP:
1066	case ISD::VP_BSWAP:
1067	case ISD::CTLZ:
1068	case ISD::VP_CTLZ:
1069	case ISD::CTTZ:
1070	case ISD::VP_CTTZ:
1071	case ISD::CTLZ_ZERO_UNDEF:
1072	case ISD::VP_CTLZ_ZERO_UNDEF:
1073	case ISD::CTTZ_ZERO_UNDEF:
1074	case ISD::VP_CTTZ_ZERO_UNDEF:
1075	case ISD::CTPOP:
1076	case ISD::VP_CTPOP:
1077	case ISD::FABS: case ISD::VP_FABS:
1078	case ISD::FCEIL:
1079	case ISD::VP_FCEIL:
1080	case ISD::FCOS:
1081	case ISD::FEXP:
1082	case ISD::FEXP2:
1083	case ISD::FEXP10:
1084	case ISD::FFLOOR:
1085	case ISD::VP_FFLOOR:
1086	case ISD::FLOG:
1087	case ISD::FLOG10:
1088	case ISD::FLOG2:
1089	case ISD::FNEARBYINT:
1090	case ISD::VP_FNEARBYINT:
1091	case ISD::FNEG: case ISD::VP_FNEG:
1092	case ISD::FREEZE:
1093	case ISD::ARITH_FENCE:
1094	case ISD::FP_EXTEND:
1095	case ISD::VP_FP_EXTEND:
1096	case ISD::FP_ROUND:
1097	case ISD::VP_FP_ROUND:
1098	case ISD::FP_TO_SINT:
1099	case ISD::VP_FP_TO_SINT:
1100	case ISD::FP_TO_UINT:
1101	case ISD::VP_FP_TO_UINT:
1102	case ISD::FRINT:
1103	case ISD::VP_FRINT:
1104	case ISD::LRINT:
1105	case ISD::VP_LRINT:
1106	case ISD::LLRINT:
1107	case ISD::VP_LLRINT:
1108	case ISD::FROUND:
1109	case ISD::VP_FROUND:
1110	case ISD::FROUNDEVEN:
1111	case ISD::VP_FROUNDEVEN:
1112	case ISD::FSIN:
1113	case ISD::FSQRT: case ISD::VP_SQRT:
1114	case ISD::FTRUNC:
1115	case ISD::VP_FROUNDTOZERO:
1116	case ISD::SINT_TO_FP:
1117	case ISD::VP_SINT_TO_FP:
1118	case ISD::TRUNCATE:
1119	case ISD::VP_TRUNCATE:
1120	case ISD::UINT_TO_FP:
1121	case ISD::VP_UINT_TO_FP:
1122	case ISD::FCANONICALIZE:
1123	SplitVecRes_UnaryOp(N, Lo, Hi);
1124	break;
1125	case ISD::FFREXP:
1126	SplitVecRes_FFREXP(N, ResNo, Lo, Hi);
1127	break;
1128
1129	case ISD::ANY_EXTEND:
1130	case ISD::SIGN_EXTEND:
1131	case ISD::ZERO_EXTEND:
1132	case ISD::VP_SIGN_EXTEND:
1133	case ISD::VP_ZERO_EXTEND:
1134	SplitVecRes_ExtendOp(N, Lo, Hi);
1135	break;
1136
1137	case ISD::ADD: case ISD::VP_ADD:
1138	case ISD::SUB: case ISD::VP_SUB:
1139	case ISD::MUL: case ISD::VP_MUL:
1140	case ISD::MULHS:
1141	case ISD::MULHU:
1142	case ISD::FADD: case ISD::VP_FADD:
1143	case ISD::FSUB: case ISD::VP_FSUB:
1144	case ISD::FMUL: case ISD::VP_FMUL:
1145	case ISD::FMINNUM: case ISD::VP_FMINNUM:
1146	case ISD::FMAXNUM: case ISD::VP_FMAXNUM:
1147	case ISD::FMINIMUM:
1148	case ISD::VP_FMINIMUM:
1149	case ISD::FMAXIMUM:
1150	case ISD::VP_FMAXIMUM:
1151	case ISD::SDIV: case ISD::VP_SDIV:
1152	case ISD::UDIV: case ISD::VP_UDIV:
1153	case ISD::FDIV: case ISD::VP_FDIV:
1154	case ISD::FPOW:
1155	case ISD::AND: case ISD::VP_AND:
1156	case ISD::OR: case ISD::VP_OR:
1157	case ISD::XOR: case ISD::VP_XOR:
1158	case ISD::SHL: case ISD::VP_SHL:
1159	case ISD::SRA: case ISD::VP_ASHR:
1160	case ISD::SRL: case ISD::VP_LSHR:
1161	case ISD::UREM: case ISD::VP_UREM:
1162	case ISD::SREM: case ISD::VP_SREM:
1163	case ISD::FREM: case ISD::VP_FREM:
1164	case ISD::SMIN: case ISD::VP_SMIN:
1165	case ISD::SMAX: case ISD::VP_SMAX:
1166	case ISD::UMIN: case ISD::VP_UMIN:
1167	case ISD::UMAX: case ISD::VP_UMAX:
1168	case ISD::SADDSAT: case ISD::VP_SADDSAT:
1169	case ISD::UADDSAT: case ISD::VP_UADDSAT:
1170	case ISD::SSUBSAT: case ISD::VP_SSUBSAT:
1171	case ISD::USUBSAT: case ISD::VP_USUBSAT:
1172	case ISD::SSHLSAT:
1173	case ISD::USHLSAT:
1174	case ISD::ROTL:
1175	case ISD::ROTR:
1176	case ISD::VP_FCOPYSIGN:
1177	SplitVecRes_BinOp(N, Lo, Hi);
1178	break;
1179	case ISD::FMA: case ISD::VP_FMA:
1180	case ISD::FSHL:
1181	case ISD::VP_FSHL:
1182	case ISD::FSHR:
1183	case ISD::VP_FSHR:
1184	SplitVecRes_TernaryOp(N, Lo, Hi);
1185	break;
1186
1187	#define DAG_INSTRUCTION(NAME, NARG, ROUND_MODE, INTRINSIC, DAGN) \
1188	case ISD::STRICT_##DAGN:
1189	#include "llvm/IR/ConstrainedOps.def"
1190	SplitVecRes_StrictFPOp(N, Lo, Hi);
1191	break;
1192
1193	case ISD::FP_TO_UINT_SAT:
1194	case ISD::FP_TO_SINT_SAT:
1195	SplitVecRes_FP_TO_XINT_SAT(N, Lo, Hi);
1196	break;
1197
1198	case ISD::UADDO:
1199	case ISD::SADDO:
1200	case ISD::USUBO:
1201	case ISD::SSUBO:
1202	case ISD::UMULO:
1203	case ISD::SMULO:
1204	SplitVecRes_OverflowOp(N, ResNo, Lo, Hi);
1205	break;
1206	case ISD::SMULFIX:
1207	case ISD::SMULFIXSAT:
1208	case ISD::UMULFIX:
1209	case ISD::UMULFIXSAT:
1210	case ISD::SDIVFIX:
1211	case ISD::SDIVFIXSAT:
1212	case ISD::UDIVFIX:
1213	case ISD::UDIVFIXSAT:
1214	SplitVecRes_FIX(N, Lo, Hi);
1215	break;
1216	case ISD::EXPERIMENTAL_VP_REVERSE:
1217	SplitVecRes_VP_REVERSE(N, Lo, Hi);
1218	break;
1219	}
1220
1221	// If Lo/Hi is null, the sub-method took care of registering results etc.
1222	if (Lo.getNode())
1223	SetSplitVector(Op: SDValue(N, ResNo), Lo, Hi);
1224	}
1225
1226	void DAGTypeLegalizer::IncrementPointer(MemSDNode *N, EVT MemVT,
1227	MachinePointerInfo &MPI, SDValue &Ptr,
1228	uint64_t *ScaledOffset) {
1229	SDLoc DL(N);
1230	unsigned IncrementSize = MemVT.getSizeInBits().getKnownMinValue() / 8;
1231
1232	if (MemVT.isScalableVector()) {
1233	SDNodeFlags Flags;
1234	SDValue BytesIncrement = DAG.getVScale(
1235	DL, VT: Ptr.getValueType(),
1236	MulImm: APInt(Ptr.getValueSizeInBits().getFixedValue(), IncrementSize));
1237	MPI = MachinePointerInfo(N->getPointerInfo().getAddrSpace());
1238	Flags.setNoUnsignedWrap(true);
1239	if (ScaledOffset)
1240	*ScaledOffset += IncrementSize;
1241	Ptr = DAG.getNode(Opcode: ISD::ADD, DL, VT: Ptr.getValueType(), N1: Ptr, N2: BytesIncrement,
1242	Flags);
1243	} else {
1244	MPI = N->getPointerInfo().getWithOffset(O: IncrementSize);
1245	// Increment the pointer to the other half.
1246	Ptr = DAG.getObjectPtrOffset(SL: DL, Ptr, Offset: TypeSize::getFixed(ExactSize: IncrementSize));
1247	}
1248	}
1249
1250	std::pair<SDValue, SDValue> DAGTypeLegalizer::SplitMask(SDValue Mask) {
1251	return SplitMask(Mask, DL: SDLoc(Mask));
1252	}
1253
1254	std::pair<SDValue, SDValue> DAGTypeLegalizer::SplitMask(SDValue Mask,
1255	const SDLoc &DL) {
1256	SDValue MaskLo, MaskHi;
1257	EVT MaskVT = Mask.getValueType();
1258	if (getTypeAction(VT: MaskVT) == TargetLowering::TypeSplitVector)
1259	GetSplitVector(Op: Mask, Lo&: MaskLo, Hi&: MaskHi);
1260	else
1261	std::tie(args&: MaskLo, args&: MaskHi) = DAG.SplitVector(N: Mask, DL);
1262	return std::make_pair(x&: MaskLo, y&: MaskHi);
1263	}
1264
1265	void DAGTypeLegalizer::SplitVecRes_BinOp(SDNode *N, SDValue &Lo, SDValue &Hi) {
1266	SDValue LHSLo, LHSHi;
1267	GetSplitVector(Op: N->getOperand(Num: 0), Lo&: LHSLo, Hi&: LHSHi);
1268	SDValue RHSLo, RHSHi;
1269	GetSplitVector(Op: N->getOperand(Num: 1), Lo&: RHSLo, Hi&: RHSHi);
1270	SDLoc dl(N);
1271
1272	const SDNodeFlags Flags = N->getFlags();
1273	unsigned Opcode = N->getOpcode();
1274	if (N->getNumOperands() == 2) {
1275	Lo = DAG.getNode(Opcode, DL: dl, VT: LHSLo.getValueType(), N1: LHSLo, N2: RHSLo, Flags);
1276	Hi = DAG.getNode(Opcode, DL: dl, VT: LHSHi.getValueType(), N1: LHSHi, N2: RHSHi, Flags);
1277	return;
1278	}
1279
1280	assert(N->getNumOperands() == 4 && "Unexpected number of operands!");
1281	assert(N->isVPOpcode() && "Expected VP opcode");
1282
1283	SDValue MaskLo, MaskHi;
1284	std::tie(args&: MaskLo, args&: MaskHi) = SplitMask(Mask: N->getOperand(Num: 2));
1285
1286	SDValue EVLLo, EVLHi;
1287	std::tie(args&: EVLLo, args&: EVLHi) =
1288	DAG.SplitEVL(N: N->getOperand(Num: 3), VecVT: N->getValueType(ResNo: 0), DL: dl);
1289
1290	Lo = DAG.getNode(Opcode, DL: dl, VT: LHSLo.getValueType(),
1291	Ops: {LHSLo, RHSLo, MaskLo, EVLLo}, Flags);
1292	Hi = DAG.getNode(Opcode, DL: dl, VT: LHSHi.getValueType(),
1293	Ops: {LHSHi, RHSHi, MaskHi, EVLHi}, Flags);
1294	}
1295
1296	void DAGTypeLegalizer::SplitVecRes_TernaryOp(SDNode *N, SDValue &Lo,
1297	SDValue &Hi) {
1298	SDValue Op0Lo, Op0Hi;
1299	GetSplitVector(Op: N->getOperand(Num: 0), Lo&: Op0Lo, Hi&: Op0Hi);
1300	SDValue Op1Lo, Op1Hi;
1301	GetSplitVector(Op: N->getOperand(Num: 1), Lo&: Op1Lo, Hi&: Op1Hi);
1302	SDValue Op2Lo, Op2Hi;
1303	GetSplitVector(Op: N->getOperand(Num: 2), Lo&: Op2Lo, Hi&: Op2Hi);
1304	SDLoc dl(N);
1305
1306	const SDNodeFlags Flags = N->getFlags();
1307	unsigned Opcode = N->getOpcode();
1308	if (N->getNumOperands() == 3) {
1309	Lo = DAG.getNode(Opcode, DL: dl, VT: Op0Lo.getValueType(), N1: Op0Lo, N2: Op1Lo, N3: Op2Lo, Flags);
1310	Hi = DAG.getNode(Opcode, DL: dl, VT: Op0Hi.getValueType(), N1: Op0Hi, N2: Op1Hi, N3: Op2Hi, Flags);
1311	return;
1312	}
1313
1314	assert(N->getNumOperands() == 5 && "Unexpected number of operands!");
1315	assert(N->isVPOpcode() && "Expected VP opcode");
1316
1317	SDValue MaskLo, MaskHi;
1318	std::tie(args&: MaskLo, args&: MaskHi) = SplitMask(Mask: N->getOperand(Num: 3));
1319
1320	SDValue EVLLo, EVLHi;
1321	std::tie(args&: EVLLo, args&: EVLHi) =
1322	DAG.SplitEVL(N: N->getOperand(Num: 4), VecVT: N->getValueType(ResNo: 0), DL: dl);
1323
1324	Lo = DAG.getNode(Opcode, DL: dl, VT: Op0Lo.getValueType(),
1325	Ops: {Op0Lo, Op1Lo, Op2Lo, MaskLo, EVLLo}, Flags);
1326	Hi = DAG.getNode(Opcode, DL: dl, VT: Op0Hi.getValueType(),
1327	Ops: {Op0Hi, Op1Hi, Op2Hi, MaskHi, EVLHi}, Flags);
1328	}
1329
1330	void DAGTypeLegalizer::SplitVecRes_FIX(SDNode *N, SDValue &Lo, SDValue &Hi) {
1331	SDValue LHSLo, LHSHi;
1332	GetSplitVector(Op: N->getOperand(Num: 0), Lo&: LHSLo, Hi&: LHSHi);
1333	SDValue RHSLo, RHSHi;
1334	GetSplitVector(Op: N->getOperand(Num: 1), Lo&: RHSLo, Hi&: RHSHi);
1335	SDLoc dl(N);
1336	SDValue Op2 = N->getOperand(Num: 2);
1337
1338	unsigned Opcode = N->getOpcode();
1339	Lo = DAG.getNode(Opcode, DL: dl, VT: LHSLo.getValueType(), N1: LHSLo, N2: RHSLo, N3: Op2,
1340	Flags: N->getFlags());
1341	Hi = DAG.getNode(Opcode, DL: dl, VT: LHSHi.getValueType(), N1: LHSHi, N2: RHSHi, N3: Op2,
1342	Flags: N->getFlags());
1343	}
1344
1345	void DAGTypeLegalizer::SplitVecRes_BITCAST(SDNode *N, SDValue &Lo,
1346	SDValue &Hi) {
1347	// We know the result is a vector. The input may be either a vector or a
1348	// scalar value.
1349	EVT LoVT, HiVT;
1350	std::tie(args&: LoVT, args&: HiVT) = DAG.GetSplitDestVTs(VT: N->getValueType(ResNo: 0));
1351	SDLoc dl(N);
1352
1353	SDValue InOp = N->getOperand(Num: 0);
1354	EVT InVT = InOp.getValueType();
1355
1356	// Handle some special cases efficiently.
1357	switch (getTypeAction(VT: InVT)) {
1358	case TargetLowering::TypeLegal:
1359	case TargetLowering::TypePromoteInteger:
1360	case TargetLowering::TypePromoteFloat:
1361	case TargetLowering::TypeSoftPromoteHalf:
1362	case TargetLowering::TypeSoftenFloat:
1363	case TargetLowering::TypeScalarizeVector:
1364	case TargetLowering::TypeWidenVector:
1365	break;
1366	case TargetLowering::TypeExpandInteger:
1367	case TargetLowering::TypeExpandFloat:
1368	// A scalar to vector conversion, where the scalar needs expansion.
1369	// If the vector is being split in two then we can just convert the
1370	// expanded pieces.
1371	if (LoVT == HiVT) {
1372	GetExpandedOp(Op: InOp, Lo, Hi);
1373	if (DAG.getDataLayout().isBigEndian())
1374	std::swap(a&: Lo, b&: Hi);
1375	Lo = DAG.getNode(Opcode: ISD::BITCAST, DL: dl, VT: LoVT, Operand: Lo);
1376	Hi = DAG.getNode(Opcode: ISD::BITCAST, DL: dl, VT: HiVT, Operand: Hi);
1377	return;
1378	}
1379	break;
1380	case TargetLowering::TypeSplitVector:
1381	// If the input is a vector that needs to be split, convert each split
1382	// piece of the input now.
1383	GetSplitVector(Op: InOp, Lo, Hi);
1384	Lo = DAG.getNode(Opcode: ISD::BITCAST, DL: dl, VT: LoVT, Operand: Lo);
1385	Hi = DAG.getNode(Opcode: ISD::BITCAST, DL: dl, VT: HiVT, Operand: Hi);
1386	return;
1387	case TargetLowering::TypeScalarizeScalableVector:
1388	report_fatal_error(reason: "Scalarization of scalable vectors is not supported.");
1389	}
1390
1391	if (LoVT.isScalableVector()) {
1392	auto [InLo, InHi] = DAG.SplitVectorOperand(N, OpNo: 0);
1393	Lo = DAG.getNode(Opcode: ISD::BITCAST, DL: dl, VT: LoVT, Operand: InLo);
1394	Hi = DAG.getNode(Opcode: ISD::BITCAST, DL: dl, VT: HiVT, Operand: InHi);
1395	return;
1396	}
1397
1398	// In the general case, convert the input to an integer and split it by hand.
1399	EVT LoIntVT = EVT::getIntegerVT(Context&: *DAG.getContext(), BitWidth: LoVT.getSizeInBits());
1400	EVT HiIntVT = EVT::getIntegerVT(Context&: *DAG.getContext(), BitWidth: HiVT.getSizeInBits());
1401	if (DAG.getDataLayout().isBigEndian())
1402	std::swap(a&: LoIntVT, b&: HiIntVT);
1403
1404	SplitInteger(Op: BitConvertToInteger(Op: InOp), LoVT: LoIntVT, HiVT: HiIntVT, Lo, Hi);
1405
1406	if (DAG.getDataLayout().isBigEndian())
1407	std::swap(a&: Lo, b&: Hi);
1408	Lo = DAG.getNode(Opcode: ISD::BITCAST, DL: dl, VT: LoVT, Operand: Lo);
1409	Hi = DAG.getNode(Opcode: ISD::BITCAST, DL: dl, VT: HiVT, Operand: Hi);
1410	}
1411
1412	void DAGTypeLegalizer::SplitVecRes_BUILD_VECTOR(SDNode *N, SDValue &Lo,
1413	SDValue &Hi) {
1414	EVT LoVT, HiVT;
1415	SDLoc dl(N);
1416	std::tie(args&: LoVT, args&: HiVT) = DAG.GetSplitDestVTs(VT: N->getValueType(ResNo: 0));
1417	unsigned LoNumElts = LoVT.getVectorNumElements();
1418	SmallVector<SDValue, 8> LoOps(N->op_begin(), N->op_begin()+LoNumElts);
1419	Lo = DAG.getBuildVector(VT: LoVT, DL: dl, Ops: LoOps);
1420
1421	SmallVector<SDValue, 8> HiOps(N->op_begin()+LoNumElts, N->op_end());
1422	Hi = DAG.getBuildVector(VT: HiVT, DL: dl, Ops: HiOps);
1423	}
1424
1425	void DAGTypeLegalizer::SplitVecRes_CONCAT_VECTORS(SDNode *N, SDValue &Lo,
1426	SDValue &Hi) {
1427	assert(!(N->getNumOperands() & 1) && "Unsupported CONCAT_VECTORS");
1428	SDLoc dl(N);
1429	unsigned NumSubvectors = N->getNumOperands() / 2;
1430	if (NumSubvectors == 1) {
1431	Lo = N->getOperand(Num: 0);
1432	Hi = N->getOperand(Num: 1);
1433	return;
1434	}
1435
1436	EVT LoVT, HiVT;
1437	std::tie(args&: LoVT, args&: HiVT) = DAG.GetSplitDestVTs(VT: N->getValueType(ResNo: 0));
1438
1439	SmallVector<SDValue, 8> LoOps(N->op_begin(), N->op_begin()+NumSubvectors);
1440	Lo = DAG.getNode(Opcode: ISD::CONCAT_VECTORS, DL: dl, VT: LoVT, Ops: LoOps);
1441
1442	SmallVector<SDValue, 8> HiOps(N->op_begin()+NumSubvectors, N->op_end());
1443	Hi = DAG.getNode(Opcode: ISD::CONCAT_VECTORS, DL: dl, VT: HiVT, Ops: HiOps);
1444	}
1445
1446	void DAGTypeLegalizer::SplitVecRes_EXTRACT_SUBVECTOR(SDNode *N, SDValue &Lo,
1447	SDValue &Hi) {
1448	SDValue Vec = N->getOperand(Num: 0);
1449	SDValue Idx = N->getOperand(Num: 1);
1450	SDLoc dl(N);
1451
1452	EVT LoVT, HiVT;
1453	std::tie(args&: LoVT, args&: HiVT) = DAG.GetSplitDestVTs(VT: N->getValueType(ResNo: 0));
1454
1455	Lo = DAG.getNode(Opcode: ISD::EXTRACT_SUBVECTOR, DL: dl, VT: LoVT, N1: Vec, N2: Idx);
1456	uint64_t IdxVal = Idx->getAsZExtVal();
1457	Hi = DAG.getNode(
1458	Opcode: ISD::EXTRACT_SUBVECTOR, DL: dl, VT: HiVT, N1: Vec,
1459	N2: DAG.getVectorIdxConstant(Val: IdxVal + LoVT.getVectorMinNumElements(), DL: dl));
1460	}
1461
1462	void DAGTypeLegalizer::SplitVecRes_INSERT_SUBVECTOR(SDNode *N, SDValue &Lo,
1463	SDValue &Hi) {
1464	SDValue Vec = N->getOperand(Num: 0);
1465	SDValue SubVec = N->getOperand(Num: 1);
1466	SDValue Idx = N->getOperand(Num: 2);
1467	SDLoc dl(N);
1468	GetSplitVector(Op: Vec, Lo, Hi);
1469
1470	EVT VecVT = Vec.getValueType();
1471	EVT LoVT = Lo.getValueType();
1472	EVT SubVecVT = SubVec.getValueType();
1473	unsigned VecElems = VecVT.getVectorMinNumElements();
1474	unsigned SubElems = SubVecVT.getVectorMinNumElements();
1475	unsigned LoElems = LoVT.getVectorMinNumElements();
1476
1477	// If we know the index is in the first half, and we know the subvector
1478	// doesn't cross the boundary between the halves, we can avoid spilling the
1479	// vector, and insert into the lower half of the split vector directly.
1480	unsigned IdxVal = Idx->getAsZExtVal();
1481	if (IdxVal + SubElems <= LoElems) {
1482	Lo = DAG.getNode(Opcode: ISD::INSERT_SUBVECTOR, DL: dl, VT: LoVT, N1: Lo, N2: SubVec, N3: Idx);
1483	return;
1484	}
1485	// Similarly if the subvector is fully in the high half, but mind that we
1486	// can't tell whether a fixed-length subvector is fully within the high half
1487	// of a scalable vector.
1488	if (VecVT.isScalableVector() == SubVecVT.isScalableVector() &&
1489	IdxVal >= LoElems && IdxVal + SubElems <= VecElems) {
1490	Hi = DAG.getNode(Opcode: ISD::INSERT_SUBVECTOR, DL: dl, VT: Hi.getValueType(), N1: Hi, N2: SubVec,
1491	N3: DAG.getVectorIdxConstant(Val: IdxVal - LoElems, DL: dl));
1492	return;
1493	}
1494
1495	// Spill the vector to the stack.
1496	// In cases where the vector is illegal it will be broken down into parts
1497	// and stored in parts - we should use the alignment for the smallest part.
1498	Align SmallestAlign = DAG.getReducedAlign(VT: VecVT, /*UseABI=*/false);
1499	SDValue StackPtr =
1500	DAG.CreateStackTemporary(Bytes: VecVT.getStoreSize(), Alignment: SmallestAlign);
1501	auto &MF = DAG.getMachineFunction();
1502	auto FrameIndex = cast<FrameIndexSDNode>(Val: StackPtr.getNode())->getIndex();
1503	auto PtrInfo = MachinePointerInfo::getFixedStack(MF, FI: FrameIndex);
1504
1505	SDValue Store = DAG.getStore(Chain: DAG.getEntryNode(), dl, Val: Vec, Ptr: StackPtr, PtrInfo,
1506	Alignment: SmallestAlign);
1507
1508	// Store the new subvector into the specified index.
1509	SDValue SubVecPtr =
1510	TLI.getVectorSubVecPointer(DAG, VecPtr: StackPtr, VecVT, SubVecVT, Index: Idx);
1511	Store = DAG.getStore(Chain: Store, dl, Val: SubVec, Ptr: SubVecPtr,
1512	PtrInfo: MachinePointerInfo::getUnknownStack(MF));
1513
1514	// Load the Lo part from the stack slot.
1515	Lo = DAG.getLoad(VT: Lo.getValueType(), dl, Chain: Store, Ptr: StackPtr, PtrInfo,
1516	Alignment: SmallestAlign);
1517
1518	// Increment the pointer to the other part.
1519	auto *Load = cast<LoadSDNode>(Val&: Lo);
1520	MachinePointerInfo MPI = Load->getPointerInfo();
1521	IncrementPointer(N: Load, MemVT: LoVT, MPI, Ptr&: StackPtr);
1522
1523	// Load the Hi part from the stack slot.
1524	Hi = DAG.getLoad(VT: Hi.getValueType(), dl, Chain: Store, Ptr: StackPtr, PtrInfo: MPI, Alignment: SmallestAlign);
1525	}
1526
1527	// Handle splitting an FP where the second operand does not match the first
1528	// type. The second operand may be a scalar, or a vector that has exactly as
1529	// many elements as the first
1530	void DAGTypeLegalizer::SplitVecRes_FPOp_MultiType(SDNode *N, SDValue &Lo,
1531	SDValue &Hi) {
1532	SDValue LHSLo, LHSHi;
1533	GetSplitVector(Op: N->getOperand(Num: 0), Lo&: LHSLo, Hi&: LHSHi);
1534	SDLoc DL(N);
1535
1536	SDValue RHSLo, RHSHi;
1537	SDValue RHS = N->getOperand(Num: 1);
1538	EVT RHSVT = RHS.getValueType();
1539	if (RHSVT.isVector()) {
1540	if (getTypeAction(VT: RHSVT) == TargetLowering::TypeSplitVector)
1541	GetSplitVector(Op: RHS, Lo&: RHSLo, Hi&: RHSHi);
1542	else
1543	std::tie(args&: RHSLo, args&: RHSHi) = DAG.SplitVector(N: RHS, DL: SDLoc(RHS));
1544
1545	Lo = DAG.getNode(Opcode: N->getOpcode(), DL, VT: LHSLo.getValueType(), N1: LHSLo, N2: RHSLo);
1546	Hi = DAG.getNode(Opcode: N->getOpcode(), DL, VT: LHSHi.getValueType(), N1: LHSHi, N2: RHSHi);
1547	} else {
1548	Lo = DAG.getNode(Opcode: N->getOpcode(), DL, VT: LHSLo.getValueType(), N1: LHSLo, N2: RHS);
1549	Hi = DAG.getNode(Opcode: N->getOpcode(), DL, VT: LHSHi.getValueType(), N1: LHSHi, N2: RHS);
1550	}
1551	}
1552
1553	void DAGTypeLegalizer::SplitVecRes_IS_FPCLASS(SDNode *N, SDValue &Lo,
1554	SDValue &Hi) {
1555	SDLoc DL(N);
1556	SDValue ArgLo, ArgHi;
1557	SDValue Test = N->getOperand(Num: 1);
1558	SDValue FpValue = N->getOperand(Num: 0);
1559	if (getTypeAction(VT: FpValue.getValueType()) == TargetLowering::TypeSplitVector)
1560	GetSplitVector(Op: FpValue, Lo&: ArgLo, Hi&: ArgHi);
1561	else
1562	std::tie(args&: ArgLo, args&: ArgHi) = DAG.SplitVector(N: FpValue, DL: SDLoc(FpValue));
1563	EVT LoVT, HiVT;
1564	std::tie(args&: LoVT, args&: HiVT) = DAG.GetSplitDestVTs(VT: N->getValueType(ResNo: 0));
1565
1566	Lo = DAG.getNode(Opcode: ISD::IS_FPCLASS, DL, VT: LoVT, N1: ArgLo, N2: Test, Flags: N->getFlags());
1567	Hi = DAG.getNode(Opcode: ISD::IS_FPCLASS, DL, VT: HiVT, N1: ArgHi, N2: Test, Flags: N->getFlags());
1568	}
1569
1570	void DAGTypeLegalizer::SplitVecRes_InregOp(SDNode *N, SDValue &Lo,
1571	SDValue &Hi) {
1572	SDValue LHSLo, LHSHi;
1573	GetSplitVector(Op: N->getOperand(Num: 0), Lo&: LHSLo, Hi&: LHSHi);
1574	SDLoc dl(N);
1575
1576	EVT LoVT, HiVT;
1577	std::tie(args&: LoVT, args&: HiVT) =
1578	DAG.GetSplitDestVTs(VT: cast<VTSDNode>(Val: N->getOperand(Num: 1))->getVT());
1579
1580	Lo = DAG.getNode(Opcode: N->getOpcode(), DL: dl, VT: LHSLo.getValueType(), N1: LHSLo,
1581	N2: DAG.getValueType(LoVT));
1582	Hi = DAG.getNode(Opcode: N->getOpcode(), DL: dl, VT: LHSHi.getValueType(), N1: LHSHi,
1583	N2: DAG.getValueType(HiVT));
1584	}
1585
1586	void DAGTypeLegalizer::SplitVecRes_ExtVecInRegOp(SDNode *N, SDValue &Lo,
1587	SDValue &Hi) {
1588	unsigned Opcode = N->getOpcode();
1589	SDValue N0 = N->getOperand(Num: 0);
1590
1591	SDLoc dl(N);
1592	SDValue InLo, InHi;
1593
1594	if (getTypeAction(VT: N0.getValueType()) == TargetLowering::TypeSplitVector)
1595	GetSplitVector(Op: N0, Lo&: InLo, Hi&: InHi);
1596	else
1597	std::tie(args&: InLo, args&: InHi) = DAG.SplitVectorOperand(N, OpNo: 0);
1598
1599	EVT InLoVT = InLo.getValueType();
1600	unsigned InNumElements = InLoVT.getVectorNumElements();
1601
1602	EVT OutLoVT, OutHiVT;
1603	std::tie(args&: OutLoVT, args&: OutHiVT) = DAG.GetSplitDestVTs(VT: N->getValueType(ResNo: 0));
1604	unsigned OutNumElements = OutLoVT.getVectorNumElements();
1605	assert((2 * OutNumElements) <= InNumElements &&
1606	"Illegal extend vector in reg split");
1607
1608	// *_EXTEND_VECTOR_INREG instructions extend the lowest elements of the
1609	// input vector (i.e. we only use InLo):
1610	// OutLo will extend the first OutNumElements from InLo.
1611	// OutHi will extend the next OutNumElements from InLo.
1612
1613	// Shuffle the elements from InLo for OutHi into the bottom elements to
1614	// create a 'fake' InHi.
1615	SmallVector<int, 8> SplitHi(InNumElements, -1);
1616	for (unsigned i = 0; i != OutNumElements; ++i)
1617	SplitHi[i] = i + OutNumElements;
1618	InHi = DAG.getVectorShuffle(VT: InLoVT, dl, N1: InLo, N2: DAG.getUNDEF(VT: InLoVT), Mask: SplitHi);
1619
1620	Lo = DAG.getNode(Opcode, DL: dl, VT: OutLoVT, Operand: InLo);
1621	Hi = DAG.getNode(Opcode, DL: dl, VT: OutHiVT, Operand: InHi);
1622	}
1623
1624	void DAGTypeLegalizer::SplitVecRes_StrictFPOp(SDNode *N, SDValue &Lo,
1625	SDValue &Hi) {
1626	unsigned NumOps = N->getNumOperands();
1627	SDValue Chain = N->getOperand(Num: 0);
1628	EVT LoVT, HiVT;
1629	SDLoc dl(N);
1630	std::tie(args&: LoVT, args&: HiVT) = DAG.GetSplitDestVTs(VT: N->getValueType(ResNo: 0));
1631
1632	SmallVector<SDValue, 4> OpsLo(NumOps);
1633	SmallVector<SDValue, 4> OpsHi(NumOps);
1634
1635	// The Chain is the first operand.
1636	OpsLo[0] = Chain;
1637	OpsHi[0] = Chain;
1638
1639	// Now process the remaining operands.
1640	for (unsigned i = 1; i < NumOps; ++i) {
1641	SDValue Op = N->getOperand(Num: i);
1642	SDValue OpLo = Op;
1643	SDValue OpHi = Op;
1644
1645	EVT InVT = Op.getValueType();
1646	if (InVT.isVector()) {
1647	// If the input also splits, handle it directly for a
1648	// compile time speedup. Otherwise split it by hand.
1649	if (getTypeAction(VT: InVT) == TargetLowering::TypeSplitVector)
1650	GetSplitVector(Op, Lo&: OpLo, Hi&: OpHi);
1651	else
1652	std::tie(args&: OpLo, args&: OpHi) = DAG.SplitVectorOperand(N, OpNo: i);
1653	}
1654
1655	OpsLo[i] = OpLo;
1656	OpsHi[i] = OpHi;
1657	}
1658
1659	EVT LoValueVTs[] = {LoVT, MVT::Other};
1660	EVT HiValueVTs[] = {HiVT, MVT::Other};
1661	Lo = DAG.getNode(Opcode: N->getOpcode(), DL: dl, VTList: DAG.getVTList(LoValueVTs), Ops: OpsLo,
1662	Flags: N->getFlags());
1663	Hi = DAG.getNode(Opcode: N->getOpcode(), DL: dl, VTList: DAG.getVTList(HiValueVTs), Ops: OpsHi,
1664	Flags: N->getFlags());
1665
1666	// Build a factor node to remember that this Op is independent of the
1667	// other one.
1668	Chain = DAG.getNode(ISD::TokenFactor, dl, MVT::Other,
1669	Lo.getValue(R: 1), Hi.getValue(R: 1));
1670
1671	// Legalize the chain result - switch anything that used the old chain to
1672	// use the new one.
1673	ReplaceValueWith(From: SDValue(N, 1), To: Chain);
1674	}
1675
1676	SDValue DAGTypeLegalizer::UnrollVectorOp_StrictFP(SDNode *N, unsigned ResNE) {
1677	SDValue Chain = N->getOperand(Num: 0);
1678	EVT VT = N->getValueType(ResNo: 0);
1679	unsigned NE = VT.getVectorNumElements();
1680	EVT EltVT = VT.getVectorElementType();
1681	SDLoc dl(N);
1682
1683	SmallVector<SDValue, 8> Scalars;
1684	SmallVector<SDValue, 4> Operands(N->getNumOperands());
1685
1686	// If ResNE is 0, fully unroll the vector op.
1687	if (ResNE == 0)
1688	ResNE = NE;
1689	else if (NE > ResNE)
1690	NE = ResNE;
1691
1692	//The results of each unrolled operation, including the chain.
1693	EVT ChainVTs[] = {EltVT, MVT::Other};
1694	SmallVector<SDValue, 8> Chains;
1695
1696	unsigned i;
1697	for (i = 0; i != NE; ++i) {
1698	Operands[0] = Chain;
1699	for (unsigned j = 1, e = N->getNumOperands(); j != e; ++j) {
1700	SDValue Operand = N->getOperand(Num: j);
1701	EVT OperandVT = Operand.getValueType();
1702	if (OperandVT.isVector()) {
1703	EVT OperandEltVT = OperandVT.getVectorElementType();
1704	Operands[j] = DAG.getNode(Opcode: ISD::EXTRACT_VECTOR_ELT, DL: dl, VT: OperandEltVT,
1705	N1: Operand, N2: DAG.getVectorIdxConstant(Val: i, DL: dl));
1706	} else {
1707	Operands[j] = Operand;
1708	}
1709	}
1710	SDValue Scalar = DAG.getNode(N->getOpcode(), dl, ChainVTs, Operands);
1711	Scalar.getNode()->setFlags(N->getFlags());
1712
1713	//Add in the scalar as well as its chain value to the
1714	//result vectors.
1715	Scalars.push_back(Elt: Scalar);
1716	Chains.push_back(Elt: Scalar.getValue(R: 1));
1717	}
1718
1719	for (; i < ResNE; ++i)
1720	Scalars.push_back(Elt: DAG.getUNDEF(VT: EltVT));
1721
1722	// Build a new factor node to connect the chain back together.
1723	Chain = DAG.getNode(ISD::TokenFactor, dl, MVT::Other, Chains);
1724	ReplaceValueWith(From: SDValue(N, 1), To: Chain);
1725
1726	// Create a new BUILD_VECTOR node
1727	EVT VecVT = EVT::getVectorVT(Context&: *DAG.getContext(), VT: EltVT, NumElements: ResNE);
1728	return DAG.getBuildVector(VT: VecVT, DL: dl, Ops: Scalars);
1729	}
1730
1731	void DAGTypeLegalizer::SplitVecRes_OverflowOp(SDNode *N, unsigned ResNo,
1732	SDValue &Lo, SDValue &Hi) {
1733	SDLoc dl(N);
1734	EVT ResVT = N->getValueType(ResNo: 0);
1735	EVT OvVT = N->getValueType(ResNo: 1);
1736	EVT LoResVT, HiResVT, LoOvVT, HiOvVT;
1737	std::tie(args&: LoResVT, args&: HiResVT) = DAG.GetSplitDestVTs(VT: ResVT);
1738	std::tie(args&: LoOvVT, args&: HiOvVT) = DAG.GetSplitDestVTs(VT: OvVT);
1739
1740	SDValue LoLHS, HiLHS, LoRHS, HiRHS;
1741	if (getTypeAction(VT: ResVT) == TargetLowering::TypeSplitVector) {
1742	GetSplitVector(Op: N->getOperand(Num: 0), Lo&: LoLHS, Hi&: HiLHS);
1743	GetSplitVector(Op: N->getOperand(Num: 1), Lo&: LoRHS, Hi&: HiRHS);
1744	} else {
1745	std::tie(args&: LoLHS, args&: HiLHS) = DAG.SplitVectorOperand(N, OpNo: 0);
1746	std::tie(args&: LoRHS, args&: HiRHS) = DAG.SplitVectorOperand(N, OpNo: 1);
1747	}
1748
1749	unsigned Opcode = N->getOpcode();
1750	SDVTList LoVTs = DAG.getVTList(VT1: LoResVT, VT2: LoOvVT);
1751	SDVTList HiVTs = DAG.getVTList(VT1: HiResVT, VT2: HiOvVT);
1752	SDNode *LoNode = DAG.getNode(Opcode, DL: dl, VTList: LoVTs, N1: LoLHS, N2: LoRHS).getNode();
1753	SDNode *HiNode = DAG.getNode(Opcode, DL: dl, VTList: HiVTs, N1: HiLHS, N2: HiRHS).getNode();
1754	LoNode->setFlags(N->getFlags());
1755	HiNode->setFlags(N->getFlags());
1756
1757	Lo = SDValue(LoNode, ResNo);
1758	Hi = SDValue(HiNode, ResNo);
1759
1760	// Replace the other vector result not being explicitly split here.
1761	unsigned OtherNo = 1 - ResNo;
1762	EVT OtherVT = N->getValueType(ResNo: OtherNo);
1763	if (getTypeAction(VT: OtherVT) == TargetLowering::TypeSplitVector) {
1764	SetSplitVector(Op: SDValue(N, OtherNo),
1765	Lo: SDValue(LoNode, OtherNo), Hi: SDValue(HiNode, OtherNo));
1766	} else {
1767	SDValue OtherVal = DAG.getNode(
1768	Opcode: ISD::CONCAT_VECTORS, DL: dl, VT: OtherVT,
1769	N1: SDValue(LoNode, OtherNo), N2: SDValue(HiNode, OtherNo));
1770	ReplaceValueWith(From: SDValue(N, OtherNo), To: OtherVal);
1771	}
1772	}
1773
1774	void DAGTypeLegalizer::SplitVecRes_INSERT_VECTOR_ELT(SDNode *N, SDValue &Lo,
1775	SDValue &Hi) {
1776	SDValue Vec = N->getOperand(Num: 0);
1777	SDValue Elt = N->getOperand(Num: 1);
1778	SDValue Idx = N->getOperand(Num: 2);
1779	SDLoc dl(N);
1780	GetSplitVector(Op: Vec, Lo, Hi);
1781
1782	if (ConstantSDNode *CIdx = dyn_cast<ConstantSDNode>(Val&: Idx)) {
1783	unsigned IdxVal = CIdx->getZExtValue();
1784	unsigned LoNumElts = Lo.getValueType().getVectorMinNumElements();
1785	if (IdxVal < LoNumElts) {
1786	Lo = DAG.getNode(Opcode: ISD::INSERT_VECTOR_ELT, DL: dl,
1787	VT: Lo.getValueType(), N1: Lo, N2: Elt, N3: Idx);
1788	return;
1789	} else if (!Vec.getValueType().isScalableVector()) {
1790	Hi = DAG.getNode(Opcode: ISD::INSERT_VECTOR_ELT, DL: dl, VT: Hi.getValueType(), N1: Hi, N2: Elt,
1791	N3: DAG.getVectorIdxConstant(Val: IdxVal - LoNumElts, DL: dl));
1792	return;
1793	}
1794	}
1795
1796	// See if the target wants to custom expand this node.
1797	if (CustomLowerNode(N, VT: N->getValueType(ResNo: 0), LegalizeResult: true))
1798	return;
1799
1800	// Make the vector elements byte-addressable if they aren't already.
1801	EVT VecVT = Vec.getValueType();
1802	EVT EltVT = VecVT.getVectorElementType();
1803	if (VecVT.getScalarSizeInBits() < 8) {
1804	EltVT = MVT::i8;
1805	VecVT = EVT::getVectorVT(Context&: *DAG.getContext(), VT: EltVT,
1806	EC: VecVT.getVectorElementCount());
1807	Vec = DAG.getNode(Opcode: ISD::ANY_EXTEND, DL: dl, VT: VecVT, Operand: Vec);
1808	// Extend the element type to match if needed.
1809	if (EltVT.bitsGT(VT: Elt.getValueType()))
1810	Elt = DAG.getNode(Opcode: ISD::ANY_EXTEND, DL: dl, VT: EltVT, Operand: Elt);
1811	}
1812
1813	// Spill the vector to the stack.
1814	// In cases where the vector is illegal it will be broken down into parts
1815	// and stored in parts - we should use the alignment for the smallest part.
1816	Align SmallestAlign = DAG.getReducedAlign(VT: VecVT, /*UseABI=*/false);
1817	SDValue StackPtr =
1818	DAG.CreateStackTemporary(Bytes: VecVT.getStoreSize(), Alignment: SmallestAlign);
1819	auto &MF = DAG.getMachineFunction();
1820	auto FrameIndex = cast<FrameIndexSDNode>(Val: StackPtr.getNode())->getIndex();
1821	auto PtrInfo = MachinePointerInfo::getFixedStack(MF, FI: FrameIndex);
1822
1823	SDValue Store = DAG.getStore(Chain: DAG.getEntryNode(), dl, Val: Vec, Ptr: StackPtr, PtrInfo,
1824	Alignment: SmallestAlign);
1825
1826	// Store the new element. This may be larger than the vector element type,
1827	// so use a truncating store.
1828	SDValue EltPtr = TLI.getVectorElementPointer(DAG, VecPtr: StackPtr, VecVT, Index: Idx);
1829	Store = DAG.getTruncStore(
1830	Chain: Store, dl, Val: Elt, Ptr: EltPtr, PtrInfo: MachinePointerInfo::getUnknownStack(MF), SVT: EltVT,
1831	Alignment: commonAlignment(A: SmallestAlign,
1832	Offset: EltVT.getFixedSizeInBits() / 8));
1833
1834	EVT LoVT, HiVT;
1835	std::tie(args&: LoVT, args&: HiVT) = DAG.GetSplitDestVTs(VT: VecVT);
1836
1837	// Load the Lo part from the stack slot.
1838	Lo = DAG.getLoad(VT: LoVT, dl, Chain: Store, Ptr: StackPtr, PtrInfo, Alignment: SmallestAlign);
1839
1840	// Increment the pointer to the other part.
1841	auto Load = cast<LoadSDNode>(Val&: Lo);
1842	MachinePointerInfo MPI = Load->getPointerInfo();
1843	IncrementPointer(N: Load, MemVT: LoVT, MPI, Ptr&: StackPtr);
1844
1845	Hi = DAG.getLoad(VT: HiVT, dl, Chain: Store, Ptr: StackPtr, PtrInfo: MPI, Alignment: SmallestAlign);
1846
1847	// If we adjusted the original type, we need to truncate the results.
1848	std::tie(args&: LoVT, args&: HiVT) = DAG.GetSplitDestVTs(VT: N->getValueType(ResNo: 0));
1849	if (LoVT != Lo.getValueType())
1850	Lo = DAG.getNode(Opcode: ISD::TRUNCATE, DL: dl, VT: LoVT, Operand: Lo);
1851	if (HiVT != Hi.getValueType())
1852	Hi = DAG.getNode(Opcode: ISD::TRUNCATE, DL: dl, VT: HiVT, Operand: Hi);
1853	}
1854
1855	void DAGTypeLegalizer::SplitVecRes_STEP_VECTOR(SDNode *N, SDValue &Lo,
1856	SDValue &Hi) {
1857	EVT LoVT, HiVT;
1858	SDLoc dl(N);
1859	assert(N->getValueType(0).isScalableVector() &&
1860	"Only scalable vectors are supported for STEP_VECTOR");
1861	std::tie(args&: LoVT, args&: HiVT) = DAG.GetSplitDestVTs(VT: N->getValueType(ResNo: 0));
1862	SDValue Step = N->getOperand(Num: 0);
1863
1864	Lo = DAG.getNode(Opcode: ISD::STEP_VECTOR, DL: dl, VT: LoVT, Operand: Step);
1865
1866	// Hi = Lo + (EltCnt * Step)
1867	EVT EltVT = Step.getValueType();
1868	APInt StepVal = Step->getAsAPIntVal();
1869	SDValue StartOfHi =
1870	DAG.getVScale(DL: dl, VT: EltVT, MulImm: StepVal * LoVT.getVectorMinNumElements());
1871	StartOfHi = DAG.getSExtOrTrunc(Op: StartOfHi, DL: dl, VT: HiVT.getVectorElementType());
1872	StartOfHi = DAG.getNode(Opcode: ISD::SPLAT_VECTOR, DL: dl, VT: HiVT, Operand: StartOfHi);
1873
1874	Hi = DAG.getNode(Opcode: ISD::STEP_VECTOR, DL: dl, VT: HiVT, Operand: Step);
1875	Hi = DAG.getNode(Opcode: ISD::ADD, DL: dl, VT: HiVT, N1: Hi, N2: StartOfHi);
1876	}
1877
1878	void DAGTypeLegalizer::SplitVecRes_ScalarOp(SDNode *N, SDValue &Lo,
1879	SDValue &Hi) {
1880	EVT LoVT, HiVT;
1881	SDLoc dl(N);
1882	std::tie(args&: LoVT, args&: HiVT) = DAG.GetSplitDestVTs(VT: N->getValueType(ResNo: 0));
1883	Lo = DAG.getNode(Opcode: N->getOpcode(), DL: dl, VT: LoVT, Operand: N->getOperand(Num: 0));
1884	if (N->getOpcode() == ISD::SCALAR_TO_VECTOR) {
1885	Hi = DAG.getUNDEF(VT: HiVT);
1886	} else {
1887	assert(N->getOpcode() == ISD::SPLAT_VECTOR && "Unexpected opcode");
1888	Hi = Lo;
1889	}
1890	}
1891
1892	void DAGTypeLegalizer::SplitVecRes_LOAD(LoadSDNode *LD, SDValue &Lo,
1893	SDValue &Hi) {
1894	assert(ISD::isUNINDEXEDLoad(LD) && "Indexed load during type legalization!");
1895	EVT LoVT, HiVT;
1896	SDLoc dl(LD);
1897	std::tie(args&: LoVT, args&: HiVT) = DAG.GetSplitDestVTs(VT: LD->getValueType(ResNo: 0));
1898
1899	ISD::LoadExtType ExtType = LD->getExtensionType();
1900	SDValue Ch = LD->getChain();
1901	SDValue Ptr = LD->getBasePtr();
1902	SDValue Offset = DAG.getUNDEF(VT: Ptr.getValueType());
1903	EVT MemoryVT = LD->getMemoryVT();
1904	MachineMemOperand::Flags MMOFlags = LD->getMemOperand()->getFlags();
1905	AAMDNodes AAInfo = LD->getAAInfo();
1906
1907	EVT LoMemVT, HiMemVT;
1908	std::tie(args&: LoMemVT, args&: HiMemVT) = DAG.GetSplitDestVTs(VT: MemoryVT);
1909
1910	if (!LoMemVT.isByteSized() || !HiMemVT.isByteSized()) {
1911	SDValue Value, NewChain;
1912	std::tie(args&: Value, args&: NewChain) = TLI.scalarizeVectorLoad(LD, DAG);
1913	std::tie(args&: Lo, args&: Hi) = DAG.SplitVector(N: Value, DL: dl);
1914	ReplaceValueWith(From: SDValue(LD, 1), To: NewChain);
1915	return;
1916	}
1917
1918	Lo = DAG.getLoad(AM: ISD::UNINDEXED, ExtType, VT: LoVT, dl, Chain: Ch, Ptr, Offset,
1919	PtrInfo: LD->getPointerInfo(), MemVT: LoMemVT, Alignment: LD->getOriginalAlign(),
1920	MMOFlags, AAInfo);
1921
1922	MachinePointerInfo MPI;
1923	IncrementPointer(N: LD, MemVT: LoMemVT, MPI, Ptr);
1924
1925	Hi = DAG.getLoad(AM: ISD::UNINDEXED, ExtType, VT: HiVT, dl, Chain: Ch, Ptr, Offset, PtrInfo: MPI,
1926	MemVT: HiMemVT, Alignment: LD->getOriginalAlign(), MMOFlags, AAInfo);
1927
1928	// Build a factor node to remember that this load is independent of the
1929	// other one.
1930	Ch = DAG.getNode(ISD::TokenFactor, dl, MVT::Other, Lo.getValue(R: 1),
1931	Hi.getValue(R: 1));
1932
1933	// Legalize the chain result - switch anything that used the old chain to
1934	// use the new one.
1935	ReplaceValueWith(From: SDValue(LD, 1), To: Ch);
1936	}
1937
1938	void DAGTypeLegalizer::SplitVecRes_VP_LOAD(VPLoadSDNode *LD, SDValue &Lo,
1939	SDValue &Hi) {
1940	assert(LD->isUnindexed() && "Indexed VP load during type legalization!");
1941	EVT LoVT, HiVT;
1942	SDLoc dl(LD);
1943	std::tie(args&: LoVT, args&: HiVT) = DAG.GetSplitDestVTs(VT: LD->getValueType(ResNo: 0));
1944
1945	ISD::LoadExtType ExtType = LD->getExtensionType();
1946	SDValue Ch = LD->getChain();
1947	SDValue Ptr = LD->getBasePtr();
1948	SDValue Offset = LD->getOffset();
1949	assert(Offset.isUndef() && "Unexpected indexed variable-length load offset");
1950	Align Alignment = LD->getOriginalAlign();
1951	SDValue Mask = LD->getMask();
1952	SDValue EVL = LD->getVectorLength();
1953	EVT MemoryVT = LD->getMemoryVT();
1954
1955	EVT LoMemVT, HiMemVT;
1956	bool HiIsEmpty = false;
1957	std::tie(args&: LoMemVT, args&: HiMemVT) =
1958	DAG.GetDependentSplitDestVTs(VT: MemoryVT, EnvVT: LoVT, HiIsEmpty: &HiIsEmpty);
1959
1960	// Split Mask operand
1961	SDValue MaskLo, MaskHi;
1962	if (Mask.getOpcode() == ISD::SETCC) {
1963	SplitVecRes_SETCC(N: Mask.getNode(), Lo&: MaskLo, Hi&: MaskHi);
1964	} else {
1965	if (getTypeAction(VT: Mask.getValueType()) == TargetLowering::TypeSplitVector)
1966	GetSplitVector(Op: Mask, Lo&: MaskLo, Hi&: MaskHi);
1967	else
1968	std::tie(args&: MaskLo, args&: MaskHi) = DAG.SplitVector(N: Mask, DL: dl);
1969	}
1970
1971	// Split EVL operand
1972	SDValue EVLLo, EVLHi;
1973	std::tie(args&: EVLLo, args&: EVLHi) = DAG.SplitEVL(N: EVL, VecVT: LD->getValueType(ResNo: 0), DL: dl);
1974
1975	MachineMemOperand *MMO = DAG.getMachineFunction().getMachineMemOperand(
1976	PtrInfo: LD->getPointerInfo(), F: MachineMemOperand::MOLoad,
1977	Size: LocationSize::beforeOrAfterPointer(), BaseAlignment: Alignment, AAInfo: LD->getAAInfo(),
1978	Ranges: LD->getRanges());
1979
1980	Lo =
1981	DAG.getLoadVP(AM: LD->getAddressingMode(), ExtType, VT: LoVT, dl, Chain: Ch, Ptr, Offset,
1982	Mask: MaskLo, EVL: EVLLo, MemVT: LoMemVT, MMO, IsExpanding: LD->isExpandingLoad());
1983
1984	if (HiIsEmpty) {
1985	// The hi vp_load has zero storage size. We therefore simply set it to
1986	// the low vp_load and rely on subsequent removal from the chain.
1987	Hi = Lo;
1988	} else {
1989	// Generate hi vp_load.
1990	Ptr = TLI.IncrementMemoryAddress(Addr: Ptr, Mask: MaskLo, DL: dl, DataVT: LoMemVT, DAG,
1991	IsCompressedMemory: LD->isExpandingLoad());
1992
1993	MachinePointerInfo MPI;
1994	if (LoMemVT.isScalableVector())
1995	MPI = MachinePointerInfo(LD->getPointerInfo().getAddrSpace());
1996	else
1997	MPI = LD->getPointerInfo().getWithOffset(
1998	O: LoMemVT.getStoreSize().getFixedValue());
1999
2000	MMO = DAG.getMachineFunction().getMachineMemOperand(
2001	PtrInfo: MPI, F: MachineMemOperand::MOLoad, Size: LocationSize::beforeOrAfterPointer(),
2002	BaseAlignment: Alignment, AAInfo: LD->getAAInfo(), Ranges: LD->getRanges());
2003
2004	Hi = DAG.getLoadVP(AM: LD->getAddressingMode(), ExtType, VT: HiVT, dl, Chain: Ch, Ptr,
2005	Offset, Mask: MaskHi, EVL: EVLHi, MemVT: HiMemVT, MMO,
2006	IsExpanding: LD->isExpandingLoad());
2007	}
2008
2009	// Build a factor node to remember that this load is independent of the
2010	// other one.
2011	Ch = DAG.getNode(ISD::TokenFactor, dl, MVT::Other, Lo.getValue(R: 1),
2012	Hi.getValue(R: 1));
2013
2014	// Legalize the chain result - switch anything that used the old chain to
2015	// use the new one.
2016	ReplaceValueWith(From: SDValue(LD, 1), To: Ch);
2017	}
2018
2019	void DAGTypeLegalizer::SplitVecRes_VP_STRIDED_LOAD(VPStridedLoadSDNode *SLD,
2020	SDValue &Lo, SDValue &Hi) {
2021	assert(SLD->isUnindexed() &&
2022	"Indexed VP strided load during type legalization!");
2023	assert(SLD->getOffset().isUndef() &&
2024	"Unexpected indexed variable-length load offset");
2025
2026	SDLoc DL(SLD);
2027
2028	EVT LoVT, HiVT;
2029	std::tie(args&: LoVT, args&: HiVT) = DAG.GetSplitDestVTs(VT: SLD->getValueType(ResNo: 0));
2030
2031	EVT LoMemVT, HiMemVT;
2032	bool HiIsEmpty = false;
2033	std::tie(args&: LoMemVT, args&: HiMemVT) =
2034	DAG.GetDependentSplitDestVTs(VT: SLD->getMemoryVT(), EnvVT: LoVT, HiIsEmpty: &HiIsEmpty);
2035
2036	SDValue Mask = SLD->getMask();
2037	SDValue LoMask, HiMask;
2038	if (Mask.getOpcode() == ISD::SETCC) {
2039	SplitVecRes_SETCC(N: Mask.getNode(), Lo&: LoMask, Hi&: HiMask);
2040	} else {
2041	if (getTypeAction(VT: Mask.getValueType()) == TargetLowering::TypeSplitVector)
2042	GetSplitVector(Op: Mask, Lo&: LoMask, Hi&: HiMask);
2043	else
2044	std::tie(args&: LoMask, args&: HiMask) = DAG.SplitVector(N: Mask, DL);
2045	}
2046
2047	SDValue LoEVL, HiEVL;
2048	std::tie(args&: LoEVL, args&: HiEVL) =
2049	DAG.SplitEVL(N: SLD->getVectorLength(), VecVT: SLD->getValueType(ResNo: 0), DL);
2050
2051	// Generate the low vp_strided_load
2052	Lo = DAG.getStridedLoadVP(
2053	AM: SLD->getAddressingMode(), ExtType: SLD->getExtensionType(), VT: LoVT, DL,
2054	Chain: SLD->getChain(), Ptr: SLD->getBasePtr(), Offset: SLD->getOffset(), Stride: SLD->getStride(),
2055	Mask: LoMask, EVL: LoEVL, MemVT: LoMemVT, MMO: SLD->getMemOperand(), IsExpanding: SLD->isExpandingLoad());
2056
2057	if (HiIsEmpty) {
2058	// The high vp_strided_load has zero storage size. We therefore simply set
2059	// it to the low vp_strided_load and rely on subsequent removal from the
2060	// chain.
2061	Hi = Lo;
2062	} else {
2063	// Generate the high vp_strided_load.
2064	// To calculate the high base address, we need to sum to the low base
2065	// address stride number of bytes for each element already loaded by low,
2066	// that is: Ptr = Ptr + (LoEVL * Stride)
2067	EVT PtrVT = SLD->getBasePtr().getValueType();
2068	SDValue Increment =
2069	DAG.getNode(Opcode: ISD::MUL, DL, VT: PtrVT, N1: LoEVL,
2070	N2: DAG.getSExtOrTrunc(Op: SLD->getStride(), DL, VT: PtrVT));
2071	SDValue Ptr =
2072	DAG.getNode(Opcode: ISD::ADD, DL, VT: PtrVT, N1: SLD->getBasePtr(), N2: Increment);
2073
2074	Align Alignment = SLD->getOriginalAlign();
2075	if (LoMemVT.isScalableVector())
2076	Alignment = commonAlignment(
2077	A: Alignment, Offset: LoMemVT.getSizeInBits().getKnownMinValue() / 8);
2078
2079	MachineMemOperand *MMO = DAG.getMachineFunction().getMachineMemOperand(
2080	PtrInfo: MachinePointerInfo(SLD->getPointerInfo().getAddrSpace()),
2081	F: MachineMemOperand::MOLoad, Size: LocationSize::beforeOrAfterPointer(),
2082	BaseAlignment: Alignment, AAInfo: SLD->getAAInfo(), Ranges: SLD->getRanges());
2083
2084	Hi = DAG.getStridedLoadVP(AM: SLD->getAddressingMode(), ExtType: SLD->getExtensionType(),
2085	VT: HiVT, DL, Chain: SLD->getChain(), Ptr, Offset: SLD->getOffset(),
2086	Stride: SLD->getStride(), Mask: HiMask, EVL: HiEVL, MemVT: HiMemVT, MMO,
2087	IsExpanding: SLD->isExpandingLoad());
2088	}
2089
2090	// Build a factor node to remember that this load is independent of the
2091	// other one.
2092	SDValue Ch = DAG.getNode(ISD::TokenFactor, DL, MVT::Other, Lo.getValue(R: 1),
2093	Hi.getValue(R: 1));
2094
2095	// Legalize the chain result - switch anything that used the old chain to
2096	// use the new one.
2097	ReplaceValueWith(From: SDValue(SLD, 1), To: Ch);
2098	}
2099
2100	void DAGTypeLegalizer::SplitVecRes_MLOAD(MaskedLoadSDNode *MLD,
2101	SDValue &Lo, SDValue &Hi) {
2102	assert(MLD->isUnindexed() && "Indexed masked load during type legalization!");
2103	EVT LoVT, HiVT;
2104	SDLoc dl(MLD);
2105	std::tie(args&: LoVT, args&: HiVT) = DAG.GetSplitDestVTs(VT: MLD->getValueType(ResNo: 0));
2106
2107	SDValue Ch = MLD->getChain();
2108	SDValue Ptr = MLD->getBasePtr();
2109	SDValue Offset = MLD->getOffset();
2110	assert(Offset.isUndef() && "Unexpected indexed masked load offset");
2111	SDValue Mask = MLD->getMask();
2112	SDValue PassThru = MLD->getPassThru();
2113	Align Alignment = MLD->getOriginalAlign();
2114	ISD::LoadExtType ExtType = MLD->getExtensionType();
2115
2116	// Split Mask operand
2117	SDValue MaskLo, MaskHi;
2118	if (Mask.getOpcode() == ISD::SETCC) {
2119	SplitVecRes_SETCC(N: Mask.getNode(), Lo&: MaskLo, Hi&: MaskHi);
2120	} else {
2121	if (getTypeAction(VT: Mask.getValueType()) == TargetLowering::TypeSplitVector)
2122	GetSplitVector(Op: Mask, Lo&: MaskLo, Hi&: MaskHi);
2123	else
2124	std::tie(args&: MaskLo, args&: MaskHi) = DAG.SplitVector(N: Mask, DL: dl);
2125	}
2126
2127	EVT MemoryVT = MLD->getMemoryVT();
2128	EVT LoMemVT, HiMemVT;
2129	bool HiIsEmpty = false;
2130	std::tie(args&: LoMemVT, args&: HiMemVT) =
2131	DAG.GetDependentSplitDestVTs(VT: MemoryVT, EnvVT: LoVT, HiIsEmpty: &HiIsEmpty);
2132
2133	SDValue PassThruLo, PassThruHi;
2134	if (getTypeAction(VT: PassThru.getValueType()) == TargetLowering::TypeSplitVector)
2135	GetSplitVector(Op: PassThru, Lo&: PassThruLo, Hi&: PassThruHi);
2136	else
2137	std::tie(args&: PassThruLo, args&: PassThruHi) = DAG.SplitVector(N: PassThru, DL: dl);
2138
2139	MachineMemOperand *MMO = DAG.getMachineFunction().getMachineMemOperand(
2140	PtrInfo: MLD->getPointerInfo(), F: MachineMemOperand::MOLoad,
2141	Size: LocationSize::beforeOrAfterPointer(), BaseAlignment: Alignment, AAInfo: MLD->getAAInfo(),
2142	Ranges: MLD->getRanges());
2143
2144	Lo = DAG.getMaskedLoad(VT: LoVT, dl, Chain: Ch, Base: Ptr, Offset, Mask: MaskLo, Src0: PassThruLo, MemVT: LoMemVT,
2145	MMO, AM: MLD->getAddressingMode(), ExtType,
2146	IsExpanding: MLD->isExpandingLoad());
2147
2148	if (HiIsEmpty) {
2149	// The hi masked load has zero storage size. We therefore simply set it to
2150	// the low masked load and rely on subsequent removal from the chain.
2151	Hi = Lo;
2152	} else {
2153	// Generate hi masked load.
2154	Ptr = TLI.IncrementMemoryAddress(Addr: Ptr, Mask: MaskLo, DL: dl, DataVT: LoMemVT, DAG,
2155	IsCompressedMemory: MLD->isExpandingLoad());
2156
2157	MachinePointerInfo MPI;
2158	if (LoMemVT.isScalableVector())
2159	MPI = MachinePointerInfo(MLD->getPointerInfo().getAddrSpace());
2160	else
2161	MPI = MLD->getPointerInfo().getWithOffset(
2162	O: LoMemVT.getStoreSize().getFixedValue());
2163
2164	MMO = DAG.getMachineFunction().getMachineMemOperand(
2165	PtrInfo: MPI, F: MachineMemOperand::MOLoad, Size: LocationSize::beforeOrAfterPointer(),
2166	BaseAlignment: Alignment, AAInfo: MLD->getAAInfo(), Ranges: MLD->getRanges());
2167
2168	Hi = DAG.getMaskedLoad(VT: HiVT, dl, Chain: Ch, Base: Ptr, Offset, Mask: MaskHi, Src0: PassThruHi,
2169	MemVT: HiMemVT, MMO, AM: MLD->getAddressingMode(), ExtType,
2170	IsExpanding: MLD->isExpandingLoad());
2171	}
2172
2173	// Build a factor node to remember that this load is independent of the
2174	// other one.
2175	Ch = DAG.getNode(ISD::TokenFactor, dl, MVT::Other, Lo.getValue(R: 1),
2176	Hi.getValue(R: 1));
2177
2178	// Legalize the chain result - switch anything that used the old chain to
2179	// use the new one.
2180	ReplaceValueWith(From: SDValue(MLD, 1), To: Ch);
2181
2182	}
2183
2184	void DAGTypeLegalizer::SplitVecRes_Gather(MemSDNode *N, SDValue &Lo,
2185	SDValue &Hi, bool SplitSETCC) {
2186	EVT LoVT, HiVT;
2187	SDLoc dl(N);
2188	std::tie(args&: LoVT, args&: HiVT) = DAG.GetSplitDestVTs(VT: N->getValueType(ResNo: 0));
2189
2190	SDValue Ch = N->getChain();
2191	SDValue Ptr = N->getBasePtr();
2192	struct Operands {
2193	SDValue Mask;
2194	SDValue Index;
2195	SDValue Scale;
2196	} Ops = [&]() -> Operands {
2197	if (auto *MSC = dyn_cast<MaskedGatherSDNode>(Val: N)) {
2198	return {.Mask: MSC->getMask(), .Index: MSC->getIndex(), .Scale: MSC->getScale()};
2199	}
2200	auto *VPSC = cast<VPGatherSDNode>(Val: N);
2201	return {.Mask: VPSC->getMask(), .Index: VPSC->getIndex(), .Scale: VPSC->getScale()};
2202	}();
2203
2204	EVT MemoryVT = N->getMemoryVT();
2205	Align Alignment = N->getOriginalAlign();
2206
2207	// Split Mask operand
2208	SDValue MaskLo, MaskHi;
2209	if (SplitSETCC && Ops.Mask.getOpcode() == ISD::SETCC) {
2210	SplitVecRes_SETCC(N: Ops.Mask.getNode(), Lo&: MaskLo, Hi&: MaskHi);
2211	} else {
2212	std::tie(args&: MaskLo, args&: MaskHi) = SplitMask(Mask: Ops.Mask, DL: dl);
2213	}
2214
2215	EVT LoMemVT, HiMemVT;
2216	// Split MemoryVT
2217	std::tie(args&: LoMemVT, args&: HiMemVT) = DAG.GetSplitDestVTs(VT: MemoryVT);
2218
2219	SDValue IndexHi, IndexLo;
2220	if (getTypeAction(VT: Ops.Index.getValueType()) ==
2221	TargetLowering::TypeSplitVector)
2222	GetSplitVector(Op: Ops.Index, Lo&: IndexLo, Hi&: IndexHi);
2223	else
2224	std::tie(args&: IndexLo, args&: IndexHi) = DAG.SplitVector(N: Ops.Index, DL: dl);
2225
2226	MachineMemOperand *MMO = DAG.getMachineFunction().getMachineMemOperand(
2227	PtrInfo: N->getPointerInfo(), F: MachineMemOperand::MOLoad,
2228	Size: LocationSize::beforeOrAfterPointer(), BaseAlignment: Alignment, AAInfo: N->getAAInfo(),
2229	Ranges: N->getRanges());
2230
2231	if (auto *MGT = dyn_cast<MaskedGatherSDNode>(Val: N)) {
2232	SDValue PassThru = MGT->getPassThru();
2233	SDValue PassThruLo, PassThruHi;
2234	if (getTypeAction(VT: PassThru.getValueType()) ==
2235	TargetLowering::TypeSplitVector)
2236	GetSplitVector(Op: PassThru, Lo&: PassThruLo, Hi&: PassThruHi);
2237	else
2238	std::tie(args&: PassThruLo, args&: PassThruHi) = DAG.SplitVector(N: PassThru, DL: dl);
2239
2240	ISD::LoadExtType ExtType = MGT->getExtensionType();
2241	ISD::MemIndexType IndexTy = MGT->getIndexType();
2242
2243	SDValue OpsLo[] = {Ch, PassThruLo, MaskLo, Ptr, IndexLo, Ops.Scale};
2244	Lo = DAG.getMaskedGather(VTs: DAG.getVTList(LoVT, MVT::Other), MemVT: LoMemVT, dl,
2245	Ops: OpsLo, MMO, IndexType: IndexTy, ExtTy: ExtType);
2246
2247	SDValue OpsHi[] = {Ch, PassThruHi, MaskHi, Ptr, IndexHi, Ops.Scale};
2248	Hi = DAG.getMaskedGather(VTs: DAG.getVTList(HiVT, MVT::Other), MemVT: HiMemVT, dl,
2249	Ops: OpsHi, MMO, IndexType: IndexTy, ExtTy: ExtType);
2250	} else {
2251	auto *VPGT = cast<VPGatherSDNode>(Val: N);
2252	SDValue EVLLo, EVLHi;
2253	std::tie(args&: EVLLo, args&: EVLHi) =
2254	DAG.SplitEVL(N: VPGT->getVectorLength(), VecVT: MemoryVT, DL: dl);
2255
2256	SDValue OpsLo[] = {Ch, Ptr, IndexLo, Ops.Scale, MaskLo, EVLLo};
2257	Lo = DAG.getGatherVP(VTs: DAG.getVTList(LoVT, MVT::Other), VT: LoMemVT, dl, Ops: OpsLo,
2258	MMO, IndexType: VPGT->getIndexType());
2259
2260	SDValue OpsHi[] = {Ch, Ptr, IndexHi, Ops.Scale, MaskHi, EVLHi};
2261	Hi = DAG.getGatherVP(VTs: DAG.getVTList(HiVT, MVT::Other), VT: HiMemVT, dl, Ops: OpsHi,
2262	MMO, IndexType: VPGT->getIndexType());
2263	}
2264
2265	// Build a factor node to remember that this load is independent of the
2266	// other one.
2267	Ch = DAG.getNode(ISD::TokenFactor, dl, MVT::Other, Lo.getValue(R: 1),
2268	Hi.getValue(R: 1));
2269
2270	// Legalize the chain result - switch anything that used the old chain to
2271	// use the new one.
2272	ReplaceValueWith(From: SDValue(N, 1), To: Ch);
2273	}
2274
2275	void DAGTypeLegalizer::SplitVecRes_SETCC(SDNode *N, SDValue &Lo, SDValue &Hi) {
2276	assert(N->getValueType(0).isVector() &&
2277	N->getOperand(0).getValueType().isVector() &&
2278	"Operand types must be vectors");
2279
2280	EVT LoVT, HiVT;
2281	SDLoc DL(N);
2282	std::tie(args&: LoVT, args&: HiVT) = DAG.GetSplitDestVTs(VT: N->getValueType(ResNo: 0));
2283
2284	// If the input also splits, handle it directly. Otherwise split it by hand.
2285	SDValue LL, LH, RL, RH;
2286	if (getTypeAction(VT: N->getOperand(Num: 0).getValueType()) ==
2287	TargetLowering::TypeSplitVector)
2288	GetSplitVector(Op: N->getOperand(Num: 0), Lo&: LL, Hi&: LH);
2289	else
2290	std::tie(args&: LL, args&: LH) = DAG.SplitVectorOperand(N, OpNo: 0);
2291
2292	if (getTypeAction(VT: N->getOperand(Num: 1).getValueType()) ==
2293	TargetLowering::TypeSplitVector)
2294	GetSplitVector(Op: N->getOperand(Num: 1), Lo&: RL, Hi&: RH);
2295	else
2296	std::tie(args&: RL, args&: RH) = DAG.SplitVectorOperand(N, OpNo: 1);
2297
2298	if (N->getOpcode() == ISD::SETCC) {
2299	Lo = DAG.getNode(Opcode: N->getOpcode(), DL, VT: LoVT, N1: LL, N2: RL, N3: N->getOperand(Num: 2));
2300	Hi = DAG.getNode(Opcode: N->getOpcode(), DL, VT: HiVT, N1: LH, N2: RH, N3: N->getOperand(Num: 2));
2301	} else {
2302	assert(N->getOpcode() == ISD::VP_SETCC && "Expected VP_SETCC opcode");
2303	SDValue MaskLo, MaskHi, EVLLo, EVLHi;
2304	std::tie(args&: MaskLo, args&: MaskHi) = SplitMask(Mask: N->getOperand(Num: 3));
2305	std::tie(args&: EVLLo, args&: EVLHi) =
2306	DAG.SplitEVL(N: N->getOperand(Num: 4), VecVT: N->getValueType(ResNo: 0), DL);
2307	Lo = DAG.getNode(Opcode: N->getOpcode(), DL, VT: LoVT, N1: LL, N2: RL, N3: N->getOperand(Num: 2), N4: MaskLo,
2308	N5: EVLLo);
2309	Hi = DAG.getNode(Opcode: N->getOpcode(), DL, VT: HiVT, N1: LH, N2: RH, N3: N->getOperand(Num: 2), N4: MaskHi,
2310	N5: EVLHi);
2311	}
2312	}
2313
2314	void DAGTypeLegalizer::SplitVecRes_UnaryOp(SDNode *N, SDValue &Lo,
2315	SDValue &Hi) {
2316	// Get the dest types - they may not match the input types, e.g. int_to_fp.
2317	EVT LoVT, HiVT;
2318	SDLoc dl(N);
2319	std::tie(args&: LoVT, args&: HiVT) = DAG.GetSplitDestVTs(VT: N->getValueType(ResNo: 0));
2320
2321	// If the input also splits, handle it directly for a compile time speedup.
2322	// Otherwise split it by hand.
2323	EVT InVT = N->getOperand(Num: 0).getValueType();
2324	if (getTypeAction(VT: InVT) == TargetLowering::TypeSplitVector)
2325	GetSplitVector(Op: N->getOperand(Num: 0), Lo, Hi);
2326	else
2327	std::tie(args&: Lo, args&: Hi) = DAG.SplitVectorOperand(N, OpNo: 0);
2328
2329	const SDNodeFlags Flags = N->getFlags();
2330	unsigned Opcode = N->getOpcode();
2331	if (N->getNumOperands() <= 2) {
2332	if (Opcode == ISD::FP_ROUND) {
2333	Lo = DAG.getNode(Opcode, DL: dl, VT: LoVT, N1: Lo, N2: N->getOperand(Num: 1), Flags);
2334	Hi = DAG.getNode(Opcode, DL: dl, VT: HiVT, N1: Hi, N2: N->getOperand(Num: 1), Flags);
2335	} else {
2336	Lo = DAG.getNode(Opcode, DL: dl, VT: LoVT, Operand: Lo, Flags);
2337	Hi = DAG.getNode(Opcode, DL: dl, VT: HiVT, Operand: Hi, Flags);
2338	}
2339	return;
2340	}
2341
2342	assert(N->getNumOperands() == 3 && "Unexpected number of operands!");
2343	assert(N->isVPOpcode() && "Expected VP opcode");
2344
2345	SDValue MaskLo, MaskHi;
2346	std::tie(args&: MaskLo, args&: MaskHi) = SplitMask(Mask: N->getOperand(Num: 1));
2347
2348	SDValue EVLLo, EVLHi;
2349	std::tie(args&: EVLLo, args&: EVLHi) =
2350	DAG.SplitEVL(N: N->getOperand(Num: 2), VecVT: N->getValueType(ResNo: 0), DL: dl);
2351
2352	Lo = DAG.getNode(Opcode, DL: dl, VT: LoVT, Ops: {Lo, MaskLo, EVLLo}, Flags);
2353	Hi = DAG.getNode(Opcode, DL: dl, VT: HiVT, Ops: {Hi, MaskHi, EVLHi}, Flags);
2354	}
2355
2356	void DAGTypeLegalizer::SplitVecRes_FFREXP(SDNode *N, unsigned ResNo,
2357	SDValue &Lo, SDValue &Hi) {
2358	SDLoc dl(N);
2359	auto [LoVT, HiVT] = DAG.GetSplitDestVTs(VT: N->getValueType(ResNo: 0));
2360	auto [LoVT1, HiVT1] = DAG.GetSplitDestVTs(VT: N->getValueType(ResNo: 1));
2361
2362	// If the input also splits, handle it directly for a compile time speedup.
2363	// Otherwise split it by hand.
2364	EVT InVT = N->getOperand(Num: 0).getValueType();
2365	if (getTypeAction(VT: InVT) == TargetLowering::TypeSplitVector)
2366	GetSplitVector(Op: N->getOperand(Num: 0), Lo, Hi);
2367	else
2368	std::tie(args&: Lo, args&: Hi) = DAG.SplitVectorOperand(N, OpNo: 0);
2369
2370	Lo = DAG.getNode(Opcode: N->getOpcode(), DL: dl, ResultTys: {LoVT, LoVT1}, Ops: Lo);
2371	Hi = DAG.getNode(Opcode: N->getOpcode(), DL: dl, ResultTys: {HiVT, HiVT1}, Ops: Hi);
2372	Lo->setFlags(N->getFlags());
2373	Hi->setFlags(N->getFlags());
2374
2375	SDNode *HiNode = Hi.getNode();
2376	SDNode *LoNode = Lo.getNode();
2377
2378	// Replace the other vector result not being explicitly split here.
2379	unsigned OtherNo = 1 - ResNo;
2380	EVT OtherVT = N->getValueType(ResNo: OtherNo);
2381	if (getTypeAction(VT: OtherVT) == TargetLowering::TypeSplitVector) {
2382	SetSplitVector(Op: SDValue(N, OtherNo), Lo: SDValue(LoNode, OtherNo),
2383	Hi: SDValue(HiNode, OtherNo));
2384	} else {
2385	SDValue OtherVal =
2386	DAG.getNode(Opcode: ISD::CONCAT_VECTORS, DL: dl, VT: OtherVT, N1: SDValue(LoNode, OtherNo),
2387	N2: SDValue(HiNode, OtherNo));
2388	ReplaceValueWith(From: SDValue(N, OtherNo), To: OtherVal);
2389	}
2390	}
2391
2392	void DAGTypeLegalizer::SplitVecRes_ExtendOp(SDNode *N, SDValue &Lo,
2393	SDValue &Hi) {
2394	SDLoc dl(N);
2395	EVT SrcVT = N->getOperand(Num: 0).getValueType();
2396	EVT DestVT = N->getValueType(ResNo: 0);
2397	EVT LoVT, HiVT;
2398	std::tie(args&: LoVT, args&: HiVT) = DAG.GetSplitDestVTs(VT: DestVT);
2399
2400	// We can do better than a generic split operation if the extend is doing
2401	// more than just doubling the width of the elements and the following are
2402	// true:
2403	// - The number of vector elements is even,
2404	// - the source type is legal,
2405	// - the type of a split source is illegal,
2406	// - the type of an extended (by doubling element size) source is legal, and
2407	// - the type of that extended source when split is legal.
2408	//
2409	// This won't necessarily completely legalize the operation, but it will
2410	// more effectively move in the right direction and prevent falling down
2411	// to scalarization in many cases due to the input vector being split too
2412	// far.
2413	if (SrcVT.getVectorElementCount().isKnownEven() &&
2414	SrcVT.getScalarSizeInBits() * 2 < DestVT.getScalarSizeInBits()) {
2415	LLVMContext &Ctx = *DAG.getContext();
2416	EVT NewSrcVT = SrcVT.widenIntegerVectorElementType(Context&: Ctx);
2417	EVT SplitSrcVT = SrcVT.getHalfNumVectorElementsVT(Context&: Ctx);
2418
2419	EVT SplitLoVT, SplitHiVT;
2420	std::tie(args&: SplitLoVT, args&: SplitHiVT) = DAG.GetSplitDestVTs(VT: NewSrcVT);
2421	if (TLI.isTypeLegal(VT: SrcVT) && !TLI.isTypeLegal(VT: SplitSrcVT) &&
2422	TLI.isTypeLegal(VT: NewSrcVT) && TLI.isTypeLegal(VT: SplitLoVT)) {
2423	LLVM_DEBUG(dbgs() << "Split vector extend via incremental extend:";
2424	N->dump(&DAG); dbgs() << "\n");
2425	if (!N->isVPOpcode()) {
2426	// Extend the source vector by one step.
2427	SDValue NewSrc =
2428	DAG.getNode(Opcode: N->getOpcode(), DL: dl, VT: NewSrcVT, Operand: N->getOperand(Num: 0));
2429	// Get the low and high halves of the new, extended one step, vector.
2430	std::tie(args&: Lo, args&: Hi) = DAG.SplitVector(N: NewSrc, DL: dl);
2431	// Extend those vector halves the rest of the way.
2432	Lo = DAG.getNode(Opcode: N->getOpcode(), DL: dl, VT: LoVT, Operand: Lo);
2433	Hi = DAG.getNode(Opcode: N->getOpcode(), DL: dl, VT: HiVT, Operand: Hi);
2434	return;
2435	}
2436
2437	// Extend the source vector by one step.
2438	SDValue NewSrc =
2439	DAG.getNode(Opcode: N->getOpcode(), DL: dl, VT: NewSrcVT, N1: N->getOperand(Num: 0),
2440	N2: N->getOperand(Num: 1), N3: N->getOperand(Num: 2));
2441	// Get the low and high halves of the new, extended one step, vector.
2442	std::tie(args&: Lo, args&: Hi) = DAG.SplitVector(N: NewSrc, DL: dl);
2443
2444	SDValue MaskLo, MaskHi;
2445	std::tie(args&: MaskLo, args&: MaskHi) = SplitMask(Mask: N->getOperand(Num: 1));
2446
2447	SDValue EVLLo, EVLHi;
2448	std::tie(args&: EVLLo, args&: EVLHi) =
2449	DAG.SplitEVL(N: N->getOperand(Num: 2), VecVT: N->getValueType(ResNo: 0), DL: dl);
2450	// Extend those vector halves the rest of the way.
2451	Lo = DAG.getNode(Opcode: N->getOpcode(), DL: dl, VT: LoVT, Ops: {Lo, MaskLo, EVLLo});
2452	Hi = DAG.getNode(Opcode: N->getOpcode(), DL: dl, VT: HiVT, Ops: {Hi, MaskHi, EVLHi});
2453	return;
2454	}
2455	}
2456	// Fall back to the generic unary operator splitting otherwise.
2457	SplitVecRes_UnaryOp(N, Lo, Hi);
2458	}
2459
2460	void DAGTypeLegalizer::SplitVecRes_VECTOR_SHUFFLE(ShuffleVectorSDNode *N,
2461	SDValue &Lo, SDValue &Hi) {
2462	// The low and high parts of the original input give four input vectors.
2463	SDValue Inputs[4];
2464	SDLoc DL(N);
2465	GetSplitVector(Op: N->getOperand(Num: 0), Lo&: Inputs[0], Hi&: Inputs[1]);
2466	GetSplitVector(Op: N->getOperand(Num: 1), Lo&: Inputs[2], Hi&: Inputs[3]);
2467	EVT NewVT = Inputs[0].getValueType();
2468	unsigned NewElts = NewVT.getVectorNumElements();
2469
2470	auto &&IsConstant = [](const SDValue &N) {
2471	APInt SplatValue;
2472	return N.getResNo() == 0 &&
2473	(ISD::isConstantSplatVector(N: N.getNode(), SplatValue) ||
2474	ISD::isBuildVectorOfConstantSDNodes(N: N.getNode()));
2475	};
2476	auto &&BuildVector = [NewElts, &DAG = DAG, NewVT, &DL](SDValue &Input1,
2477	SDValue &Input2,
2478	ArrayRef<int> Mask) {
2479	assert(Input1->getOpcode() == ISD::BUILD_VECTOR &&
2480	Input2->getOpcode() == ISD::BUILD_VECTOR &&
2481	"Expected build vector node.");
2482	EVT EltVT = NewVT.getVectorElementType();
2483	SmallVector<SDValue> Ops(NewElts, DAG.getUNDEF(VT: EltVT));
2484	for (unsigned I = 0; I < NewElts; ++I) {
2485	if (Mask[I] == PoisonMaskElem)
2486	continue;
2487	unsigned Idx = Mask[I];
2488	if (Idx >= NewElts)
2489	Ops[I] = Input2.getOperand(i: Idx - NewElts);
2490	else
2491	Ops[I] = Input1.getOperand(i: Idx);
2492	// Make the type of all elements the same as the element type.
2493	if (Ops[I].getValueType().bitsGT(VT: EltVT))
2494	Ops[I] = DAG.getNode(Opcode: ISD::TRUNCATE, DL, VT: EltVT, Operand: Ops[I]);
2495	}
2496	return DAG.getBuildVector(VT: NewVT, DL, Ops);
2497	};
2498
2499	// If Lo or Hi uses elements from at most two of the four input vectors, then
2500	// express it as a vector shuffle of those two inputs. Otherwise extract the
2501	// input elements by hand and construct the Lo/Hi output using a BUILD_VECTOR.
2502	SmallVector<int> OrigMask(N->getMask());
2503	// Try to pack incoming shuffles/inputs.
2504	auto &&TryPeekThroughShufflesInputs = [&Inputs, &NewVT, this, NewElts,
2505	&DL](SmallVectorImpl<int> &Mask) {
2506	// Check if all inputs are shuffles of the same operands or non-shuffles.
2507	MapVector<std::pair<SDValue, SDValue>, SmallVector<unsigned>> ShufflesIdxs;
2508	for (unsigned Idx = 0; Idx < std::size(Inputs); ++Idx) {
2509	SDValue Input = Inputs[Idx];
2510	auto *Shuffle = dyn_cast<ShuffleVectorSDNode>(Val: Input.getNode());
2511	if (!Shuffle ||
2512	Input.getOperand(i: 0).getValueType() != Input.getValueType())
2513	continue;
2514	ShufflesIdxs[std::make_pair(x: Input.getOperand(i: 0), y: Input.getOperand(i: 1))]
2515	.push_back(Elt: Idx);
2516	ShufflesIdxs[std::make_pair(x: Input.getOperand(i: 1), y: Input.getOperand(i: 0))]
2517	.push_back(Elt: Idx);
2518	}
2519	for (auto &P : ShufflesIdxs) {
2520	if (P.second.size() < 2)
2521	continue;
2522	// Use shuffles operands instead of shuffles themselves.
2523	// 1. Adjust mask.
2524	for (int &Idx : Mask) {
2525	if (Idx == PoisonMaskElem)
2526	continue;
2527	unsigned SrcRegIdx = Idx / NewElts;
2528	if (Inputs[SrcRegIdx].isUndef()) {
2529	Idx = PoisonMaskElem;
2530	continue;
2531	}
2532	auto *Shuffle =
2533	dyn_cast<ShuffleVectorSDNode>(Val: Inputs[SrcRegIdx].getNode());
2534	if (!Shuffle || !is_contained(Range&: P.second, Element: SrcRegIdx))
2535	continue;
2536	int MaskElt = Shuffle->getMaskElt(Idx: Idx % NewElts);
2537	if (MaskElt == PoisonMaskElem) {
2538	Idx = PoisonMaskElem;
2539	continue;
2540	}
2541	Idx = MaskElt % NewElts +
2542	P.second[Shuffle->getOperand(Num: MaskElt / NewElts) == P.first.first
2543	? 0
2544	: 1] *
2545	NewElts;
2546	}
2547	// 2. Update inputs.
2548	Inputs[P.second[0]] = P.first.first;
2549	Inputs[P.second[1]] = P.first.second;
2550	// Clear the pair data.
2551	P.second.clear();
2552	ShufflesIdxs[std::make_pair(x&: P.first.second, y&: P.first.first)].clear();
2553	}
2554	// Check if any concat_vectors can be simplified.
2555	SmallBitVector UsedSubVector(2 * std::size(Inputs));
2556	for (int &Idx : Mask) {
2557	if (Idx == PoisonMaskElem)
2558	continue;
2559	unsigned SrcRegIdx = Idx / NewElts;
2560	if (Inputs[SrcRegIdx].isUndef()) {
2561	Idx = PoisonMaskElem;
2562	continue;
2563	}
2564	TargetLowering::LegalizeTypeAction TypeAction =
2565	getTypeAction(VT: Inputs[SrcRegIdx].getValueType());
2566	if (Inputs[SrcRegIdx].getOpcode() == ISD::CONCAT_VECTORS &&
2567	Inputs[SrcRegIdx].getNumOperands() == 2 &&
2568	!Inputs[SrcRegIdx].getOperand(i: 1).isUndef() &&
2569	(TypeAction == TargetLowering::TypeLegal ||
2570	TypeAction == TargetLowering::TypeWidenVector))
2571	UsedSubVector.set(2 * SrcRegIdx + (Idx % NewElts) / (NewElts / 2));
2572	}
2573	if (UsedSubVector.count() > 1) {
2574	SmallVector<SmallVector<std::pair<unsigned, int>, 2>> Pairs;
2575	for (unsigned I = 0; I < std::size(Inputs); ++I) {
2576	if (UsedSubVector.test(Idx: 2 * I) == UsedSubVector.test(Idx: 2 * I + 1))
2577	continue;
2578	if (Pairs.empty() || Pairs.back().size() == 2)
2579	Pairs.emplace_back();
2580	if (UsedSubVector.test(Idx: 2 * I)) {
2581	Pairs.back().emplace_back(Args&: I, Args: 0);
2582	} else {
2583	assert(UsedSubVector.test(2 * I + 1) &&
2584	"Expected to be used one of the subvectors.");
2585	Pairs.back().emplace_back(Args&: I, Args: 1);
2586	}
2587	}
2588	if (!Pairs.empty() && Pairs.front().size() > 1) {
2589	// Adjust mask.
2590	for (int &Idx : Mask) {
2591	if (Idx == PoisonMaskElem)
2592	continue;
2593	unsigned SrcRegIdx = Idx / NewElts;
2594	auto *It = find_if(
2595	Range&: Pairs, P: [SrcRegIdx](ArrayRef<std::pair<unsigned, int>> Idxs) {
2596	return Idxs.front().first == SrcRegIdx ||
2597	Idxs.back().first == SrcRegIdx;
2598	});
2599	if (It == Pairs.end())
2600	continue;
2601	Idx = It->front().first * NewElts + (Idx % NewElts) % (NewElts / 2) +
2602	(SrcRegIdx == It->front().first ? 0 : (NewElts / 2));
2603	}
2604	// Adjust inputs.
2605	for (ArrayRef<std::pair<unsigned, int>> Idxs : Pairs) {
2606	Inputs[Idxs.front().first] = DAG.getNode(
2607	Opcode: ISD::CONCAT_VECTORS, DL,
2608	VT: Inputs[Idxs.front().first].getValueType(),
2609	N1: Inputs[Idxs.front().first].getOperand(i: Idxs.front().second),
2610	N2: Inputs[Idxs.back().first].getOperand(i: Idxs.back().second));
2611	}
2612	}
2613	}
2614	bool Changed;
2615	do {
2616	// Try to remove extra shuffles (except broadcasts) and shuffles with the
2617	// reused operands.
2618	Changed = false;
2619	for (unsigned I = 0; I < std::size(Inputs); ++I) {
2620	auto *Shuffle = dyn_cast<ShuffleVectorSDNode>(Val: Inputs[I].getNode());
2621	if (!Shuffle)
2622	continue;
2623	if (Shuffle->getOperand(Num: 0).getValueType() != NewVT)
2624	continue;
2625	int Op = -1;
2626	if (!Inputs[I].hasOneUse() && Shuffle->getOperand(Num: 1).isUndef() &&
2627	!Shuffle->isSplat()) {
2628	Op = 0;
2629	} else if (!Inputs[I].hasOneUse() &&
2630	!Shuffle->getOperand(Num: 1).isUndef()) {
2631	// Find the only used operand, if possible.
2632	for (int &Idx : Mask) {
2633	if (Idx == PoisonMaskElem)
2634	continue;
2635	unsigned SrcRegIdx = Idx / NewElts;
2636	if (SrcRegIdx != I)
2637	continue;
2638	int MaskElt = Shuffle->getMaskElt(Idx: Idx % NewElts);
2639	if (MaskElt == PoisonMaskElem) {
2640	Idx = PoisonMaskElem;
2641	continue;
2642	}
2643	int OpIdx = MaskElt / NewElts;
2644	if (Op == -1) {
2645	Op = OpIdx;
2646	continue;
2647	}
2648	if (Op != OpIdx) {
2649	Op = -1;
2650	break;
2651	}
2652	}
2653	}
2654	if (Op < 0) {
2655	// Try to check if one of the shuffle operands is used already.
2656	for (int OpIdx = 0; OpIdx < 2; ++OpIdx) {
2657	if (Shuffle->getOperand(Num: OpIdx).isUndef())
2658	continue;
2659	auto *It = find(Range&: Inputs, Val: Shuffle->getOperand(Num: OpIdx));
2660	if (It == std::end(arr&: Inputs))
2661	continue;
2662	int FoundOp = std::distance(first: std::begin(arr&: Inputs), last: It);
2663	// Found that operand is used already.
2664	// 1. Fix the mask for the reused operand.
2665	for (int &Idx : Mask) {
2666	if (Idx == PoisonMaskElem)
2667	continue;
2668	unsigned SrcRegIdx = Idx / NewElts;
2669	if (SrcRegIdx != I)
2670	continue;
2671	int MaskElt = Shuffle->getMaskElt(Idx: Idx % NewElts);
2672	if (MaskElt == PoisonMaskElem) {
2673	Idx = PoisonMaskElem;
2674	continue;
2675	}
2676	int MaskIdx = MaskElt / NewElts;
2677	if (OpIdx == MaskIdx)
2678	Idx = MaskElt % NewElts + FoundOp * NewElts;
2679	}
2680	// 2. Set Op to the unused OpIdx.
2681	Op = (OpIdx + 1) % 2;
2682	break;
2683	}
2684	}
2685	if (Op >= 0) {
2686	Changed = true;
2687	Inputs[I] = Shuffle->getOperand(Num: Op);
2688	// Adjust mask.
2689	for (int &Idx : Mask) {
2690	if (Idx == PoisonMaskElem)
2691	continue;
2692	unsigned SrcRegIdx = Idx / NewElts;
2693	if (SrcRegIdx != I)
2694	continue;
2695	int MaskElt = Shuffle->getMaskElt(Idx: Idx % NewElts);
2696	int OpIdx = MaskElt / NewElts;
2697	if (OpIdx != Op)
2698	continue;
2699	Idx = MaskElt % NewElts + SrcRegIdx * NewElts;
2700	}
2701	}
2702	}
2703	} while (Changed);
2704	};
2705	TryPeekThroughShufflesInputs(OrigMask);
2706	// Proces unique inputs.
2707	auto &&MakeUniqueInputs = [&Inputs, &IsConstant,
2708	NewElts](SmallVectorImpl<int> &Mask) {
2709	SetVector<SDValue> UniqueInputs;
2710	SetVector<SDValue> UniqueConstantInputs;
2711	for (const auto &I : Inputs) {
2712	if (IsConstant(I))
2713	UniqueConstantInputs.insert(X: I);
2714	else if (!I.isUndef())
2715	UniqueInputs.insert(X: I);
2716	}
2717	// Adjust mask in case of reused inputs. Also, need to insert constant
2718	// inputs at first, otherwise it affects the final outcome.
2719	if (UniqueInputs.size() != std::size(Inputs)) {
2720	auto &&UniqueVec = UniqueInputs.takeVector();
2721	auto &&UniqueConstantVec = UniqueConstantInputs.takeVector();
2722	unsigned ConstNum = UniqueConstantVec.size();
2723	for (int &Idx : Mask) {
2724	if (Idx == PoisonMaskElem)
2725	continue;
2726	unsigned SrcRegIdx = Idx / NewElts;
2727	if (Inputs[SrcRegIdx].isUndef()) {
2728	Idx = PoisonMaskElem;
2729	continue;
2730	}
2731	const auto It = find(Range&: UniqueConstantVec, Val: Inputs[SrcRegIdx]);
2732	if (It != UniqueConstantVec.end()) {
2733	Idx = (Idx % NewElts) +
2734	NewElts * std::distance(first: UniqueConstantVec.begin(), last: It);
2735	assert(Idx >= 0 && "Expected defined mask idx.");
2736	continue;
2737	}
2738	const auto RegIt = find(Range&: UniqueVec, Val: Inputs[SrcRegIdx]);
2739	assert(RegIt != UniqueVec.end() && "Cannot find non-const value.");
2740	Idx = (Idx % NewElts) +
2741	NewElts * (std::distance(first: UniqueVec.begin(), last: RegIt) + ConstNum);
2742	assert(Idx >= 0 && "Expected defined mask idx.");
2743	}
2744	copy(Range&: UniqueConstantVec, Out: std::begin(arr&: Inputs));
2745	copy(Range&: UniqueVec, Out: std::next(x: std::begin(arr&: Inputs), n: ConstNum));
2746	}
2747	};
2748	MakeUniqueInputs(OrigMask);
2749	SDValue OrigInputs[4];
2750	copy(Range&: Inputs, Out: std::begin(arr&: OrigInputs));
2751	for (unsigned High = 0; High < 2; ++High) {
2752	SDValue &Output = High ? Hi : Lo;
2753
2754	// Build a shuffle mask for the output, discovering on the fly which
2755	// input vectors to use as shuffle operands.
2756	unsigned FirstMaskIdx = High * NewElts;
2757	SmallVector<int> Mask(NewElts * std::size(Inputs), PoisonMaskElem);
2758	copy(Range: ArrayRef(OrigMask).slice(N: FirstMaskIdx, M: NewElts), Out: Mask.begin());
2759	assert(!Output && "Expected default initialized initial value.");
2760	TryPeekThroughShufflesInputs(Mask);
2761	MakeUniqueInputs(Mask);
2762	SDValue TmpInputs[4];
2763	copy(Range&: Inputs, Out: std::begin(arr&: TmpInputs));
2764	// Track changes in the output registers.
2765	int UsedIdx = -1;
2766	bool SecondIteration = false;
2767	auto &&AccumulateResults = [&UsedIdx, &SecondIteration](unsigned Idx) {
2768	if (UsedIdx < 0) {
2769	UsedIdx = Idx;
2770	return false;
2771	}
2772	if (UsedIdx >= 0 && static_cast<unsigned>(UsedIdx) == Idx)
2773	SecondIteration = true;
2774	return SecondIteration;
2775	};
2776	processShuffleMasks(
2777	Mask, NumOfSrcRegs: std::size(Inputs), NumOfDestRegs: std::size(Inputs),
2778	/*NumOfUsedRegs=*/1,
2779	NoInputAction: [&Output, &DAG = DAG, NewVT]() { Output = DAG.getUNDEF(VT: NewVT); },
2780	SingleInputAction: [&Output, &DAG = DAG, NewVT, &DL, &Inputs,
2781	&BuildVector](ArrayRef<int> Mask, unsigned Idx, unsigned /*Unused*/) {
2782	if (Inputs[Idx]->getOpcode() == ISD::BUILD_VECTOR)
2783	Output = BuildVector(Inputs[Idx], Inputs[Idx], Mask);
2784	else
2785	Output = DAG.getVectorShuffle(VT: NewVT, dl: DL, N1: Inputs[Idx],
2786	N2: DAG.getUNDEF(VT: NewVT), Mask);
2787	Inputs[Idx] = Output;
2788	},
2789	ManyInputsAction: [&AccumulateResults, &Output, &DAG = DAG, NewVT, &DL, &Inputs,
2790	&TmpInputs,
2791	&BuildVector](ArrayRef<int> Mask, unsigned Idx1, unsigned Idx2) {
2792	if (AccumulateResults(Idx1)) {
2793	if (Inputs[Idx1]->getOpcode() == ISD::BUILD_VECTOR &&
2794	Inputs[Idx2]->getOpcode() == ISD::BUILD_VECTOR)
2795	Output = BuildVector(Inputs[Idx1], Inputs[Idx2], Mask);
2796	else
2797	Output = DAG.getVectorShuffle(VT: NewVT, dl: DL, N1: Inputs[Idx1],
2798	N2: Inputs[Idx2], Mask);
2799	} else {
2800	if (TmpInputs[Idx1]->getOpcode() == ISD::BUILD_VECTOR &&
2801	TmpInputs[Idx2]->getOpcode() == ISD::BUILD_VECTOR)
2802	Output = BuildVector(TmpInputs[Idx1], TmpInputs[Idx2], Mask);
2803	else
2804	Output = DAG.getVectorShuffle(VT: NewVT, dl: DL, N1: TmpInputs[Idx1],
2805	N2: TmpInputs[Idx2], Mask);
2806	}
2807	Inputs[Idx1] = Output;
2808	});
2809	copy(Range&: OrigInputs, Out: std::begin(arr&: Inputs));
2810	}
2811	}
2812
2813	void DAGTypeLegalizer::SplitVecRes_VAARG(SDNode *N, SDValue &Lo, SDValue &Hi) {
2814	EVT OVT = N->getValueType(ResNo: 0);
2815	EVT NVT = OVT.getHalfNumVectorElementsVT(Context&: *DAG.getContext());
2816	SDValue Chain = N->getOperand(Num: 0);
2817	SDValue Ptr = N->getOperand(Num: 1);
2818	SDValue SV = N->getOperand(Num: 2);
2819	SDLoc dl(N);
2820
2821	const Align Alignment =
2822	DAG.getDataLayout().getABITypeAlign(Ty: NVT.getTypeForEVT(Context&: *DAG.getContext()));
2823
2824	Lo = DAG.getVAArg(VT: NVT, dl, Chain, Ptr, SV, Align: Alignment.value());
2825	Hi = DAG.getVAArg(VT: NVT, dl, Chain: Lo.getValue(R: 1), Ptr, SV, Align: Alignment.value());
2826	Chain = Hi.getValue(R: 1);
2827
2828	// Modified the chain - switch anything that used the old chain to use
2829	// the new one.
2830	ReplaceValueWith(From: SDValue(N, 1), To: Chain);
2831	}
2832
2833	void DAGTypeLegalizer::SplitVecRes_FP_TO_XINT_SAT(SDNode *N, SDValue &Lo,
2834	SDValue &Hi) {
2835	EVT DstVTLo, DstVTHi;
2836	std::tie(args&: DstVTLo, args&: DstVTHi) = DAG.GetSplitDestVTs(VT: N->getValueType(ResNo: 0));
2837	SDLoc dl(N);
2838
2839	SDValue SrcLo, SrcHi;
2840	EVT SrcVT = N->getOperand(Num: 0).getValueType();
2841	if (getTypeAction(VT: SrcVT) == TargetLowering::TypeSplitVector)
2842	GetSplitVector(Op: N->getOperand(Num: 0), Lo&: SrcLo, Hi&: SrcHi);
2843	else
2844	std::tie(args&: SrcLo, args&: SrcHi) = DAG.SplitVectorOperand(N, OpNo: 0);
2845
2846	Lo = DAG.getNode(Opcode: N->getOpcode(), DL: dl, VT: DstVTLo, N1: SrcLo, N2: N->getOperand(Num: 1));
2847	Hi = DAG.getNode(Opcode: N->getOpcode(), DL: dl, VT: DstVTHi, N1: SrcHi, N2: N->getOperand(Num: 1));
2848	}
2849
2850	void DAGTypeLegalizer::SplitVecRes_VECTOR_REVERSE(SDNode *N, SDValue &Lo,
2851	SDValue &Hi) {
2852	SDValue InLo, InHi;
2853	GetSplitVector(Op: N->getOperand(Num: 0), Lo&: InLo, Hi&: InHi);
2854	SDLoc DL(N);
2855
2856	Lo = DAG.getNode(Opcode: ISD::VECTOR_REVERSE, DL, VT: InHi.getValueType(), Operand: InHi);
2857	Hi = DAG.getNode(Opcode: ISD::VECTOR_REVERSE, DL, VT: InLo.getValueType(), Operand: InLo);
2858	}
2859
2860	void DAGTypeLegalizer::SplitVecRes_VECTOR_SPLICE(SDNode *N, SDValue &Lo,
2861	SDValue &Hi) {
2862	EVT VT = N->getValueType(ResNo: 0);
2863	SDLoc DL(N);
2864
2865	EVT LoVT, HiVT;
2866	std::tie(args&: LoVT, args&: HiVT) = DAG.GetSplitDestVTs(VT);
2867
2868	SDValue Expanded = TLI.expandVectorSplice(Node: N, DAG);
2869	Lo = DAG.getNode(Opcode: ISD::EXTRACT_SUBVECTOR, DL, VT: LoVT, N1: Expanded,
2870	N2: DAG.getVectorIdxConstant(Val: 0, DL));
2871	Hi =
2872	DAG.getNode(Opcode: ISD::EXTRACT_SUBVECTOR, DL, VT: HiVT, N1: Expanded,
2873	N2: DAG.getVectorIdxConstant(Val: LoVT.getVectorMinNumElements(), DL));
2874	}
2875
2876	void DAGTypeLegalizer::SplitVecRes_VP_REVERSE(SDNode *N, SDValue &Lo,
2877	SDValue &Hi) {
2878	EVT VT = N->getValueType(ResNo: 0);
2879	SDValue Val = N->getOperand(Num: 0);
2880	SDValue Mask = N->getOperand(Num: 1);
2881	SDValue EVL = N->getOperand(Num: 2);
2882	SDLoc DL(N);
2883
2884	// Fallback to VP_STRIDED_STORE to stack followed by VP_LOAD.
2885	Align Alignment = DAG.getReducedAlign(VT, /*UseABI=*/false);
2886
2887	EVT MemVT = EVT::getVectorVT(Context&: *DAG.getContext(), VT: VT.getVectorElementType(),
2888	EC: VT.getVectorElementCount());
2889	SDValue StackPtr = DAG.CreateStackTemporary(Bytes: MemVT.getStoreSize(), Alignment);
2890	EVT PtrVT = StackPtr.getValueType();
2891	auto &MF = DAG.getMachineFunction();
2892	auto FrameIndex = cast<FrameIndexSDNode>(Val: StackPtr.getNode())->getIndex();
2893	auto PtrInfo = MachinePointerInfo::getFixedStack(MF, FI: FrameIndex);
2894
2895	MachineMemOperand *StoreMMO = DAG.getMachineFunction().getMachineMemOperand(
2896	PtrInfo, F: MachineMemOperand::MOStore, Size: LocationSize::beforeOrAfterPointer(),
2897	BaseAlignment: Alignment);
2898	MachineMemOperand *LoadMMO = DAG.getMachineFunction().getMachineMemOperand(
2899	PtrInfo, F: MachineMemOperand::MOLoad, Size: LocationSize::beforeOrAfterPointer(),
2900	BaseAlignment: Alignment);
2901
2902	unsigned EltWidth = VT.getScalarSizeInBits() / 8;
2903	SDValue NumElemMinus1 =
2904	DAG.getNode(Opcode: ISD::SUB, DL, VT: PtrVT, N1: DAG.getZExtOrTrunc(Op: EVL, DL, VT: PtrVT),
2905	N2: DAG.getConstant(Val: 1, DL, VT: PtrVT));
2906	SDValue StartOffset = DAG.getNode(Opcode: ISD::MUL, DL, VT: PtrVT, N1: NumElemMinus1,
2907	N2: DAG.getConstant(Val: EltWidth, DL, VT: PtrVT));
2908	SDValue StorePtr = DAG.getNode(Opcode: ISD::ADD, DL, VT: PtrVT, N1: StackPtr, N2: StartOffset);
2909	SDValue Stride = DAG.getConstant(Val: -(int64_t)EltWidth, DL, VT: PtrVT);
2910
2911	SDValue TrueMask = DAG.getBoolConstant(V: true, DL, VT: Mask.getValueType(), OpVT: VT);
2912	SDValue Store = DAG.getStridedStoreVP(Chain: DAG.getEntryNode(), DL, Val, Ptr: StorePtr,
2913	Offset: DAG.getUNDEF(VT: PtrVT), Stride, Mask: TrueMask,
2914	EVL, MemVT, MMO: StoreMMO, AM: ISD::UNINDEXED);
2915
2916	SDValue Load = DAG.getLoadVP(VT, dl: DL, Chain: Store, Ptr: StackPtr, Mask, EVL, MMO: LoadMMO);
2917
2918	auto [LoVT, HiVT] = DAG.GetSplitDestVTs(VT);
2919	Lo = DAG.getNode(Opcode: ISD::EXTRACT_SUBVECTOR, DL, VT: LoVT, N1: Load,
2920	N2: DAG.getVectorIdxConstant(Val: 0, DL));
2921	Hi =
2922	DAG.getNode(Opcode: ISD::EXTRACT_SUBVECTOR, DL, VT: HiVT, N1: Load,
2923	N2: DAG.getVectorIdxConstant(Val: LoVT.getVectorMinNumElements(), DL));
2924	}
2925
2926	void DAGTypeLegalizer::SplitVecRes_VECTOR_DEINTERLEAVE(SDNode *N) {
2927
2928	SDValue Op0Lo, Op0Hi, Op1Lo, Op1Hi;
2929	GetSplitVector(Op: N->getOperand(Num: 0), Lo&: Op0Lo, Hi&: Op0Hi);
2930	GetSplitVector(Op: N->getOperand(Num: 1), Lo&: Op1Lo, Hi&: Op1Hi);
2931	EVT VT = Op0Lo.getValueType();
2932	SDLoc DL(N);
2933	SDValue ResLo = DAG.getNode(Opcode: ISD::VECTOR_DEINTERLEAVE, DL,
2934	VTList: DAG.getVTList(VT1: VT, VT2: VT), N1: Op0Lo, N2: Op0Hi);
2935	SDValue ResHi = DAG.getNode(Opcode: ISD::VECTOR_DEINTERLEAVE, DL,
2936	VTList: DAG.getVTList(VT1: VT, VT2: VT), N1: Op1Lo, N2: Op1Hi);
2937
2938	SetSplitVector(Op: SDValue(N, 0), Lo: ResLo.getValue(R: 0), Hi: ResHi.getValue(R: 0));
2939	SetSplitVector(Op: SDValue(N, 1), Lo: ResLo.getValue(R: 1), Hi: ResHi.getValue(R: 1));
2940	}
2941
2942	void DAGTypeLegalizer::SplitVecRes_VECTOR_INTERLEAVE(SDNode *N) {
2943	SDValue Op0Lo, Op0Hi, Op1Lo, Op1Hi;
2944	GetSplitVector(Op: N->getOperand(Num: 0), Lo&: Op0Lo, Hi&: Op0Hi);
2945	GetSplitVector(Op: N->getOperand(Num: 1), Lo&: Op1Lo, Hi&: Op1Hi);
2946	EVT VT = Op0Lo.getValueType();
2947	SDLoc DL(N);
2948	SDValue Res[] = {DAG.getNode(Opcode: ISD::VECTOR_INTERLEAVE, DL,
2949	VTList: DAG.getVTList(VT1: VT, VT2: VT), N1: Op0Lo, N2: Op1Lo),
2950	DAG.getNode(Opcode: ISD::VECTOR_INTERLEAVE, DL,
2951	VTList: DAG.getVTList(VT1: VT, VT2: VT), N1: Op0Hi, N2: Op1Hi)};
2952
2953	SetSplitVector(Op: SDValue(N, 0), Lo: Res[0].getValue(R: 0), Hi: Res[0].getValue(R: 1));
2954	SetSplitVector(Op: SDValue(N, 1), Lo: Res[1].getValue(R: 0), Hi: Res[1].getValue(R: 1));
2955	}
2956
2957	//===----------------------------------------------------------------------===//
2958	// Operand Vector Splitting
2959	//===----------------------------------------------------------------------===//
2960
2961	/// This method is called when the specified operand of the specified node is
2962	/// found to need vector splitting. At this point, all of the result types of
2963	/// the node are known to be legal, but other operands of the node may need
2964	/// legalization as well as the specified one.
2965	bool DAGTypeLegalizer::SplitVectorOperand(SDNode *N, unsigned OpNo) {
2966	LLVM_DEBUG(dbgs() << "Split node operand: "; N->dump(&DAG));
2967	SDValue Res = SDValue();
2968
2969	// See if the target wants to custom split this node.
2970	if (CustomLowerNode(N, VT: N->getOperand(Num: OpNo).getValueType(), LegalizeResult: false))
2971	return false;
2972
2973	switch (N->getOpcode()) {
2974	default:
2975	#ifndef NDEBUG
2976	dbgs() << "SplitVectorOperand Op #" << OpNo << ": ";
2977	N->dump(G: &DAG);
2978	dbgs() << "\n";
2979	#endif
2980	report_fatal_error(reason: "Do not know how to split this operator's "
2981	"operand!\n");
2982
2983	case ISD::VP_SETCC:
2984	case ISD::SETCC: Res = SplitVecOp_VSETCC(N); break;
2985	case ISD::BITCAST: Res = SplitVecOp_BITCAST(N); break;
2986	case ISD::EXTRACT_SUBVECTOR: Res = SplitVecOp_EXTRACT_SUBVECTOR(N); break;
2987	case ISD::INSERT_SUBVECTOR: Res = SplitVecOp_INSERT_SUBVECTOR(N, OpNo); break;
2988	case ISD::EXTRACT_VECTOR_ELT:Res = SplitVecOp_EXTRACT_VECTOR_ELT(N); break;
2989	case ISD::CONCAT_VECTORS: Res = SplitVecOp_CONCAT_VECTORS(N); break;
2990	case ISD::VP_TRUNCATE:
2991	case ISD::TRUNCATE:
2992	Res = SplitVecOp_TruncateHelper(N);
2993	break;
2994	case ISD::STRICT_FP_ROUND:
2995	case ISD::VP_FP_ROUND:
2996	case ISD::FP_ROUND: Res = SplitVecOp_FP_ROUND(N); break;
2997	case ISD::FCOPYSIGN: Res = SplitVecOp_FPOpDifferentTypes(N); break;
2998	case ISD::STORE:
2999	Res = SplitVecOp_STORE(N: cast<StoreSDNode>(Val: N), OpNo);
3000	break;
3001	case ISD::VP_STORE:
3002	Res = SplitVecOp_VP_STORE(N: cast<VPStoreSDNode>(Val: N), OpNo);
3003	break;
3004	case ISD::EXPERIMENTAL_VP_STRIDED_STORE:
3005	Res = SplitVecOp_VP_STRIDED_STORE(N: cast<VPStridedStoreSDNode>(Val: N), OpNo);
3006	break;
3007	case ISD::MSTORE:
3008	Res = SplitVecOp_MSTORE(N: cast<MaskedStoreSDNode>(Val: N), OpNo);
3009	break;
3010	case ISD::MSCATTER:
3011	case ISD::VP_SCATTER:
3012	Res = SplitVecOp_Scatter(N: cast<MemSDNode>(Val: N), OpNo);
3013	break;
3014	case ISD::MGATHER:
3015	case ISD::VP_GATHER:
3016	Res = SplitVecOp_Gather(MGT: cast<MemSDNode>(Val: N), OpNo);
3017	break;
3018	case ISD::VSELECT:
3019	Res = SplitVecOp_VSELECT(N, OpNo);
3020	break;
3021	case ISD::STRICT_SINT_TO_FP:
3022	case ISD::STRICT_UINT_TO_FP:
3023	case ISD::SINT_TO_FP:
3024	case ISD::UINT_TO_FP:
3025	case ISD::VP_SINT_TO_FP:
3026	case ISD::VP_UINT_TO_FP:
3027	if (N->getValueType(ResNo: 0).bitsLT(
3028	VT: N->getOperand(Num: N->isStrictFPOpcode() ? 1 : 0).getValueType()))
3029	Res = SplitVecOp_TruncateHelper(N);
3030	else
3031	Res = SplitVecOp_UnaryOp(N);
3032	break;
3033	case ISD::FP_TO_SINT_SAT:
3034	case ISD::FP_TO_UINT_SAT:
3035	Res = SplitVecOp_FP_TO_XINT_SAT(N);
3036	break;
3037	case ISD::FP_TO_SINT:
3038	case ISD::FP_TO_UINT:
3039	case ISD::VP_FP_TO_SINT:
3040	case ISD::VP_FP_TO_UINT:
3041	case ISD::STRICT_FP_TO_SINT:
3042	case ISD::STRICT_FP_TO_UINT:
3043	case ISD::STRICT_FP_EXTEND:
3044	case ISD::FP_EXTEND:
3045	case ISD::SIGN_EXTEND:
3046	case ISD::ZERO_EXTEND:
3047	case ISD::ANY_EXTEND:
3048	case ISD::FTRUNC:
3049	case ISD::LRINT:
3050	case ISD::LLRINT:
3051	Res = SplitVecOp_UnaryOp(N);
3052	break;
3053	case ISD::FLDEXP:
3054	Res = SplitVecOp_FPOpDifferentTypes(N);
3055	break;
3056
3057	case ISD::ANY_EXTEND_VECTOR_INREG:
3058	case ISD::SIGN_EXTEND_VECTOR_INREG:
3059	case ISD::ZERO_EXTEND_VECTOR_INREG:
3060	Res = SplitVecOp_ExtVecInRegOp(N);
3061	break;
3062
3063	case ISD::VECREDUCE_FADD:
3064	case ISD::VECREDUCE_FMUL:
3065	case ISD::VECREDUCE_ADD:
3066	case ISD::VECREDUCE_MUL:
3067	case ISD::VECREDUCE_AND:
3068	case ISD::VECREDUCE_OR:
3069	case ISD::VECREDUCE_XOR:
3070	case ISD::VECREDUCE_SMAX:
3071	case ISD::VECREDUCE_SMIN:
3072	case ISD::VECREDUCE_UMAX:
3073	case ISD::VECREDUCE_UMIN:
3074	case ISD::VECREDUCE_FMAX:
3075	case ISD::VECREDUCE_FMIN:
3076	case ISD::VECREDUCE_FMAXIMUM:
3077	case ISD::VECREDUCE_FMINIMUM:
3078	Res = SplitVecOp_VECREDUCE(N, OpNo);
3079	break;
3080	case ISD::VECREDUCE_SEQ_FADD:
3081	case ISD::VECREDUCE_SEQ_FMUL:
3082	Res = SplitVecOp_VECREDUCE_SEQ(N);
3083	break;
3084	case ISD::VP_REDUCE_FADD:
3085	case ISD::VP_REDUCE_SEQ_FADD:
3086	case ISD::VP_REDUCE_FMUL:
3087	case ISD::VP_REDUCE_SEQ_FMUL:
3088	case ISD::VP_REDUCE_ADD:
3089	case ISD::VP_REDUCE_MUL:
3090	case ISD::VP_REDUCE_AND:
3091	case ISD::VP_REDUCE_OR:
3092	case ISD::VP_REDUCE_XOR:
3093	case ISD::VP_REDUCE_SMAX:
3094	case ISD::VP_REDUCE_SMIN:
3095	case ISD::VP_REDUCE_UMAX:
3096	case ISD::VP_REDUCE_UMIN:
3097	case ISD::VP_REDUCE_FMAX:
3098	case ISD::VP_REDUCE_FMIN:
3099	Res = SplitVecOp_VP_REDUCE(N, OpNo);
3100	break;
3101	}
3102
3103	// If the result is null, the sub-method took care of registering results etc.
3104	if (!Res.getNode()) return false;
3105
3106	// If the result is N, the sub-method updated N in place. Tell the legalizer
3107	// core about this.
3108	if (Res.getNode() == N)
3109	return true;
3110
3111	if (N->isStrictFPOpcode())
3112	assert(Res.getValueType() == N->getValueType(0) && N->getNumValues() == 2 &&
3113	"Invalid operand expansion");
3114	else
3115	assert(Res.getValueType() == N->getValueType(0) && N->getNumValues() == 1 &&
3116	"Invalid operand expansion");
3117
3118	ReplaceValueWith(From: SDValue(N, 0), To: Res);
3119	return false;
3120	}
3121
3122	SDValue DAGTypeLegalizer::SplitVecOp_VSELECT(SDNode *N, unsigned OpNo) {
3123	// The only possibility for an illegal operand is the mask, since result type
3124	// legalization would have handled this node already otherwise.
3125	assert(OpNo == 0 && "Illegal operand must be mask");
3126
3127	SDValue Mask = N->getOperand(Num: 0);
3128	SDValue Src0 = N->getOperand(Num: 1);
3129	SDValue Src1 = N->getOperand(Num: 2);
3130	EVT Src0VT = Src0.getValueType();
3131	SDLoc DL(N);
3132	assert(Mask.getValueType().isVector() && "VSELECT without a vector mask?");
3133
3134	SDValue Lo, Hi;
3135	GetSplitVector(Op: N->getOperand(Num: 0), Lo, Hi);
3136	assert(Lo.getValueType() == Hi.getValueType() &&
3137	"Lo and Hi have differing types");
3138
3139	EVT LoOpVT, HiOpVT;
3140	std::tie(args&: LoOpVT, args&: HiOpVT) = DAG.GetSplitDestVTs(VT: Src0VT);
3141	assert(LoOpVT == HiOpVT && "Asymmetric vector split?");
3142
3143	SDValue LoOp0, HiOp0, LoOp1, HiOp1, LoMask, HiMask;
3144	std::tie(args&: LoOp0, args&: HiOp0) = DAG.SplitVector(N: Src0, DL);
3145	std::tie(args&: LoOp1, args&: HiOp1) = DAG.SplitVector(N: Src1, DL);
3146	std::tie(args&: LoMask, args&: HiMask) = DAG.SplitVector(N: Mask, DL);
3147
3148	SDValue LoSelect =
3149	DAG.getNode(Opcode: ISD::VSELECT, DL, VT: LoOpVT, N1: LoMask, N2: LoOp0, N3: LoOp1);
3150	SDValue HiSelect =
3151	DAG.getNode(Opcode: ISD::VSELECT, DL, VT: HiOpVT, N1: HiMask, N2: HiOp0, N3: HiOp1);
3152
3153	return DAG.getNode(Opcode: ISD::CONCAT_VECTORS, DL, VT: Src0VT, N1: LoSelect, N2: HiSelect);
3154	}
3155
3156	SDValue DAGTypeLegalizer::SplitVecOp_VECREDUCE(SDNode *N, unsigned OpNo) {
3157	EVT ResVT = N->getValueType(ResNo: 0);
3158	SDValue Lo, Hi;
3159	SDLoc dl(N);
3160
3161	SDValue VecOp = N->getOperand(Num: OpNo);
3162	EVT VecVT = VecOp.getValueType();
3163	assert(VecVT.isVector() && "Can only split reduce vector operand");
3164	GetSplitVector(Op: VecOp, Lo, Hi);
3165	EVT LoOpVT, HiOpVT;
3166	std::tie(args&: LoOpVT, args&: HiOpVT) = DAG.GetSplitDestVTs(VT: VecVT);
3167
3168	// Use the appropriate scalar instruction on the split subvectors before
3169	// reducing the now partially reduced smaller vector.
3170	unsigned CombineOpc = ISD::getVecReduceBaseOpcode(VecReduceOpcode: N->getOpcode());
3171	SDValue Partial = DAG.getNode(Opcode: CombineOpc, DL: dl, VT: LoOpVT, N1: Lo, N2: Hi, Flags: N->getFlags());
3172	return DAG.getNode(Opcode: N->getOpcode(), DL: dl, VT: ResVT, Operand: Partial, Flags: N->getFlags());
3173	}
3174
3175	SDValue DAGTypeLegalizer::SplitVecOp_VECREDUCE_SEQ(SDNode *N) {
3176	EVT ResVT = N->getValueType(ResNo: 0);
3177	SDValue Lo, Hi;
3178	SDLoc dl(N);
3179
3180	SDValue AccOp = N->getOperand(Num: 0);
3181	SDValue VecOp = N->getOperand(Num: 1);
3182	SDNodeFlags Flags = N->getFlags();
3183
3184	EVT VecVT = VecOp.getValueType();
3185	assert(VecVT.isVector() && "Can only split reduce vector operand");
3186	GetSplitVector(Op: VecOp, Lo, Hi);
3187	EVT LoOpVT, HiOpVT;
3188	std::tie(args&: LoOpVT, args&: HiOpVT) = DAG.GetSplitDestVTs(VT: VecVT);
3189
3190	// Reduce low half.
3191	SDValue Partial = DAG.getNode(Opcode: N->getOpcode(), DL: dl, VT: ResVT, N1: AccOp, N2: Lo, Flags);
3192
3193	// Reduce high half, using low half result as initial value.
3194	return DAG.getNode(Opcode: N->getOpcode(), DL: dl, VT: ResVT, N1: Partial, N2: Hi, Flags);
3195	}
3196
3197	SDValue DAGTypeLegalizer::SplitVecOp_VP_REDUCE(SDNode *N, unsigned OpNo) {
3198	assert(N->isVPOpcode() && "Expected VP opcode");
3199	assert(OpNo == 1 && "Can only split reduce vector operand");
3200
3201	unsigned Opc = N->getOpcode();
3202	EVT ResVT = N->getValueType(ResNo: 0);
3203	SDValue Lo, Hi;
3204	SDLoc dl(N);
3205
3206	SDValue VecOp = N->getOperand(Num: OpNo);
3207	EVT VecVT = VecOp.getValueType();
3208	assert(VecVT.isVector() && "Can only split reduce vector operand");
3209	GetSplitVector(Op: VecOp, Lo, Hi);
3210
3211	SDValue MaskLo, MaskHi;
3212	std::tie(args&: MaskLo, args&: MaskHi) = SplitMask(Mask: N->getOperand(Num: 2));
3213
3214	SDValue EVLLo, EVLHi;
3215	std::tie(args&: EVLLo, args&: EVLHi) = DAG.SplitEVL(N: N->getOperand(Num: 3), VecVT, DL: dl);
3216
3217	const SDNodeFlags Flags = N->getFlags();
3218
3219	SDValue ResLo =
3220	DAG.getNode(Opcode: Opc, DL: dl, VT: ResVT, Ops: {N->getOperand(Num: 0), Lo, MaskLo, EVLLo}, Flags);
3221	return DAG.getNode(Opcode: Opc, DL: dl, VT: ResVT, Ops: {ResLo, Hi, MaskHi, EVLHi}, Flags);
3222	}
3223
3224	SDValue DAGTypeLegalizer::SplitVecOp_UnaryOp(SDNode *N) {
3225	// The result has a legal vector type, but the input needs splitting.
3226	EVT ResVT = N->getValueType(ResNo: 0);
3227	SDValue Lo, Hi;
3228	SDLoc dl(N);
3229	GetSplitVector(Op: N->getOperand(Num: N->isStrictFPOpcode() ? 1 : 0), Lo, Hi);
3230	EVT InVT = Lo.getValueType();
3231
3232	EVT OutVT = EVT::getVectorVT(Context&: *DAG.getContext(), VT: ResVT.getVectorElementType(),
3233	EC: InVT.getVectorElementCount());
3234
3235	if (N->isStrictFPOpcode()) {
3236	Lo = DAG.getNode(N->getOpcode(), dl, { OutVT, MVT::Other },
3237	{ N->getOperand(0), Lo });
3238	Hi = DAG.getNode(N->getOpcode(), dl, { OutVT, MVT::Other },
3239	{ N->getOperand(0), Hi });
3240
3241	// Build a factor node to remember that this operation is independent
3242	// of the other one.
3243	SDValue Ch = DAG.getNode(ISD::TokenFactor, dl, MVT::Other, Lo.getValue(R: 1),
3244	Hi.getValue(R: 1));
3245
3246	// Legalize the chain result - switch anything that used the old chain to
3247	// use the new one.
3248	ReplaceValueWith(From: SDValue(N, 1), To: Ch);
3249	} else if (N->getNumOperands() == 3) {
3250	assert(N->isVPOpcode() && "Expected VP opcode");
3251	SDValue MaskLo, MaskHi, EVLLo, EVLHi;
3252	std::tie(args&: MaskLo, args&: MaskHi) = SplitMask(Mask: N->getOperand(Num: 1));
3253	std::tie(args&: EVLLo, args&: EVLHi) =
3254	DAG.SplitEVL(N: N->getOperand(Num: 2), VecVT: N->getValueType(ResNo: 0), DL: dl);
3255	Lo = DAG.getNode(Opcode: N->getOpcode(), DL: dl, VT: OutVT, N1: Lo, N2: MaskLo, N3: EVLLo);
3256	Hi = DAG.getNode(Opcode: N->getOpcode(), DL: dl, VT: OutVT, N1: Hi, N2: MaskHi, N3: EVLHi);
3257	} else {
3258	Lo = DAG.getNode(Opcode: N->getOpcode(), DL: dl, VT: OutVT, Operand: Lo);
3259	Hi = DAG.getNode(Opcode: N->getOpcode(), DL: dl, VT: OutVT, Operand: Hi);
3260	}
3261
3262	return DAG.getNode(Opcode: ISD::CONCAT_VECTORS, DL: dl, VT: ResVT, N1: Lo, N2: Hi);
3263	}
3264
3265	SDValue DAGTypeLegalizer::SplitVecOp_BITCAST(SDNode *N) {
3266	// For example, i64 = BITCAST v4i16 on alpha. Typically the vector will
3267	// end up being split all the way down to individual components. Convert the
3268	// split pieces into integers and reassemble.
3269	EVT ResVT = N->getValueType(ResNo: 0);
3270	SDValue Lo, Hi;
3271	GetSplitVector(Op: N->getOperand(Num: 0), Lo, Hi);
3272	SDLoc dl(N);
3273
3274	if (ResVT.isScalableVector()) {
3275	auto [LoVT, HiVT] = DAG.GetSplitDestVTs(VT: ResVT);
3276	Lo = DAG.getNode(Opcode: ISD::BITCAST, DL: dl, VT: LoVT, Operand: Lo);
3277	Hi = DAG.getNode(Opcode: ISD::BITCAST, DL: dl, VT: HiVT, Operand: Hi);
3278	return DAG.getNode(Opcode: ISD::CONCAT_VECTORS, DL: dl, VT: ResVT, N1: Lo, N2: Hi);
3279	}
3280
3281	Lo = BitConvertToInteger(Op: Lo);
3282	Hi = BitConvertToInteger(Op: Hi);
3283
3284	if (DAG.getDataLayout().isBigEndian())
3285	std::swap(a&: Lo, b&: Hi);
3286
3287	return DAG.getNode(Opcode: ISD::BITCAST, DL: dl, VT: ResVT, Operand: JoinIntegers(Lo, Hi));
3288	}
3289
3290	SDValue DAGTypeLegalizer::SplitVecOp_INSERT_SUBVECTOR(SDNode *N,
3291	unsigned OpNo) {
3292	assert(OpNo == 1 && "Invalid OpNo; can only split SubVec.");
3293	// We know that the result type is legal.
3294	EVT ResVT = N->getValueType(ResNo: 0);
3295
3296	SDValue Vec = N->getOperand(Num: 0);
3297	SDValue SubVec = N->getOperand(Num: 1);
3298	SDValue Idx = N->getOperand(Num: 2);
3299	SDLoc dl(N);
3300
3301	SDValue Lo, Hi;
3302	GetSplitVector(Op: SubVec, Lo, Hi);
3303
3304	uint64_t IdxVal = Idx->getAsZExtVal();
3305	uint64_t LoElts = Lo.getValueType().getVectorMinNumElements();
3306
3307	SDValue FirstInsertion =
3308	DAG.getNode(Opcode: ISD::INSERT_SUBVECTOR, DL: dl, VT: ResVT, N1: Vec, N2: Lo, N3: Idx);
3309	SDValue SecondInsertion =
3310	DAG.getNode(Opcode: ISD::INSERT_SUBVECTOR, DL: dl, VT: ResVT, N1: FirstInsertion, N2: Hi,
3311	N3: DAG.getVectorIdxConstant(Val: IdxVal + LoElts, DL: dl));
3312
3313	return SecondInsertion;
3314	}
3315
3316	SDValue DAGTypeLegalizer::SplitVecOp_EXTRACT_SUBVECTOR(SDNode *N) {
3317	// We know that the extracted result type is legal.
3318	EVT SubVT = N->getValueType(ResNo: 0);
3319	SDValue Idx = N->getOperand(Num: 1);
3320	SDLoc dl(N);
3321	SDValue Lo, Hi;
3322
3323	GetSplitVector(Op: N->getOperand(Num: 0), Lo, Hi);
3324
3325	uint64_t LoEltsMin = Lo.getValueType().getVectorMinNumElements();
3326	uint64_t IdxVal = Idx->getAsZExtVal();
3327
3328	if (IdxVal < LoEltsMin) {
3329	assert(IdxVal + SubVT.getVectorMinNumElements() <= LoEltsMin &&
3330	"Extracted subvector crosses vector split!");
3331	return DAG.getNode(Opcode: ISD::EXTRACT_SUBVECTOR, DL: dl, VT: SubVT, N1: Lo, N2: Idx);
3332	} else if (SubVT.isScalableVector() ==
3333	N->getOperand(Num: 0).getValueType().isScalableVector())
3334	return DAG.getNode(Opcode: ISD::EXTRACT_SUBVECTOR, DL: dl, VT: SubVT, N1: Hi,
3335	N2: DAG.getVectorIdxConstant(Val: IdxVal - LoEltsMin, DL: dl));
3336
3337	// After this point the DAG node only permits extracting fixed-width
3338	// subvectors from scalable vectors.
3339	assert(SubVT.isFixedLengthVector() &&
3340	"Extracting scalable subvector from fixed-width unsupported");
3341
3342	// If the element type is i1 and we're not promoting the result, then we may
3343	// end up loading the wrong data since the bits are packed tightly into
3344	// bytes. For example, if we extract a v4i1 (legal) from a nxv4i1 (legal)
3345	// type at index 4, then we will load a byte starting at index 0.
3346	if (SubVT.getScalarType() == MVT::i1)
3347	report_fatal_error(reason: "Don't know how to extract fixed-width predicate "
3348	"subvector from a scalable predicate vector");
3349
3350	// Spill the vector to the stack. We should use the alignment for
3351	// the smallest part.
3352	SDValue Vec = N->getOperand(Num: 0);
3353	EVT VecVT = Vec.getValueType();
3354	Align SmallestAlign = DAG.getReducedAlign(VT: VecVT, /*UseABI=*/false);
3355	SDValue StackPtr =
3356	DAG.CreateStackTemporary(Bytes: VecVT.getStoreSize(), Alignment: SmallestAlign);
3357	auto &MF = DAG.getMachineFunction();
3358	auto FrameIndex = cast<FrameIndexSDNode>(Val: StackPtr.getNode())->getIndex();
3359	auto PtrInfo = MachinePointerInfo::getFixedStack(MF, FI: FrameIndex);
3360
3361	SDValue Store = DAG.getStore(Chain: DAG.getEntryNode(), dl, Val: Vec, Ptr: StackPtr, PtrInfo,
3362	Alignment: SmallestAlign);
3363
3364	// Extract the subvector by loading the correct part.
3365	StackPtr = TLI.getVectorSubVecPointer(DAG, VecPtr: StackPtr, VecVT, SubVecVT: SubVT, Index: Idx);
3366
3367	return DAG.getLoad(
3368	VT: SubVT, dl, Chain: Store, Ptr: StackPtr,
3369	PtrInfo: MachinePointerInfo::getUnknownStack(MF&: DAG.getMachineFunction()));
3370	}
3371
3372	SDValue DAGTypeLegalizer::SplitVecOp_EXTRACT_VECTOR_ELT(SDNode *N) {
3373	SDValue Vec = N->getOperand(Num: 0);
3374	SDValue Idx = N->getOperand(Num: 1);
3375	EVT VecVT = Vec.getValueType();
3376
3377	if (const ConstantSDNode *Index = dyn_cast<ConstantSDNode>(Val&: Idx)) {
3378	uint64_t IdxVal = Index->getZExtValue();
3379
3380	SDValue Lo, Hi;
3381	GetSplitVector(Op: Vec, Lo, Hi);
3382
3383	uint64_t LoElts = Lo.getValueType().getVectorMinNumElements();
3384
3385	if (IdxVal < LoElts)
3386	return SDValue(DAG.UpdateNodeOperands(N, Op1: Lo, Op2: Idx), 0);
3387	else if (!Vec.getValueType().isScalableVector())
3388	return SDValue(DAG.UpdateNodeOperands(N, Op1: Hi,
3389	Op2: DAG.getConstant(Val: IdxVal - LoElts, DL: SDLoc(N),
3390	VT: Idx.getValueType())), 0);
3391	}
3392
3393	// See if the target wants to custom expand this node.
3394	if (CustomLowerNode(N, VT: N->getValueType(ResNo: 0), LegalizeResult: true))
3395	return SDValue();
3396
3397	// Make the vector elements byte-addressable if they aren't already.
3398	SDLoc dl(N);
3399	EVT EltVT = VecVT.getVectorElementType();
3400	if (VecVT.getScalarSizeInBits() < 8) {
3401	EltVT = MVT::i8;
3402	VecVT = EVT::getVectorVT(Context&: *DAG.getContext(), VT: EltVT,
3403	EC: VecVT.getVectorElementCount());
3404	Vec = DAG.getNode(Opcode: ISD::ANY_EXTEND, DL: dl, VT: VecVT, Operand: Vec);
3405	}
3406
3407	// Store the vector to the stack.
3408	// In cases where the vector is illegal it will be broken down into parts
3409	// and stored in parts - we should use the alignment for the smallest part.
3410	Align SmallestAlign = DAG.getReducedAlign(VT: VecVT, /*UseABI=*/false);
3411	SDValue StackPtr =
3412	DAG.CreateStackTemporary(Bytes: VecVT.getStoreSize(), Alignment: SmallestAlign);
3413	auto &MF = DAG.getMachineFunction();
3414	auto FrameIndex = cast<FrameIndexSDNode>(Val: StackPtr.getNode())->getIndex();
3415	auto PtrInfo = MachinePointerInfo::getFixedStack(MF, FI: FrameIndex);
3416	SDValue Store = DAG.getStore(Chain: DAG.getEntryNode(), dl, Val: Vec, Ptr: StackPtr, PtrInfo,
3417	Alignment: SmallestAlign);
3418
3419	// Load back the required element.
3420	StackPtr = TLI.getVectorElementPointer(DAG, VecPtr: StackPtr, VecVT, Index: Idx);
3421
3422	// FIXME: This is to handle i1 vectors with elements promoted to i8.
3423	// i1 vector handling needs general improvement.
3424	if (N->getValueType(ResNo: 0).bitsLT(VT: EltVT)) {
3425	SDValue Load = DAG.getLoad(VT: EltVT, dl, Chain: Store, Ptr: StackPtr,
3426	PtrInfo: MachinePointerInfo::getUnknownStack(MF&: DAG.getMachineFunction()));
3427	return DAG.getZExtOrTrunc(Op: Load, DL: dl, VT: N->getValueType(ResNo: 0));
3428	}
3429
3430	return DAG.getExtLoad(
3431	ExtType: ISD::EXTLOAD, dl, VT: N->getValueType(ResNo: 0), Chain: Store, Ptr: StackPtr,
3432	PtrInfo: MachinePointerInfo::getUnknownStack(MF&: DAG.getMachineFunction()), MemVT: EltVT,
3433	Alignment: commonAlignment(A: SmallestAlign, Offset: EltVT.getFixedSizeInBits() / 8));
3434	}
3435
3436	SDValue DAGTypeLegalizer::SplitVecOp_ExtVecInRegOp(SDNode *N) {
3437	SDValue Lo, Hi;
3438
3439	// *_EXTEND_VECTOR_INREG only reference the lower half of the input, so
3440	// splitting the result has the same effect as splitting the input operand.
3441	SplitVecRes_ExtVecInRegOp(N, Lo, Hi);
3442
3443	return DAG.getNode(Opcode: ISD::CONCAT_VECTORS, DL: SDLoc(N), VT: N->getValueType(ResNo: 0), N1: Lo, N2: Hi);
3444	}
3445
3446	SDValue DAGTypeLegalizer::SplitVecOp_Gather(MemSDNode *N, unsigned OpNo) {
3447	(void)OpNo;
3448	SDValue Lo, Hi;
3449	SplitVecRes_Gather(N, Lo, Hi);
3450
3451	SDValue Res = DAG.getNode(Opcode: ISD::CONCAT_VECTORS, DL: N, VT: N->getValueType(ResNo: 0), N1: Lo, N2: Hi);
3452	ReplaceValueWith(From: SDValue(N, 0), To: Res);
3453	return SDValue();
3454	}
3455
3456	SDValue DAGTypeLegalizer::SplitVecOp_VP_STORE(VPStoreSDNode *N, unsigned OpNo) {
3457	assert(N->isUnindexed() && "Indexed vp_store of vector?");
3458	SDValue Ch = N->getChain();
3459	SDValue Ptr = N->getBasePtr();
3460	SDValue Offset = N->getOffset();
3461	assert(Offset.isUndef() && "Unexpected VP store offset");
3462	SDValue Mask = N->getMask();
3463	SDValue EVL = N->getVectorLength();
3464	SDValue Data = N->getValue();
3465	Align Alignment = N->getOriginalAlign();
3466	SDLoc DL(N);
3467
3468	SDValue DataLo, DataHi;
3469	if (getTypeAction(VT: Data.getValueType()) == TargetLowering::TypeSplitVector)
3470	// Split Data operand
3471	GetSplitVector(Op: Data, Lo&: DataLo, Hi&: DataHi);
3472	else
3473	std::tie(args&: DataLo, args&: DataHi) = DAG.SplitVector(N: Data, DL);
3474
3475	// Split Mask operand
3476	SDValue MaskLo, MaskHi;
3477	if (OpNo == 1 && Mask.getOpcode() == ISD::SETCC) {
3478	SplitVecRes_SETCC(N: Mask.getNode(), Lo&: MaskLo, Hi&: MaskHi);
3479	} else {
3480	if (getTypeAction(VT: Mask.getValueType()) == TargetLowering::TypeSplitVector)
3481	GetSplitVector(Op: Mask, Lo&: MaskLo, Hi&: MaskHi);
3482	else
3483	std::tie(args&: MaskLo, args&: MaskHi) = DAG.SplitVector(N: Mask, DL);
3484	}
3485
3486	EVT MemoryVT = N->getMemoryVT();
3487	EVT LoMemVT, HiMemVT;
3488	bool HiIsEmpty = false;
3489	std::tie(args&: LoMemVT, args&: HiMemVT) =
3490	DAG.GetDependentSplitDestVTs(VT: MemoryVT, EnvVT: DataLo.getValueType(), HiIsEmpty: &HiIsEmpty);
3491
3492	// Split EVL
3493	SDValue EVLLo, EVLHi;
3494	std::tie(args&: EVLLo, args&: EVLHi) = DAG.SplitEVL(N: EVL, VecVT: Data.getValueType(), DL);
3495
3496	SDValue Lo, Hi;
3497	MachineMemOperand *MMO = DAG.getMachineFunction().getMachineMemOperand(
3498	PtrInfo: N->getPointerInfo(), F: MachineMemOperand::MOStore,
3499	Size: LocationSize::beforeOrAfterPointer(), BaseAlignment: Alignment, AAInfo: N->getAAInfo(),
3500	Ranges: N->getRanges());
3501
3502	Lo = DAG.getStoreVP(Chain: Ch, dl: DL, Val: DataLo, Ptr, Offset, Mask: MaskLo, EVL: EVLLo, MemVT: LoMemVT, MMO,
3503	AM: N->getAddressingMode(), IsTruncating: N->isTruncatingStore(),
3504	IsCompressing: N->isCompressingStore());
3505
3506	// If the hi vp_store has zero storage size, only the lo vp_store is needed.
3507	if (HiIsEmpty)
3508	return Lo;
3509
3510	Ptr = TLI.IncrementMemoryAddress(Addr: Ptr, Mask: MaskLo, DL, DataVT: LoMemVT, DAG,
3511	IsCompressedMemory: N->isCompressingStore());
3512
3513	MachinePointerInfo MPI;
3514	if (LoMemVT.isScalableVector()) {
3515	Alignment = commonAlignment(A: Alignment,
3516	Offset: LoMemVT.getSizeInBits().getKnownMinValue() / 8);
3517	MPI = MachinePointerInfo(N->getPointerInfo().getAddrSpace());
3518	} else
3519	MPI = N->getPointerInfo().getWithOffset(
3520	O: LoMemVT.getStoreSize().getFixedValue());
3521
3522	MMO = DAG.getMachineFunction().getMachineMemOperand(
3523	PtrInfo: MPI, F: MachineMemOperand::MOStore, Size: LocationSize::beforeOrAfterPointer(),
3524	BaseAlignment: Alignment, AAInfo: N->getAAInfo(), Ranges: N->getRanges());
3525
3526	Hi = DAG.getStoreVP(Chain: Ch, dl: DL, Val: DataHi, Ptr, Offset, Mask: MaskHi, EVL: EVLHi, MemVT: HiMemVT, MMO,
3527	AM: N->getAddressingMode(), IsTruncating: N->isTruncatingStore(),
3528	IsCompressing: N->isCompressingStore());
3529
3530	// Build a factor node to remember that this store is independent of the
3531	// other one.
3532	return DAG.getNode(ISD::TokenFactor, DL, MVT::Other, Lo, Hi);
3533	}
3534
3535	SDValue DAGTypeLegalizer::SplitVecOp_VP_STRIDED_STORE(VPStridedStoreSDNode *N,
3536	unsigned OpNo) {
3537	assert(N->isUnindexed() && "Indexed vp_strided_store of a vector?");
3538	assert(N->getOffset().isUndef() && "Unexpected VP strided store offset");
3539
3540	SDLoc DL(N);
3541
3542	SDValue Data = N->getValue();
3543	SDValue LoData, HiData;
3544	if (getTypeAction(VT: Data.getValueType()) == TargetLowering::TypeSplitVector)
3545	GetSplitVector(Op: Data, Lo&: LoData, Hi&: HiData);
3546	else
3547	std::tie(args&: LoData, args&: HiData) = DAG.SplitVector(N: Data, DL);
3548
3549	EVT LoMemVT, HiMemVT;
3550	bool HiIsEmpty = false;
3551	std::tie(args&: LoMemVT, args&: HiMemVT) = DAG.GetDependentSplitDestVTs(
3552	VT: N->getMemoryVT(), EnvVT: LoData.getValueType(), HiIsEmpty: &HiIsEmpty);
3553
3554	SDValue Mask = N->getMask();
3555	SDValue LoMask, HiMask;
3556	if (OpNo == 1 && Mask.getOpcode() == ISD::SETCC)
3557	SplitVecRes_SETCC(N: Mask.getNode(), Lo&: LoMask, Hi&: HiMask);
3558	else if (getTypeAction(VT: Mask.getValueType()) ==
3559	TargetLowering::TypeSplitVector)
3560	GetSplitVector(Op: Mask, Lo&: LoMask, Hi&: HiMask);
3561	else
3562	std::tie(args&: LoMask, args&: HiMask) = DAG.SplitVector(N: Mask, DL);
3563
3564	SDValue LoEVL, HiEVL;
3565	std::tie(args&: LoEVL, args&: HiEVL) =
3566	DAG.SplitEVL(N: N->getVectorLength(), VecVT: Data.getValueType(), DL);
3567
3568	// Generate the low vp_strided_store
3569	SDValue Lo = DAG.getStridedStoreVP(
3570	Chain: N->getChain(), DL, Val: LoData, Ptr: N->getBasePtr(), Offset: N->getOffset(),
3571	Stride: N->getStride(), Mask: LoMask, EVL: LoEVL, MemVT: LoMemVT, MMO: N->getMemOperand(),
3572	AM: N->getAddressingMode(), IsTruncating: N->isTruncatingStore(), IsCompressing: N->isCompressingStore());
3573
3574	// If the high vp_strided_store has zero storage size, only the low
3575	// vp_strided_store is needed.
3576	if (HiIsEmpty)
3577	return Lo;
3578
3579	// Generate the high vp_strided_store.
3580	// To calculate the high base address, we need to sum to the low base
3581	// address stride number of bytes for each element already stored by low,
3582	// that is: Ptr = Ptr + (LoEVL * Stride)
3583	EVT PtrVT = N->getBasePtr().getValueType();
3584	SDValue Increment =
3585	DAG.getNode(Opcode: ISD::MUL, DL, VT: PtrVT, N1: LoEVL,
3586	N2: DAG.getSExtOrTrunc(Op: N->getStride(), DL, VT: PtrVT));
3587	SDValue Ptr = DAG.getNode(Opcode: ISD::ADD, DL, VT: PtrVT, N1: N->getBasePtr(), N2: Increment);
3588
3589	Align Alignment = N->getOriginalAlign();
3590	if (LoMemVT.isScalableVector())
3591	Alignment = commonAlignment(A: Alignment,
3592	Offset: LoMemVT.getSizeInBits().getKnownMinValue() / 8);
3593
3594	MachineMemOperand *MMO = DAG.getMachineFunction().getMachineMemOperand(
3595	PtrInfo: MachinePointerInfo(N->getPointerInfo().getAddrSpace()),
3596	F: MachineMemOperand::MOStore, Size: LocationSize::beforeOrAfterPointer(),
3597	BaseAlignment: Alignment, AAInfo: N->getAAInfo(), Ranges: N->getRanges());
3598
3599	SDValue Hi = DAG.getStridedStoreVP(
3600	Chain: N->getChain(), DL, Val: HiData, Ptr, Offset: N->getOffset(), Stride: N->getStride(), Mask: HiMask,
3601	EVL: HiEVL, MemVT: HiMemVT, MMO, AM: N->getAddressingMode(), IsTruncating: N->isTruncatingStore(),
3602	IsCompressing: N->isCompressingStore());
3603
3604	// Build a factor node to remember that this store is independent of the
3605	// other one.
3606	return DAG.getNode(ISD::TokenFactor, DL, MVT::Other, Lo, Hi);
3607	}
3608
3609	SDValue DAGTypeLegalizer::SplitVecOp_MSTORE(MaskedStoreSDNode *N,
3610	unsigned OpNo) {
3611	assert(N->isUnindexed() && "Indexed masked store of vector?");
3612	SDValue Ch = N->getChain();
3613	SDValue Ptr = N->getBasePtr();
3614	SDValue Offset = N->getOffset();
3615	assert(Offset.isUndef() && "Unexpected indexed masked store offset");
3616	SDValue Mask = N->getMask();
3617	SDValue Data = N->getValue();
3618	Align Alignment = N->getOriginalAlign();
3619	SDLoc DL(N);
3620
3621	SDValue DataLo, DataHi;
3622	if (getTypeAction(VT: Data.getValueType()) == TargetLowering::TypeSplitVector)
3623	// Split Data operand
3624	GetSplitVector(Op: Data, Lo&: DataLo, Hi&: DataHi);
3625	else
3626	std::tie(args&: DataLo, args&: DataHi) = DAG.SplitVector(N: Data, DL);
3627
3628	// Split Mask operand
3629	SDValue MaskLo, MaskHi;
3630	if (OpNo == 1 && Mask.getOpcode() == ISD::SETCC) {
3631	SplitVecRes_SETCC(N: Mask.getNode(), Lo&: MaskLo, Hi&: MaskHi);
3632	} else {
3633	if (getTypeAction(VT: Mask.getValueType()) == TargetLowering::TypeSplitVector)
3634	GetSplitVector(Op: Mask, Lo&: MaskLo, Hi&: MaskHi);
3635	else
3636	std::tie(args&: MaskLo, args&: MaskHi) = DAG.SplitVector(N: Mask, DL);
3637	}
3638
3639	EVT MemoryVT = N->getMemoryVT();
3640	EVT LoMemVT, HiMemVT;
3641	bool HiIsEmpty = false;
3642	std::tie(args&: LoMemVT, args&: HiMemVT) =
3643	DAG.GetDependentSplitDestVTs(VT: MemoryVT, EnvVT: DataLo.getValueType(), HiIsEmpty: &HiIsEmpty);
3644
3645	SDValue Lo, Hi, Res;
3646	MachineMemOperand *MMO = DAG.getMachineFunction().getMachineMemOperand(
3647	PtrInfo: N->getPointerInfo(), F: MachineMemOperand::MOStore,
3648	Size: LocationSize::beforeOrAfterPointer(), BaseAlignment: Alignment, AAInfo: N->getAAInfo(),
3649	Ranges: N->getRanges());
3650
3651	Lo = DAG.getMaskedStore(Chain: Ch, dl: DL, Val: DataLo, Base: Ptr, Offset, Mask: MaskLo, MemVT: LoMemVT, MMO,
3652	AM: N->getAddressingMode(), IsTruncating: N->isTruncatingStore(),
3653	IsCompressing: N->isCompressingStore());
3654
3655	if (HiIsEmpty) {
3656	// The hi masked store has zero storage size.
3657	// Only the lo masked store is needed.
3658	Res = Lo;
3659	} else {
3660
3661	Ptr = TLI.IncrementMemoryAddress(Addr: Ptr, Mask: MaskLo, DL, DataVT: LoMemVT, DAG,
3662	IsCompressedMemory: N->isCompressingStore());
3663
3664	MachinePointerInfo MPI;
3665	if (LoMemVT.isScalableVector()) {
3666	Alignment = commonAlignment(
3667	A: Alignment, Offset: LoMemVT.getSizeInBits().getKnownMinValue() / 8);
3668	MPI = MachinePointerInfo(N->getPointerInfo().getAddrSpace());
3669	} else
3670	MPI = N->getPointerInfo().getWithOffset(
3671	O: LoMemVT.getStoreSize().getFixedValue());
3672
3673	MMO = DAG.getMachineFunction().getMachineMemOperand(
3674	PtrInfo: MPI, F: MachineMemOperand::MOStore, Size: LocationSize::beforeOrAfterPointer(),
3675	BaseAlignment: Alignment, AAInfo: N->getAAInfo(), Ranges: N->getRanges());
3676
3677	Hi = DAG.getMaskedStore(Chain: Ch, dl: DL, Val: DataHi, Base: Ptr, Offset, Mask: MaskHi, MemVT: HiMemVT, MMO,
3678	AM: N->getAddressingMode(), IsTruncating: N->isTruncatingStore(),
3679	IsCompressing: N->isCompressingStore());
3680
3681	// Build a factor node to remember that this store is independent of the
3682	// other one.
3683	Res = DAG.getNode(ISD::TokenFactor, DL, MVT::Other, Lo, Hi);
3684	}
3685
3686	return Res;
3687	}
3688
3689	SDValue DAGTypeLegalizer::SplitVecOp_Scatter(MemSDNode *N, unsigned OpNo) {
3690	SDValue Ch = N->getChain();
3691	SDValue Ptr = N->getBasePtr();
3692	EVT MemoryVT = N->getMemoryVT();
3693	Align Alignment = N->getOriginalAlign();
3694	SDLoc DL(N);
3695	struct Operands {
3696	SDValue Mask;
3697	SDValue Index;
3698	SDValue Scale;
3699	SDValue Data;
3700	} Ops = [&]() -> Operands {
3701	if (auto *MSC = dyn_cast<MaskedScatterSDNode>(Val: N)) {
3702	return {.Mask: MSC->getMask(), .Index: MSC->getIndex(), .Scale: MSC->getScale(),
3703	.Data: MSC->getValue()};
3704	}
3705	auto *VPSC = cast<VPScatterSDNode>(Val: N);
3706	return {.Mask: VPSC->getMask(), .Index: VPSC->getIndex(), .Scale: VPSC->getScale(),
3707	.Data: VPSC->getValue()};
3708	}();
3709	// Split all operands
3710
3711	EVT LoMemVT, HiMemVT;
3712	std::tie(args&: LoMemVT, args&: HiMemVT) = DAG.GetSplitDestVTs(VT: MemoryVT);
3713
3714	SDValue DataLo, DataHi;
3715	if (getTypeAction(VT: Ops.Data.getValueType()) == TargetLowering::TypeSplitVector)
3716	// Split Data operand
3717	GetSplitVector(Op: Ops.Data, Lo&: DataLo, Hi&: DataHi);
3718	else
3719	std::tie(args&: DataLo, args&: DataHi) = DAG.SplitVector(N: Ops.Data, DL);
3720
3721	// Split Mask operand
3722	SDValue MaskLo, MaskHi;
3723	if (OpNo == 1 && Ops.Mask.getOpcode() == ISD::SETCC) {
3724	SplitVecRes_SETCC(N: Ops.Mask.getNode(), Lo&: MaskLo, Hi&: MaskHi);
3725	} else {
3726	std::tie(args&: MaskLo, args&: MaskHi) = SplitMask(Mask: Ops.Mask, DL);
3727	}
3728
3729	SDValue IndexHi, IndexLo;
3730	if (getTypeAction(VT: Ops.Index.getValueType()) ==
3731	TargetLowering::TypeSplitVector)
3732	GetSplitVector(Op: Ops.Index, Lo&: IndexLo, Hi&: IndexHi);
3733	else
3734	std::tie(args&: IndexLo, args&: IndexHi) = DAG.SplitVector(N: Ops.Index, DL);
3735
3736	SDValue Lo;
3737	MachineMemOperand *MMO = DAG.getMachineFunction().getMachineMemOperand(
3738	PtrInfo: N->getPointerInfo(), F: MachineMemOperand::MOStore,
3739	Size: LocationSize::beforeOrAfterPointer(), BaseAlignment: Alignment, AAInfo: N->getAAInfo(),
3740	Ranges: N->getRanges());
3741
3742	if (auto *MSC = dyn_cast<MaskedScatterSDNode>(Val: N)) {
3743	SDValue OpsLo[] = {Ch, DataLo, MaskLo, Ptr, IndexLo, Ops.Scale};
3744	Lo =
3745	DAG.getMaskedScatter(VTs: DAG.getVTList(MVT::Other), MemVT: LoMemVT, dl: DL, Ops: OpsLo, MMO,
3746	IndexType: MSC->getIndexType(), IsTruncating: MSC->isTruncatingStore());
3747
3748	// The order of the Scatter operation after split is well defined. The "Hi"
3749	// part comes after the "Lo". So these two operations should be chained one
3750	// after another.
3751	SDValue OpsHi[] = {Lo, DataHi, MaskHi, Ptr, IndexHi, Ops.Scale};
3752	return DAG.getMaskedScatter(VTs: DAG.getVTList(MVT::Other), MemVT: HiMemVT, dl: DL, Ops: OpsHi,
3753	MMO, IndexType: MSC->getIndexType(),
3754	IsTruncating: MSC->isTruncatingStore());
3755	}
3756	auto *VPSC = cast<VPScatterSDNode>(Val: N);
3757	SDValue EVLLo, EVLHi;
3758	std::tie(args&: EVLLo, args&: EVLHi) =
3759	DAG.SplitEVL(N: VPSC->getVectorLength(), VecVT: Ops.Data.getValueType(), DL);
3760
3761	SDValue OpsLo[] = {Ch, DataLo, Ptr, IndexLo, Ops.Scale, MaskLo, EVLLo};
3762	Lo = DAG.getScatterVP(VTs: DAG.getVTList(MVT::Other), VT: LoMemVT, dl: DL, Ops: OpsLo, MMO,
3763	IndexType: VPSC->getIndexType());
3764
3765	// The order of the Scatter operation after split is well defined. The "Hi"
3766	// part comes after the "Lo". So these two operations should be chained one
3767	// after another.
3768	SDValue OpsHi[] = {Lo, DataHi, Ptr, IndexHi, Ops.Scale, MaskHi, EVLHi};
3769	return DAG.getScatterVP(VTs: DAG.getVTList(MVT::Other), VT: HiMemVT, dl: DL, Ops: OpsHi, MMO,
3770	IndexType: VPSC->getIndexType());
3771	}
3772
3773	SDValue DAGTypeLegalizer::SplitVecOp_STORE(StoreSDNode *N, unsigned OpNo) {
3774	assert(N->isUnindexed() && "Indexed store of vector?");
3775	assert(OpNo == 1 && "Can only split the stored value");
3776	SDLoc DL(N);
3777
3778	bool isTruncating = N->isTruncatingStore();
3779	SDValue Ch = N->getChain();
3780	SDValue Ptr = N->getBasePtr();
3781	EVT MemoryVT = N->getMemoryVT();
3782	Align Alignment = N->getOriginalAlign();
3783	MachineMemOperand::Flags MMOFlags = N->getMemOperand()->getFlags();
3784	AAMDNodes AAInfo = N->getAAInfo();
3785	SDValue Lo, Hi;
3786	GetSplitVector(Op: N->getOperand(Num: 1), Lo, Hi);
3787
3788	EVT LoMemVT, HiMemVT;
3789	std::tie(args&: LoMemVT, args&: HiMemVT) = DAG.GetSplitDestVTs(VT: MemoryVT);
3790
3791	// Scalarize if the split halves are not byte-sized.
3792	if (!LoMemVT.isByteSized() || !HiMemVT.isByteSized())
3793	return TLI.scalarizeVectorStore(ST: N, DAG);
3794
3795	if (isTruncating)
3796	Lo = DAG.getTruncStore(Chain: Ch, dl: DL, Val: Lo, Ptr, PtrInfo: N->getPointerInfo(), SVT: LoMemVT,
3797	Alignment, MMOFlags, AAInfo);
3798	else
3799	Lo = DAG.getStore(Chain: Ch, dl: DL, Val: Lo, Ptr, PtrInfo: N->getPointerInfo(), Alignment, MMOFlags,
3800	AAInfo);
3801
3802	MachinePointerInfo MPI;
3803	IncrementPointer(N, MemVT: LoMemVT, MPI, Ptr);
3804
3805	if (isTruncating)
3806	Hi = DAG.getTruncStore(Chain: Ch, dl: DL, Val: Hi, Ptr, PtrInfo: MPI,
3807	SVT: HiMemVT, Alignment, MMOFlags, AAInfo);
3808	else
3809	Hi = DAG.getStore(Chain: Ch, dl: DL, Val: Hi, Ptr, PtrInfo: MPI, Alignment, MMOFlags, AAInfo);
3810
3811	return DAG.getNode(ISD::TokenFactor, DL, MVT::Other, Lo, Hi);
3812	}
3813
3814	SDValue DAGTypeLegalizer::SplitVecOp_CONCAT_VECTORS(SDNode *N) {
3815	SDLoc DL(N);
3816
3817	// The input operands all must have the same type, and we know the result
3818	// type is valid. Convert this to a buildvector which extracts all the
3819	// input elements.
3820	// TODO: If the input elements are power-two vectors, we could convert this to
3821	// a new CONCAT_VECTORS node with elements that are half-wide.
3822	SmallVector<SDValue, 32> Elts;
3823	EVT EltVT = N->getValueType(ResNo: 0).getVectorElementType();
3824	for (const SDValue &Op : N->op_values()) {
3825	for (unsigned i = 0, e = Op.getValueType().getVectorNumElements();
3826	i != e; ++i) {
3827	Elts.push_back(Elt: DAG.getNode(Opcode: ISD::EXTRACT_VECTOR_ELT, DL, VT: EltVT, N1: Op,
3828	N2: DAG.getVectorIdxConstant(Val: i, DL)));
3829	}
3830	}
3831
3832	return DAG.getBuildVector(VT: N->getValueType(ResNo: 0), DL, Ops: Elts);
3833	}
3834
3835	SDValue DAGTypeLegalizer::SplitVecOp_TruncateHelper(SDNode *N) {
3836	// The result type is legal, but the input type is illegal. If splitting
3837	// ends up with the result type of each half still being legal, just
3838	// do that. If, however, that would result in an illegal result type,
3839	// we can try to get more clever with power-two vectors. Specifically,
3840	// split the input type, but also widen the result element size, then
3841	// concatenate the halves and truncate again. For example, consider a target
3842	// where v8i8 is legal and v8i32 is not (ARM, which doesn't have 256-bit
3843	// vectors). To perform a "%res = v8i8 trunc v8i32 %in" we do:
3844	// %inlo = v4i32 extract_subvector %in, 0
3845	// %inhi = v4i32 extract_subvector %in, 4
3846	// %lo16 = v4i16 trunc v4i32 %inlo
3847	// %hi16 = v4i16 trunc v4i32 %inhi
3848	// %in16 = v8i16 concat_vectors v4i16 %lo16, v4i16 %hi16
3849	// %res = v8i8 trunc v8i16 %in16
3850	//
3851	// Without this transform, the original truncate would end up being
3852	// scalarized, which is pretty much always a last resort.
3853	unsigned OpNo = N->isStrictFPOpcode() ? 1 : 0;
3854	SDValue InVec = N->getOperand(Num: OpNo);
3855	EVT InVT = InVec->getValueType(ResNo: 0);
3856	EVT OutVT = N->getValueType(ResNo: 0);
3857	ElementCount NumElements = OutVT.getVectorElementCount();
3858	bool IsFloat = OutVT.isFloatingPoint();
3859
3860	unsigned InElementSize = InVT.getScalarSizeInBits();
3861	unsigned OutElementSize = OutVT.getScalarSizeInBits();
3862
3863	// Determine the split output VT. If its legal we can just split dirctly.
3864	EVT LoOutVT, HiOutVT;
3865	std::tie(args&: LoOutVT, args&: HiOutVT) = DAG.GetSplitDestVTs(VT: OutVT);
3866	assert(LoOutVT == HiOutVT && "Unequal split?");
3867
3868	// If the input elements are only 1/2 the width of the result elements,
3869	// just use the normal splitting. Our trick only work if there's room
3870	// to split more than once.
3871	if (isTypeLegal(VT: LoOutVT) ||
3872	InElementSize <= OutElementSize * 2)
3873	return SplitVecOp_UnaryOp(N);
3874	SDLoc DL(N);
3875
3876	// Don't touch if this will be scalarized.
3877	EVT FinalVT = InVT;
3878	while (getTypeAction(VT: FinalVT) == TargetLowering::TypeSplitVector)
3879	FinalVT = FinalVT.getHalfNumVectorElementsVT(Context&: *DAG.getContext());
3880
3881	if (getTypeAction(VT: FinalVT) == TargetLowering::TypeScalarizeVector)
3882	return SplitVecOp_UnaryOp(N);
3883
3884	// Get the split input vector.
3885	SDValue InLoVec, InHiVec;
3886	GetSplitVector(Op: InVec, Lo&: InLoVec, Hi&: InHiVec);
3887
3888	// Truncate them to 1/2 the element size.
3889	//
3890	// This assumes the number of elements is a power of two; any vector that
3891	// isn't should be widened, not split.
3892	EVT HalfElementVT = IsFloat ?
3893	EVT::getFloatingPointVT(BitWidth: InElementSize/2) :
3894	EVT::getIntegerVT(Context&: *DAG.getContext(), BitWidth: InElementSize/2);
3895	EVT HalfVT = EVT::getVectorVT(Context&: *DAG.getContext(), VT: HalfElementVT,
3896	EC: NumElements.divideCoefficientBy(RHS: 2));
3897
3898	SDValue HalfLo;
3899	SDValue HalfHi;
3900	SDValue Chain;
3901	if (N->isStrictFPOpcode()) {
3902	HalfLo = DAG.getNode(N->getOpcode(), DL, {HalfVT, MVT::Other},
3903	{N->getOperand(0), InLoVec});
3904	HalfHi = DAG.getNode(N->getOpcode(), DL, {HalfVT, MVT::Other},
3905	{N->getOperand(0), InHiVec});
3906	// Legalize the chain result - switch anything that used the old chain to
3907	// use the new one.
3908	Chain = DAG.getNode(ISD::TokenFactor, DL, MVT::Other, HalfLo.getValue(R: 1),
3909	HalfHi.getValue(R: 1));
3910	} else {
3911	HalfLo = DAG.getNode(Opcode: N->getOpcode(), DL, VT: HalfVT, Operand: InLoVec);
3912	HalfHi = DAG.getNode(Opcode: N->getOpcode(), DL, VT: HalfVT, Operand: InHiVec);
3913	}
3914
3915	// Concatenate them to get the full intermediate truncation result.
3916	EVT InterVT = EVT::getVectorVT(Context&: *DAG.getContext(), VT: HalfElementVT, EC: NumElements);
3917	SDValue InterVec = DAG.getNode(Opcode: ISD::CONCAT_VECTORS, DL, VT: InterVT, N1: HalfLo,
3918	N2: HalfHi);
3919	// Now finish up by truncating all the way down to the original result
3920	// type. This should normally be something that ends up being legal directly,
3921	// but in theory if a target has very wide vectors and an annoyingly
3922	// restricted set of legal types, this split can chain to build things up.
3923
3924	if (N->isStrictFPOpcode()) {
3925	SDValue Res = DAG.getNode(
3926	ISD::STRICT_FP_ROUND, DL, {OutVT, MVT::Other},
3927	{Chain, InterVec,
3928	DAG.getTargetConstant(0, DL, TLI.getPointerTy(DAG.getDataLayout()))});
3929	// Relink the chain
3930	ReplaceValueWith(From: SDValue(N, 1), To: SDValue(Res.getNode(), 1));
3931	return Res;
3932	}
3933
3934	return IsFloat
3935	? DAG.getNode(Opcode: ISD::FP_ROUND, DL, VT: OutVT, N1: InterVec,
3936	N2: DAG.getTargetConstant(
3937	Val: 0, DL, VT: TLI.getPointerTy(DL: DAG.getDataLayout())))
3938	: DAG.getNode(Opcode: ISD::TRUNCATE, DL, VT: OutVT, Operand: InterVec);
3939	}
3940
3941	SDValue DAGTypeLegalizer::SplitVecOp_VSETCC(SDNode *N) {
3942	assert(N->getValueType(0).isVector() &&
3943	N->getOperand(0).getValueType().isVector() &&
3944	"Operand types must be vectors");
3945	// The result has a legal vector type, but the input needs splitting.
3946	SDValue Lo0, Hi0, Lo1, Hi1, LoRes, HiRes;
3947	SDLoc DL(N);
3948	GetSplitVector(Op: N->getOperand(Num: 0), Lo&: Lo0, Hi&: Hi0);
3949	GetSplitVector(Op: N->getOperand(Num: 1), Lo&: Lo1, Hi&: Hi1);
3950	auto PartEltCnt = Lo0.getValueType().getVectorElementCount();
3951
3952	LLVMContext &Context = *DAG.getContext();
3953	EVT PartResVT = EVT::getVectorVT(Context, MVT::i1, PartEltCnt);
3954	EVT WideResVT = EVT::getVectorVT(Context, MVT::i1, PartEltCnt*2);
3955
3956	if (N->getOpcode() == ISD::SETCC) {
3957	LoRes = DAG.getNode(Opcode: ISD::SETCC, DL, VT: PartResVT, N1: Lo0, N2: Lo1, N3: N->getOperand(Num: 2));
3958	HiRes = DAG.getNode(Opcode: ISD::SETCC, DL, VT: PartResVT, N1: Hi0, N2: Hi1, N3: N->getOperand(Num: 2));
3959	} else {
3960	assert(N->getOpcode() == ISD::VP_SETCC && "Expected VP_SETCC opcode");
3961	SDValue MaskLo, MaskHi, EVLLo, EVLHi;
3962	std::tie(args&: MaskLo, args&: MaskHi) = SplitMask(Mask: N->getOperand(Num: 3));
3963	std::tie(args&: EVLLo, args&: EVLHi) =
3964	DAG.SplitEVL(N: N->getOperand(Num: 4), VecVT: N->getValueType(ResNo: 0), DL);
3965	LoRes = DAG.getNode(Opcode: ISD::VP_SETCC, DL, VT: PartResVT, N1: Lo0, N2: Lo1,
3966	N3: N->getOperand(Num: 2), N4: MaskLo, N5: EVLLo);
3967	HiRes = DAG.getNode(Opcode: ISD::VP_SETCC, DL, VT: PartResVT, N1: Hi0, N2: Hi1,
3968	N3: N->getOperand(Num: 2), N4: MaskHi, N5: EVLHi);
3969	}
3970	SDValue Con = DAG.getNode(Opcode: ISD::CONCAT_VECTORS, DL, VT: WideResVT, N1: LoRes, N2: HiRes);
3971
3972	EVT OpVT = N->getOperand(Num: 0).getValueType();
3973	ISD::NodeType ExtendCode =
3974	TargetLowering::getExtendForContent(Content: TLI.getBooleanContents(Type: OpVT));
3975	return DAG.getNode(Opcode: ExtendCode, DL, VT: N->getValueType(ResNo: 0), Operand: Con);
3976	}
3977
3978
3979	SDValue DAGTypeLegalizer::SplitVecOp_FP_ROUND(SDNode *N) {
3980	// The result has a legal vector type, but the input needs splitting.
3981	EVT ResVT = N->getValueType(ResNo: 0);
3982	SDValue Lo, Hi;
3983	SDLoc DL(N);
3984	GetSplitVector(Op: N->getOperand(Num: N->isStrictFPOpcode() ? 1 : 0), Lo, Hi);
3985	EVT InVT = Lo.getValueType();
3986
3987	EVT OutVT = EVT::getVectorVT(Context&: *DAG.getContext(), VT: ResVT.getVectorElementType(),
3988	EC: InVT.getVectorElementCount());
3989
3990	if (N->isStrictFPOpcode()) {
3991	Lo = DAG.getNode(N->getOpcode(), DL, { OutVT, MVT::Other },
3992	{ N->getOperand(0), Lo, N->getOperand(2) });
3993	Hi = DAG.getNode(N->getOpcode(), DL, { OutVT, MVT::Other },
3994	{ N->getOperand(0), Hi, N->getOperand(2) });
3995	// Legalize the chain result - switch anything that used the old chain to
3996	// use the new one.
3997	SDValue NewChain = DAG.getNode(ISD::TokenFactor, DL, MVT::Other,
3998	Lo.getValue(1), Hi.getValue(1));
3999	ReplaceValueWith(From: SDValue(N, 1), To: NewChain);
4000	} else if (N->getOpcode() == ISD::VP_FP_ROUND) {
4001	SDValue MaskLo, MaskHi, EVLLo, EVLHi;
4002	std::tie(args&: MaskLo, args&: MaskHi) = SplitMask(Mask: N->getOperand(Num: 1));
4003	std::tie(args&: EVLLo, args&: EVLHi) =
4004	DAG.SplitEVL(N: N->getOperand(Num: 2), VecVT: N->getValueType(ResNo: 0), DL);
4005	Lo = DAG.getNode(Opcode: ISD::VP_FP_ROUND, DL, VT: OutVT, N1: Lo, N2: MaskLo, N3: EVLLo);
4006	Hi = DAG.getNode(Opcode: ISD::VP_FP_ROUND, DL, VT: OutVT, N1: Hi, N2: MaskHi, N3: EVLHi);
4007	} else {
4008	Lo = DAG.getNode(Opcode: ISD::FP_ROUND, DL, VT: OutVT, N1: Lo, N2: N->getOperand(Num: 1));
4009	Hi = DAG.getNode(Opcode: ISD::FP_ROUND, DL, VT: OutVT, N1: Hi, N2: N->getOperand(Num: 1));
4010	}
4011
4012	return DAG.getNode(Opcode: ISD::CONCAT_VECTORS, DL, VT: ResVT, N1: Lo, N2: Hi);
4013	}
4014
4015	// Split a vector type in an FP binary operation where the second operand has a
4016	// different type from the first.
4017	//
4018	// The result (and the first input) has a legal vector type, but the second
4019	// input needs splitting.
4020	SDValue DAGTypeLegalizer::SplitVecOp_FPOpDifferentTypes(SDNode *N) {
4021	SDLoc DL(N);
4022
4023	EVT LHSLoVT, LHSHiVT;
4024	std::tie(args&: LHSLoVT, args&: LHSHiVT) = DAG.GetSplitDestVTs(VT: N->getValueType(ResNo: 0));
4025
4026	if (!isTypeLegal(VT: LHSLoVT) || !isTypeLegal(VT: LHSHiVT))
4027	return DAG.UnrollVectorOp(N, ResNE: N->getValueType(ResNo: 0).getVectorNumElements());
4028
4029	SDValue LHSLo, LHSHi;
4030	std::tie(args&: LHSLo, args&: LHSHi) =
4031	DAG.SplitVector(N: N->getOperand(Num: 0), DL, LoVT: LHSLoVT, HiVT: LHSHiVT);
4032
4033	SDValue RHSLo, RHSHi;
4034	std::tie(args&: RHSLo, args&: RHSHi) = DAG.SplitVector(N: N->getOperand(Num: 1), DL);
4035
4036	SDValue Lo = DAG.getNode(Opcode: N->getOpcode(), DL, VT: LHSLoVT, N1: LHSLo, N2: RHSLo);
4037	SDValue Hi = DAG.getNode(Opcode: N->getOpcode(), DL, VT: LHSHiVT, N1: LHSHi, N2: RHSHi);
4038
4039	return DAG.getNode(Opcode: ISD::CONCAT_VECTORS, DL, VT: N->getValueType(ResNo: 0), N1: Lo, N2: Hi);
4040	}
4041
4042	SDValue DAGTypeLegalizer::SplitVecOp_FP_TO_XINT_SAT(SDNode *N) {
4043	EVT ResVT = N->getValueType(ResNo: 0);
4044	SDValue Lo, Hi;
4045	SDLoc dl(N);
4046	GetSplitVector(Op: N->getOperand(Num: 0), Lo, Hi);
4047	EVT InVT = Lo.getValueType();
4048
4049	EVT NewResVT =
4050	EVT::getVectorVT(Context&: *DAG.getContext(), VT: ResVT.getVectorElementType(),
4051	EC: InVT.getVectorElementCount());
4052
4053	Lo = DAG.getNode(Opcode: N->getOpcode(), DL: dl, VT: NewResVT, N1: Lo, N2: N->getOperand(Num: 1));
4054	Hi = DAG.getNode(Opcode: N->getOpcode(), DL: dl, VT: NewResVT, N1: Hi, N2: N->getOperand(Num: 1));
4055
4056	return DAG.getNode(Opcode: ISD::CONCAT_VECTORS, DL: dl, VT: ResVT, N1: Lo, N2: Hi);
4057	}
4058
4059	//===----------------------------------------------------------------------===//
4060	// Result Vector Widening
4061	//===----------------------------------------------------------------------===//
4062
4063	void DAGTypeLegalizer::WidenVectorResult(SDNode *N, unsigned ResNo) {
4064	LLVM_DEBUG(dbgs() << "Widen node result " << ResNo << ": "; N->dump(&DAG));
4065
4066	// See if the target wants to custom widen this node.
4067	if (CustomWidenLowerNode(N, VT: N->getValueType(ResNo)))
4068	return;
4069
4070	SDValue Res = SDValue();
4071
4072	auto unrollExpandedOp = [&]() {
4073	// We're going to widen this vector op to a legal type by padding with undef
4074	// elements. If the wide vector op is eventually going to be expanded to
4075	// scalar libcalls, then unroll into scalar ops now to avoid unnecessary
4076	// libcalls on the undef elements.
4077	EVT VT = N->getValueType(ResNo: 0);
4078	EVT WideVecVT = TLI.getTypeToTransformTo(Context&: *DAG.getContext(), VT);
4079	if (!TLI.isOperationLegalOrCustom(Op: N->getOpcode(), VT: WideVecVT) &&
4080	TLI.isOperationExpand(Op: N->getOpcode(), VT: VT.getScalarType())) {
4081	Res = DAG.UnrollVectorOp(N, ResNE: WideVecVT.getVectorNumElements());
4082	return true;
4083	}
4084	return false;
4085	};
4086
4087	switch (N->getOpcode()) {
4088	default:
4089	#ifndef NDEBUG
4090	dbgs() << "WidenVectorResult #" << ResNo << ": ";
4091	N->dump(G: &DAG);
4092	dbgs() << "\n";
4093	#endif
4094	report_fatal_error(reason: "Do not know how to widen the result of this operator!");
4095
4096	case ISD::MERGE_VALUES: Res = WidenVecRes_MERGE_VALUES(N, ResNo); break;
4097	case ISD::AssertZext: Res = WidenVecRes_AssertZext(N); break;
4098	case ISD::BITCAST: Res = WidenVecRes_BITCAST(N); break;
4099	case ISD::BUILD_VECTOR: Res = WidenVecRes_BUILD_VECTOR(N); break;
4100	case ISD::CONCAT_VECTORS: Res = WidenVecRes_CONCAT_VECTORS(N); break;
4101	case ISD::INSERT_SUBVECTOR:
4102	Res = WidenVecRes_INSERT_SUBVECTOR(N);
4103	break;
4104	case ISD::EXTRACT_SUBVECTOR: Res = WidenVecRes_EXTRACT_SUBVECTOR(N); break;
4105	case ISD::INSERT_VECTOR_ELT: Res = WidenVecRes_INSERT_VECTOR_ELT(N); break;
4106	case ISD::LOAD: Res = WidenVecRes_LOAD(N); break;
4107	case ISD::STEP_VECTOR:
4108	case ISD::SPLAT_VECTOR:
4109	case ISD::SCALAR_TO_VECTOR:
4110	Res = WidenVecRes_ScalarOp(N);
4111	break;
4112	case ISD::SIGN_EXTEND_INREG: Res = WidenVecRes_InregOp(N); break;
4113	case ISD::VSELECT:
4114	case ISD::SELECT:
4115	case ISD::VP_SELECT:
4116	case ISD::VP_MERGE:
4117	Res = WidenVecRes_Select(N);
4118	break;
4119	case ISD::SELECT_CC: Res = WidenVecRes_SELECT_CC(N); break;
4120	case ISD::VP_SETCC:
4121	case ISD::SETCC: Res = WidenVecRes_SETCC(N); break;
4122	case ISD::UNDEF: Res = WidenVecRes_UNDEF(N); break;
4123	case ISD::VECTOR_SHUFFLE:
4124	Res = WidenVecRes_VECTOR_SHUFFLE(N: cast<ShuffleVectorSDNode>(Val: N));
4125	break;
4126	case ISD::VP_LOAD:
4127	Res = WidenVecRes_VP_LOAD(N: cast<VPLoadSDNode>(Val: N));
4128	break;
4129	case ISD::EXPERIMENTAL_VP_STRIDED_LOAD:
4130	Res = WidenVecRes_VP_STRIDED_LOAD(N: cast<VPStridedLoadSDNode>(Val: N));
4131	break;
4132	case ISD::MLOAD:
4133	Res = WidenVecRes_MLOAD(N: cast<MaskedLoadSDNode>(Val: N));
4134	break;
4135	case ISD::MGATHER:
4136	Res = WidenVecRes_MGATHER(N: cast<MaskedGatherSDNode>(Val: N));
4137	break;
4138	case ISD::VP_GATHER:
4139	Res = WidenVecRes_VP_GATHER(N: cast<VPGatherSDNode>(Val: N));
4140	break;
4141	case ISD::VECTOR_REVERSE:
4142	Res = WidenVecRes_VECTOR_REVERSE(N);
4143	break;
4144
4145	case ISD::ADD: case ISD::VP_ADD:
4146	case ISD::AND: case ISD::VP_AND:
4147	case ISD::MUL: case ISD::VP_MUL:
4148	case ISD::MULHS:
4149	case ISD::MULHU:
4150	case ISD::OR: case ISD::VP_OR:
4151	case ISD::SUB: case ISD::VP_SUB:
4152	case ISD::XOR: case ISD::VP_XOR:
4153	case ISD::SHL: case ISD::VP_SHL:
4154	case ISD::SRA: case ISD::VP_ASHR:
4155	case ISD::SRL: case ISD::VP_LSHR:
4156	case ISD::FMINNUM: case ISD::VP_FMINNUM:
4157	case ISD::FMAXNUM: case ISD::VP_FMAXNUM:
4158	case ISD::FMINIMUM:
4159	case ISD::VP_FMINIMUM:
4160	case ISD::FMAXIMUM:
4161	case ISD::VP_FMAXIMUM:
4162	case ISD::SMIN: case ISD::VP_SMIN:
4163	case ISD::SMAX: case ISD::VP_SMAX:
4164	case ISD::UMIN: case ISD::VP_UMIN:
4165	case ISD::UMAX: case ISD::VP_UMAX:
4166	case ISD::UADDSAT: case ISD::VP_UADDSAT:
4167	case ISD::SADDSAT: case ISD::VP_SADDSAT:
4168	case ISD::USUBSAT: case ISD::VP_USUBSAT:
4169	case ISD::SSUBSAT: case ISD::VP_SSUBSAT:
4170	case ISD::SSHLSAT:
4171	case ISD::USHLSAT:
4172	case ISD::ROTL:
4173	case ISD::ROTR:
4174	case ISD::AVGFLOORS:
4175	case ISD::AVGFLOORU:
4176	case ISD::AVGCEILS:
4177	case ISD::AVGCEILU:
4178	// Vector-predicated binary op widening. Note that -- unlike the
4179	// unpredicated versions -- we don't have to worry about trapping on
4180	// operations like UDIV, FADD, etc., as we pass on the original vector
4181	// length parameter. This means the widened elements containing garbage
4182	// aren't active.
4183	case ISD::VP_SDIV:
4184	case ISD::VP_UDIV:
4185	case ISD::VP_SREM:
4186	case ISD::VP_UREM:
4187	case ISD::VP_FADD:
4188	case ISD::VP_FSUB:
4189	case ISD::VP_FMUL:
4190	case ISD::VP_FDIV:
4191	case ISD::VP_FREM:
4192	case ISD::VP_FCOPYSIGN:
4193	Res = WidenVecRes_Binary(N);
4194	break;
4195
4196	case ISD::FPOW:
4197	case ISD::FREM:
4198	if (unrollExpandedOp())
4199	break;
4200	// If the target has custom/legal support for the scalar FP intrinsic ops
4201	// (they are probably not destined to become libcalls), then widen those
4202	// like any other binary ops.
4203	[[fallthrough]];
4204
4205	case ISD::FADD:
4206	case ISD::FMUL:
4207	case ISD::FSUB:
4208	case ISD::FDIV:
4209	case ISD::SDIV:
4210	case ISD::UDIV:
4211	case ISD::SREM:
4212	case ISD::UREM:
4213	Res = WidenVecRes_BinaryCanTrap(N);
4214	break;
4215
4216	case ISD::SMULFIX:
4217	case ISD::SMULFIXSAT:
4218	case ISD::UMULFIX:
4219	case ISD::UMULFIXSAT:
4220	// These are binary operations, but with an extra operand that shouldn't
4221	// be widened (the scale).
4222	Res = WidenVecRes_BinaryWithExtraScalarOp(N);
4223	break;
4224
4225	#define DAG_INSTRUCTION(NAME, NARG, ROUND_MODE, INTRINSIC, DAGN) \
4226	case ISD::STRICT_##DAGN:
4227	#include "llvm/IR/ConstrainedOps.def"
4228	Res = WidenVecRes_StrictFP(N);
4229	break;
4230
4231	case ISD::UADDO:
4232	case ISD::SADDO:
4233	case ISD::USUBO:
4234	case ISD::SSUBO:
4235	case ISD::UMULO:
4236	case ISD::SMULO:
4237	Res = WidenVecRes_OverflowOp(N, ResNo);
4238	break;
4239
4240	case ISD::FCOPYSIGN:
4241	Res = WidenVecRes_FCOPYSIGN(N);
4242	break;
4243
4244	case ISD::IS_FPCLASS:
4245	case ISD::FPTRUNC_ROUND:
4246	Res = WidenVecRes_UnarySameEltsWithScalarArg(N);
4247	break;
4248
4249	case ISD::FLDEXP:
4250	case ISD::FPOWI:
4251	if (!unrollExpandedOp())
4252	Res = WidenVecRes_ExpOp(N);
4253	break;
4254
4255	case ISD::ANY_EXTEND_VECTOR_INREG:
4256	case ISD::SIGN_EXTEND_VECTOR_INREG:
4257	case ISD::ZERO_EXTEND_VECTOR_INREG:
4258	Res = WidenVecRes_EXTEND_VECTOR_INREG(N);
4259	break;
4260
4261	case ISD::ANY_EXTEND:
4262	case ISD::FP_EXTEND:
4263	case ISD::VP_FP_EXTEND:
4264	case ISD::FP_ROUND:
4265	case ISD::VP_FP_ROUND:
4266	case ISD::FP_TO_SINT:
4267	case ISD::VP_FP_TO_SINT:
4268	case ISD::FP_TO_UINT:
4269	case ISD::VP_FP_TO_UINT:
4270	case ISD::SIGN_EXTEND:
4271	case ISD::VP_SIGN_EXTEND:
4272	case ISD::SINT_TO_FP:
4273	case ISD::VP_SINT_TO_FP:
4274	case ISD::VP_TRUNCATE:
4275	case ISD::TRUNCATE:
4276	case ISD::UINT_TO_FP:
4277	case ISD::VP_UINT_TO_FP:
4278	case ISD::ZERO_EXTEND:
4279	case ISD::VP_ZERO_EXTEND:
4280	Res = WidenVecRes_Convert(N);
4281	break;
4282
4283	case ISD::FP_TO_SINT_SAT:
4284	case ISD::FP_TO_UINT_SAT:
4285	Res = WidenVecRes_FP_TO_XINT_SAT(N);
4286	break;
4287
4288	case ISD::LRINT:
4289	case ISD::LLRINT:
4290	case ISD::VP_LRINT:
4291	case ISD::VP_LLRINT:
4292	Res = WidenVecRes_XRINT(N);
4293	break;
4294
4295	case ISD::FABS:
4296	case ISD::FCEIL:
4297	case ISD::FCOS:
4298	case ISD::FEXP:
4299	case ISD::FEXP2:
4300	case ISD::FEXP10:
4301	case ISD::FFLOOR:
4302	case ISD::FLOG:
4303	case ISD::FLOG10:
4304	case ISD::FLOG2:
4305	case ISD::FNEARBYINT:
4306	case ISD::FRINT:
4307	case ISD::FROUND:
4308	case ISD::FROUNDEVEN:
4309	case ISD::FSIN:
4310	case ISD::FSQRT:
4311	case ISD::FTRUNC:
4312	if (unrollExpandedOp())
4313	break;
4314	// If the target has custom/legal support for the scalar FP intrinsic ops
4315	// (they are probably not destined to become libcalls), then widen those
4316	// like any other unary ops.
4317	[[fallthrough]];
4318
4319	case ISD::ABS:
4320	case ISD::VP_ABS:
4321	case ISD::BITREVERSE:
4322	case ISD::VP_BITREVERSE:
4323	case ISD::BSWAP:
4324	case ISD::VP_BSWAP:
4325	case ISD::CTLZ:
4326	case ISD::VP_CTLZ:
4327	case ISD::CTLZ_ZERO_UNDEF:
4328	case ISD::VP_CTLZ_ZERO_UNDEF:
4329	case ISD::CTPOP:
4330	case ISD::VP_CTPOP:
4331	case ISD::CTTZ:
4332	case ISD::VP_CTTZ:
4333	case ISD::CTTZ_ZERO_UNDEF:
4334	case ISD::VP_CTTZ_ZERO_UNDEF:
4335	case ISD::FNEG: case ISD::VP_FNEG:
4336	case ISD::VP_FABS:
4337	case ISD::VP_SQRT:
4338	case ISD::VP_FCEIL:
4339	case ISD::VP_FFLOOR:
4340	case ISD::VP_FRINT:
4341	case ISD::VP_FNEARBYINT:
4342	case ISD::VP_FROUND:
4343	case ISD::VP_FROUNDEVEN:
4344	case ISD::VP_FROUNDTOZERO:
4345	case ISD::FREEZE:
4346	case ISD::ARITH_FENCE:
4347	case ISD::FCANONICALIZE:
4348	Res = WidenVecRes_Unary(N);
4349	break;
4350	case ISD::FMA: case ISD::VP_FMA:
4351	case ISD::FSHL:
4352	case ISD::VP_FSHL:
4353	case ISD::FSHR:
4354	case ISD::VP_FSHR:
4355	Res = WidenVecRes_Ternary(N);
4356	break;
4357	}
4358
4359	// If Res is null, the sub-method took care of registering the result.
4360	if (Res.getNode())
4361	SetWidenedVector(Op: SDValue(N, ResNo), Result: Res);
4362	}
4363
4364	SDValue DAGTypeLegalizer::WidenVecRes_Ternary(SDNode *N) {
4365	// Ternary op widening.
4366	SDLoc dl(N);
4367	EVT WidenVT = TLI.getTypeToTransformTo(Context&: *DAG.getContext(), VT: N->getValueType(ResNo: 0));
4368	SDValue InOp1 = GetWidenedVector(Op: N->getOperand(Num: 0));
4369	SDValue InOp2 = GetWidenedVector(Op: N->getOperand(Num: 1));
4370	SDValue InOp3 = GetWidenedVector(Op: N->getOperand(Num: 2));
4371	if (N->getNumOperands() == 3)
4372	return DAG.getNode(Opcode: N->getOpcode(), DL: dl, VT: WidenVT, N1: InOp1, N2: InOp2, N3: InOp3);
4373
4374	assert(N->getNumOperands() == 5 && "Unexpected number of operands!");
4375	assert(N->isVPOpcode() && "Expected VP opcode");
4376
4377	SDValue Mask =
4378	GetWidenedMask(Mask: N->getOperand(Num: 3), EC: WidenVT.getVectorElementCount());
4379	return DAG.getNode(Opcode: N->getOpcode(), DL: dl, VT: WidenVT,
4380	Ops: {InOp1, InOp2, InOp3, Mask, N->getOperand(Num: 4)});
4381	}
4382
4383	SDValue DAGTypeLegalizer::WidenVecRes_Binary(SDNode *N) {
4384	// Binary op widening.
4385	SDLoc dl(N);
4386	EVT WidenVT = TLI.getTypeToTransformTo(Context&: *DAG.getContext(), VT: N->getValueType(ResNo: 0));
4387	SDValue InOp1 = GetWidenedVector(Op: N->getOperand(Num: 0));
4388	SDValue InOp2 = GetWidenedVector(Op: N->getOperand(Num: 1));
4389	if (N->getNumOperands() == 2)
4390	return DAG.getNode(Opcode: N->getOpcode(), DL: dl, VT: WidenVT, N1: InOp1, N2: InOp2,
4391	Flags: N->getFlags());
4392
4393	assert(N->getNumOperands() == 4 && "Unexpected number of operands!");
4394	assert(N->isVPOpcode() && "Expected VP opcode");
4395
4396	SDValue Mask =
4397	GetWidenedMask(Mask: N->getOperand(Num: 2), EC: WidenVT.getVectorElementCount());
4398	return DAG.getNode(Opcode: N->getOpcode(), DL: dl, VT: WidenVT,
4399	Ops: {InOp1, InOp2, Mask, N->getOperand(Num: 3)}, Flags: N->getFlags());
4400	}
4401
4402	SDValue DAGTypeLegalizer::WidenVecRes_BinaryWithExtraScalarOp(SDNode *N) {
4403	// Binary op widening, but with an extra operand that shouldn't be widened.
4404	SDLoc dl(N);
4405	EVT WidenVT = TLI.getTypeToTransformTo(Context&: *DAG.getContext(), VT: N->getValueType(ResNo: 0));
4406	SDValue InOp1 = GetWidenedVector(Op: N->getOperand(Num: 0));
4407	SDValue InOp2 = GetWidenedVector(Op: N->getOperand(Num: 1));
4408	SDValue InOp3 = N->getOperand(Num: 2);
4409	return DAG.getNode(Opcode: N->getOpcode(), DL: dl, VT: WidenVT, N1: InOp1, N2: InOp2, N3: InOp3,
4410	Flags: N->getFlags());
4411	}
4412
4413	// Given a vector of operations that have been broken up to widen, see
4414	// if we can collect them together into the next widest legal VT. This
4415	// implementation is trap-safe.
4416	static SDValue CollectOpsToWiden(SelectionDAG &DAG, const TargetLowering &TLI,
4417	SmallVectorImpl<SDValue> &ConcatOps,
4418	unsigned ConcatEnd, EVT VT, EVT MaxVT,
4419	EVT WidenVT) {
4420	// Check to see if we have a single operation with the widen type.
4421	if (ConcatEnd == 1) {
4422	VT = ConcatOps[0].getValueType();
4423	if (VT == WidenVT)
4424	return ConcatOps[0];
4425	}
4426
4427	SDLoc dl(ConcatOps[0]);
4428	EVT WidenEltVT = WidenVT.getVectorElementType();
4429
4430	// while (Some element of ConcatOps is not of type MaxVT) {
4431	// From the end of ConcatOps, collect elements of the same type and put
4432	// them into an op of the next larger supported type
4433	// }
4434	while (ConcatOps[ConcatEnd-1].getValueType() != MaxVT) {
4435	int Idx = ConcatEnd - 1;
4436	VT = ConcatOps[Idx--].getValueType();
4437	while (Idx >= 0 && ConcatOps[Idx].getValueType() == VT)
4438	Idx--;
4439
4440	int NextSize = VT.isVector() ? VT.getVectorNumElements() : 1;
4441	EVT NextVT;
4442	do {
4443	NextSize *= 2;
4444	NextVT = EVT::getVectorVT(Context&: *DAG.getContext(), VT: WidenEltVT, NumElements: NextSize);
4445	} while (!TLI.isTypeLegal(VT: NextVT));
4446
4447	if (!VT.isVector()) {
4448	// Scalar type, create an INSERT_VECTOR_ELEMENT of type NextVT
4449	SDValue VecOp = DAG.getUNDEF(VT: NextVT);
4450	unsigned NumToInsert = ConcatEnd - Idx - 1;
4451	for (unsigned i = 0, OpIdx = Idx+1; i < NumToInsert; i++, OpIdx++) {
4452	VecOp = DAG.getNode(Opcode: ISD::INSERT_VECTOR_ELT, DL: dl, VT: NextVT, N1: VecOp,
4453	N2: ConcatOps[OpIdx], N3: DAG.getVectorIdxConstant(Val: i, DL: dl));
4454	}
4455	ConcatOps[Idx+1] = VecOp;
4456	ConcatEnd = Idx + 2;
4457	} else {
4458	// Vector type, create a CONCAT_VECTORS of type NextVT
4459	SDValue undefVec = DAG.getUNDEF(VT);
4460	unsigned OpsToConcat = NextSize/VT.getVectorNumElements();
4461	SmallVector<SDValue, 16> SubConcatOps(OpsToConcat);
4462	unsigned RealVals = ConcatEnd - Idx - 1;
4463	unsigned SubConcatEnd = 0;
4464	unsigned SubConcatIdx = Idx + 1;
4465	while (SubConcatEnd < RealVals)
4466	SubConcatOps[SubConcatEnd++] = ConcatOps[++Idx];
4467	while (SubConcatEnd < OpsToConcat)
4468	SubConcatOps[SubConcatEnd++] = undefVec;
4469	ConcatOps[SubConcatIdx] = DAG.getNode(Opcode: ISD::CONCAT_VECTORS, DL: dl,
4470	VT: NextVT, Ops: SubConcatOps);
4471	ConcatEnd = SubConcatIdx + 1;
4472	}
4473	}
4474
4475	// Check to see if we have a single operation with the widen type.
4476	if (ConcatEnd == 1) {
4477	VT = ConcatOps[0].getValueType();
4478	if (VT == WidenVT)
4479	return ConcatOps[0];
4480	}
4481
4482	// add undefs of size MaxVT until ConcatOps grows to length of WidenVT
4483	unsigned NumOps = WidenVT.getVectorNumElements()/MaxVT.getVectorNumElements();
4484	if (NumOps != ConcatEnd ) {
4485	SDValue UndefVal = DAG.getUNDEF(VT: MaxVT);
4486	for (unsigned j = ConcatEnd; j < NumOps; ++j)
4487	ConcatOps[j] = UndefVal;
4488	}
4489	return DAG.getNode(Opcode: ISD::CONCAT_VECTORS, DL: dl, VT: WidenVT,
4490	Ops: ArrayRef(ConcatOps.data(), NumOps));
4491	}
4492
4493	SDValue DAGTypeLegalizer::WidenVecRes_BinaryCanTrap(SDNode *N) {
4494	// Binary op widening for operations that can trap.
4495	unsigned Opcode = N->getOpcode();
4496	SDLoc dl(N);
4497	EVT WidenVT = TLI.getTypeToTransformTo(Context&: *DAG.getContext(), VT: N->getValueType(ResNo: 0));
4498	EVT WidenEltVT = WidenVT.getVectorElementType();
4499	EVT VT = WidenVT;
4500	unsigned NumElts = VT.getVectorMinNumElements();
4501	const SDNodeFlags Flags = N->getFlags();
4502	while (!TLI.isTypeLegal(VT) && NumElts != 1) {
4503	NumElts = NumElts / 2;
4504	VT = EVT::getVectorVT(Context&: *DAG.getContext(), VT: WidenEltVT, NumElements: NumElts);
4505	}
4506
4507	if (NumElts != 1 && !TLI.canOpTrap(Op: N->getOpcode(), VT)) {
4508	// Operation doesn't trap so just widen as normal.
4509	SDValue InOp1 = GetWidenedVector(Op: N->getOperand(Num: 0));
4510	SDValue InOp2 = GetWidenedVector(Op: N->getOperand(Num: 1));
4511	return DAG.getNode(Opcode: N->getOpcode(), DL: dl, VT: WidenVT, N1: InOp1, N2: InOp2, Flags);
4512	}
4513
4514	// FIXME: Improve support for scalable vectors.
4515	assert(!VT.isScalableVector() && "Scalable vectors not handled yet.");
4516
4517	// No legal vector version so unroll the vector operation and then widen.
4518	if (NumElts == 1)
4519	return DAG.UnrollVectorOp(N, ResNE: WidenVT.getVectorNumElements());
4520
4521	// Since the operation can trap, apply operation on the original vector.
4522	EVT MaxVT = VT;
4523	SDValue InOp1 = GetWidenedVector(Op: N->getOperand(Num: 0));
4524	SDValue InOp2 = GetWidenedVector(Op: N->getOperand(Num: 1));
4525	unsigned CurNumElts = N->getValueType(ResNo: 0).getVectorNumElements();
4526
4527	SmallVector<SDValue, 16> ConcatOps(CurNumElts);
4528	unsigned ConcatEnd = 0; // Current ConcatOps index.
4529	int Idx = 0; // Current Idx into input vectors.
4530
4531	// NumElts := greatest legal vector size (at most WidenVT)
4532	// while (orig. vector has unhandled elements) {
4533	// take munches of size NumElts from the beginning and add to ConcatOps
4534	// NumElts := next smaller supported vector size or 1
4535	// }
4536	while (CurNumElts != 0) {
4537	while (CurNumElts >= NumElts) {
4538	SDValue EOp1 = DAG.getNode(Opcode: ISD::EXTRACT_SUBVECTOR, DL: dl, VT, N1: InOp1,
4539	N2: DAG.getVectorIdxConstant(Val: Idx, DL: dl));
4540	SDValue EOp2 = DAG.getNode(Opcode: ISD::EXTRACT_SUBVECTOR, DL: dl, VT, N1: InOp2,
4541	N2: DAG.getVectorIdxConstant(Val: Idx, DL: dl));
4542	ConcatOps[ConcatEnd++] = DAG.getNode(Opcode, DL: dl, VT, N1: EOp1, N2: EOp2, Flags);
4543	Idx += NumElts;
4544	CurNumElts -= NumElts;
4545	}
4546	do {
4547	NumElts = NumElts / 2;
4548	VT = EVT::getVectorVT(Context&: *DAG.getContext(), VT: WidenEltVT, NumElements: NumElts);
4549	} while (!TLI.isTypeLegal(VT) && NumElts != 1);
4550
4551	if (NumElts == 1) {
4552	for (unsigned i = 0; i != CurNumElts; ++i, ++Idx) {
4553	SDValue EOp1 = DAG.getNode(Opcode: ISD::EXTRACT_VECTOR_ELT, DL: dl, VT: WidenEltVT,
4554	N1: InOp1, N2: DAG.getVectorIdxConstant(Val: Idx, DL: dl));
4555	SDValue EOp2 = DAG.getNode(Opcode: ISD::EXTRACT_VECTOR_ELT, DL: dl, VT: WidenEltVT,
4556	N1: InOp2, N2: DAG.getVectorIdxConstant(Val: Idx, DL: dl));
4557	ConcatOps[ConcatEnd++] = DAG.getNode(Opcode, DL: dl, VT: WidenEltVT,
4558	N1: EOp1, N2: EOp2, Flags);
4559	}
4560	CurNumElts = 0;
4561	}
4562	}
4563
4564	return CollectOpsToWiden(DAG, TLI, ConcatOps, ConcatEnd, VT, MaxVT, WidenVT);
4565	}
4566
4567	SDValue DAGTypeLegalizer::WidenVecRes_StrictFP(SDNode *N) {
4568	switch (N->getOpcode()) {
4569	case ISD::STRICT_FSETCC:
4570	case ISD::STRICT_FSETCCS:
4571	return WidenVecRes_STRICT_FSETCC(N);
4572	case ISD::STRICT_FP_EXTEND:
4573	case ISD::STRICT_FP_ROUND:
4574	case ISD::STRICT_FP_TO_SINT:
4575	case ISD::STRICT_FP_TO_UINT:
4576	case ISD::STRICT_SINT_TO_FP:
4577	case ISD::STRICT_UINT_TO_FP:
4578	return WidenVecRes_Convert_StrictFP(N);
4579	default:
4580	break;
4581	}
4582
4583	// StrictFP op widening for operations that can trap.
4584	unsigned NumOpers = N->getNumOperands();
4585	unsigned Opcode = N->getOpcode();
4586	SDLoc dl(N);
4587	EVT WidenVT = TLI.getTypeToTransformTo(Context&: *DAG.getContext(), VT: N->getValueType(ResNo: 0));
4588	EVT WidenEltVT = WidenVT.getVectorElementType();
4589	EVT VT = WidenVT;
4590	unsigned NumElts = VT.getVectorNumElements();
4591	while (!TLI.isTypeLegal(VT) && NumElts != 1) {
4592	NumElts = NumElts / 2;
4593	VT = EVT::getVectorVT(Context&: *DAG.getContext(), VT: WidenEltVT, NumElements: NumElts);
4594	}
4595
4596	// No legal vector version so unroll the vector operation and then widen.
4597	if (NumElts == 1)
4598	return UnrollVectorOp_StrictFP(N, ResNE: WidenVT.getVectorNumElements());
4599
4600	// Since the operation can trap, apply operation on the original vector.
4601	EVT MaxVT = VT;
4602	SmallVector<SDValue, 4> InOps;
4603	unsigned CurNumElts = N->getValueType(ResNo: 0).getVectorNumElements();
4604
4605	SmallVector<SDValue, 16> ConcatOps(CurNumElts);
4606	SmallVector<SDValue, 16> Chains;
4607	unsigned ConcatEnd = 0; // Current ConcatOps index.
4608	int Idx = 0; // Current Idx into input vectors.
4609
4610	// The Chain is the first operand.
4611	InOps.push_back(Elt: N->getOperand(Num: 0));
4612
4613	// Now process the remaining operands.
4614	for (unsigned i = 1; i < NumOpers; ++i) {
4615	SDValue Oper = N->getOperand(Num: i);
4616
4617	EVT OpVT = Oper.getValueType();
4618	if (OpVT.isVector()) {
4619	if (getTypeAction(VT: OpVT) == TargetLowering::TypeWidenVector)
4620	Oper = GetWidenedVector(Op: Oper);
4621	else {
4622	EVT WideOpVT =
4623	EVT::getVectorVT(Context&: *DAG.getContext(), VT: OpVT.getVectorElementType(),
4624	EC: WidenVT.getVectorElementCount());
4625	Oper = DAG.getNode(Opcode: ISD::INSERT_SUBVECTOR, DL: dl, VT: WideOpVT,
4626	N1: DAG.getUNDEF(VT: WideOpVT), N2: Oper,
4627	N3: DAG.getVectorIdxConstant(Val: 0, DL: dl));
4628	}
4629	}
4630
4631	InOps.push_back(Elt: Oper);
4632	}
4633
4634	// NumElts := greatest legal vector size (at most WidenVT)
4635	// while (orig. vector has unhandled elements) {
4636	// take munches of size NumElts from the beginning and add to ConcatOps
4637	// NumElts := next smaller supported vector size or 1
4638	// }
4639	while (CurNumElts != 0) {
4640	while (CurNumElts >= NumElts) {
4641	SmallVector<SDValue, 4> EOps;
4642
4643	for (unsigned i = 0; i < NumOpers; ++i) {
4644	SDValue Op = InOps[i];
4645
4646	EVT OpVT = Op.getValueType();
4647	if (OpVT.isVector()) {
4648	EVT OpExtractVT =
4649	EVT::getVectorVT(Context&: *DAG.getContext(), VT: OpVT.getVectorElementType(),
4650	EC: VT.getVectorElementCount());
4651	Op = DAG.getNode(Opcode: ISD::EXTRACT_SUBVECTOR, DL: dl, VT: OpExtractVT, N1: Op,
4652	N2: DAG.getVectorIdxConstant(Val: Idx, DL: dl));
4653	}
4654
4655	EOps.push_back(Elt: Op);
4656	}
4657
4658	EVT OperVT[] = {VT, MVT::Other};
4659	SDValue Oper = DAG.getNode(Opcode, dl, OperVT, EOps);
4660	ConcatOps[ConcatEnd++] = Oper;
4661	Chains.push_back(Elt: Oper.getValue(R: 1));
4662	Idx += NumElts;
4663	CurNumElts -= NumElts;
4664	}
4665	do {
4666	NumElts = NumElts / 2;
4667	VT = EVT::getVectorVT(Context&: *DAG.getContext(), VT: WidenEltVT, NumElements: NumElts);
4668	} while (!TLI.isTypeLegal(VT) && NumElts != 1);
4669
4670	if (NumElts == 1) {
4671	for (unsigned i = 0; i != CurNumElts; ++i, ++Idx) {
4672	SmallVector<SDValue, 4> EOps;
4673
4674	for (unsigned i = 0; i < NumOpers; ++i) {
4675	SDValue Op = InOps[i];
4676
4677	EVT OpVT = Op.getValueType();
4678	if (OpVT.isVector())
4679	Op = DAG.getNode(Opcode: ISD::EXTRACT_VECTOR_ELT, DL: dl,
4680	VT: OpVT.getVectorElementType(), N1: Op,
4681	N2: DAG.getVectorIdxConstant(Val: Idx, DL: dl));
4682
4683	EOps.push_back(Elt: Op);
4684	}
4685
4686	EVT WidenVT[] = {WidenEltVT, MVT::Other};
4687	SDValue Oper = DAG.getNode(Opcode, dl, WidenVT, EOps);
4688	ConcatOps[ConcatEnd++] = Oper;
4689	Chains.push_back(Elt: Oper.getValue(R: 1));
4690	}
4691	CurNumElts = 0;
4692	}
4693	}
4694
4695	// Build a factor node to remember all the Ops that have been created.
4696	SDValue NewChain;
4697	if (Chains.size() == 1)
4698	NewChain = Chains[0];
4699	else
4700	NewChain = DAG.getNode(ISD::TokenFactor, dl, MVT::Other, Chains);
4701	ReplaceValueWith(From: SDValue(N, 1), To: NewChain);
4702
4703	return CollectOpsToWiden(DAG, TLI, ConcatOps, ConcatEnd, VT, MaxVT, WidenVT);
4704	}
4705
4706	SDValue DAGTypeLegalizer::WidenVecRes_OverflowOp(SDNode *N, unsigned ResNo) {
4707	SDLoc DL(N);
4708	EVT ResVT = N->getValueType(ResNo: 0);
4709	EVT OvVT = N->getValueType(ResNo: 1);
4710	EVT WideResVT, WideOvVT;
4711	SDValue WideLHS, WideRHS;
4712
4713	// TODO: This might result in a widen/split loop.
4714	if (ResNo == 0) {
4715	WideResVT = TLI.getTypeToTransformTo(Context&: *DAG.getContext(), VT: ResVT);
4716	WideOvVT = EVT::getVectorVT(
4717	Context&: *DAG.getContext(), VT: OvVT.getVectorElementType(),
4718	NumElements: WideResVT.getVectorNumElements());
4719
4720	WideLHS = GetWidenedVector(Op: N->getOperand(Num: 0));
4721	WideRHS = GetWidenedVector(Op: N->getOperand(Num: 1));
4722	} else {
4723	WideOvVT = TLI.getTypeToTransformTo(Context&: *DAG.getContext(), VT: OvVT);
4724	WideResVT = EVT::getVectorVT(
4725	Context&: *DAG.getContext(), VT: ResVT.getVectorElementType(),
4726	NumElements: WideOvVT.getVectorNumElements());
4727
4728	SDValue Zero = DAG.getVectorIdxConstant(Val: 0, DL);
4729	WideLHS = DAG.getNode(
4730	Opcode: ISD::INSERT_SUBVECTOR, DL, VT: WideResVT, N1: DAG.getUNDEF(VT: WideResVT),
4731	N2: N->getOperand(Num: 0), N3: Zero);
4732	WideRHS = DAG.getNode(
4733	Opcode: ISD::INSERT_SUBVECTOR, DL, VT: WideResVT, N1: DAG.getUNDEF(VT: WideResVT),
4734	N2: N->getOperand(Num: 1), N3: Zero);
4735	}
4736
4737	SDVTList WideVTs = DAG.getVTList(VT1: WideResVT, VT2: WideOvVT);
4738	SDNode *WideNode = DAG.getNode(
4739	Opcode: N->getOpcode(), DL, VTList: WideVTs, N1: WideLHS, N2: WideRHS).getNode();
4740
4741	// Replace the other vector result not being explicitly widened here.
4742	unsigned OtherNo = 1 - ResNo;
4743	EVT OtherVT = N->getValueType(ResNo: OtherNo);
4744	if (getTypeAction(VT: OtherVT) == TargetLowering::TypeWidenVector) {
4745	SetWidenedVector(Op: SDValue(N, OtherNo), Result: SDValue(WideNode, OtherNo));
4746	} else {
4747	SDValue Zero = DAG.getVectorIdxConstant(Val: 0, DL);
4748	SDValue OtherVal = DAG.getNode(
4749	Opcode: ISD::EXTRACT_SUBVECTOR, DL, VT: OtherVT, N1: SDValue(WideNode, OtherNo), N2: Zero);
4750	ReplaceValueWith(From: SDValue(N, OtherNo), To: OtherVal);
4751	}
4752
4753	return SDValue(WideNode, ResNo);
4754	}
4755
4756	SDValue DAGTypeLegalizer::WidenVecRes_Convert(SDNode *N) {
4757	LLVMContext &Ctx = *DAG.getContext();
4758	SDValue InOp = N->getOperand(Num: 0);
4759	SDLoc DL(N);
4760
4761	EVT WidenVT = TLI.getTypeToTransformTo(Context&: Ctx, VT: N->getValueType(ResNo: 0));
4762	ElementCount WidenEC = WidenVT.getVectorElementCount();
4763
4764	EVT InVT = InOp.getValueType();
4765
4766	unsigned Opcode = N->getOpcode();
4767	const SDNodeFlags Flags = N->getFlags();
4768
4769	// Handle the case of ZERO_EXTEND where the promoted InVT element size does
4770	// not equal that of WidenVT.
4771	if (N->getOpcode() == ISD::ZERO_EXTEND &&
4772	getTypeAction(VT: InVT) == TargetLowering::TypePromoteInteger &&
4773	TLI.getTypeToTransformTo(Context&: Ctx, VT: InVT).getScalarSizeInBits() !=
4774	WidenVT.getScalarSizeInBits()) {
4775	InOp = ZExtPromotedInteger(Op: InOp);
4776	InVT = InOp.getValueType();
4777	if (WidenVT.getScalarSizeInBits() < InVT.getScalarSizeInBits())
4778	Opcode = ISD::TRUNCATE;
4779	}
4780
4781	EVT InEltVT = InVT.getVectorElementType();
4782	EVT InWidenVT = EVT::getVectorVT(Context&: Ctx, VT: InEltVT, EC: WidenEC);
4783	ElementCount InVTEC = InVT.getVectorElementCount();
4784
4785	if (getTypeAction(VT: InVT) == TargetLowering::TypeWidenVector) {
4786	InOp = GetWidenedVector(Op: N->getOperand(Num: 0));
4787	InVT = InOp.getValueType();
4788	InVTEC = InVT.getVectorElementCount();
4789	if (InVTEC == WidenEC) {
4790	if (N->getNumOperands() == 1)
4791	return DAG.getNode(Opcode, DL, VT: WidenVT, Operand: InOp);
4792	if (N->getNumOperands() == 3) {
4793	assert(N->isVPOpcode() && "Expected VP opcode");
4794	SDValue Mask =
4795	GetWidenedMask(Mask: N->getOperand(Num: 1), EC: WidenVT.getVectorElementCount());
4796	return DAG.getNode(Opcode, DL, VT: WidenVT, N1: InOp, N2: Mask, N3: N->getOperand(Num: 2));
4797	}
4798	return DAG.getNode(Opcode, DL, VT: WidenVT, N1: InOp, N2: N->getOperand(Num: 1), Flags);
4799	}
4800	if (WidenVT.getSizeInBits() == InVT.getSizeInBits()) {
4801	// If both input and result vector types are of same width, extend
4802	// operations should be done with SIGN/ZERO_EXTEND_VECTOR_INREG, which
4803	// accepts fewer elements in the result than in the input.
4804	if (Opcode == ISD::ANY_EXTEND)
4805	return DAG.getNode(Opcode: ISD::ANY_EXTEND_VECTOR_INREG, DL, VT: WidenVT, Operand: InOp);
4806	if (Opcode == ISD::SIGN_EXTEND)
4807	return DAG.getNode(Opcode: ISD::SIGN_EXTEND_VECTOR_INREG, DL, VT: WidenVT, Operand: InOp);
4808	if (Opcode == ISD::ZERO_EXTEND)
4809	return DAG.getNode(Opcode: ISD::ZERO_EXTEND_VECTOR_INREG, DL, VT: WidenVT, Operand: InOp);
4810	}
4811	}
4812
4813	if (TLI.isTypeLegal(VT: InWidenVT)) {
4814	// Because the result and the input are different vector types, widening
4815	// the result could create a legal type but widening the input might make
4816	// it an illegal type that might lead to repeatedly splitting the input
4817	// and then widening it. To avoid this, we widen the input only if
4818	// it results in a legal type.
4819	if (WidenEC.isKnownMultipleOf(RHS: InVTEC.getKnownMinValue())) {
4820	// Widen the input and call convert on the widened input vector.
4821	unsigned NumConcat =
4822	WidenEC.getKnownMinValue() / InVTEC.getKnownMinValue();
4823	SmallVector<SDValue, 16> Ops(NumConcat, DAG.getUNDEF(VT: InVT));
4824	Ops[0] = InOp;
4825	SDValue InVec = DAG.getNode(Opcode: ISD::CONCAT_VECTORS, DL, VT: InWidenVT, Ops);
4826	if (N->getNumOperands() == 1)
4827	return DAG.getNode(Opcode, DL, VT: WidenVT, Operand: InVec);
4828	return DAG.getNode(Opcode, DL, VT: WidenVT, N1: InVec, N2: N->getOperand(Num: 1), Flags);
4829	}
4830
4831	if (InVTEC.isKnownMultipleOf(RHS: WidenEC.getKnownMinValue())) {
4832	SDValue InVal = DAG.getNode(Opcode: ISD::EXTRACT_SUBVECTOR, DL, VT: InWidenVT, N1: InOp,
4833	N2: DAG.getVectorIdxConstant(Val: 0, DL));
4834	// Extract the input and convert the shorten input vector.
4835	if (N->getNumOperands() == 1)
4836	return DAG.getNode(Opcode, DL, VT: WidenVT, Operand: InVal);
4837	return DAG.getNode(Opcode, DL, VT: WidenVT, N1: InVal, N2: N->getOperand(Num: 1), Flags);
4838	}
4839	}
4840
4841	// Otherwise unroll into some nasty scalar code and rebuild the vector.
4842	EVT EltVT = WidenVT.getVectorElementType();
4843	SmallVector<SDValue, 16> Ops(WidenEC.getFixedValue(), DAG.getUNDEF(VT: EltVT));
4844	// Use the original element count so we don't do more scalar opts than
4845	// necessary.
4846	unsigned MinElts = N->getValueType(ResNo: 0).getVectorNumElements();
4847	for (unsigned i=0; i < MinElts; ++i) {
4848	SDValue Val = DAG.getNode(Opcode: ISD::EXTRACT_VECTOR_ELT, DL, VT: InEltVT, N1: InOp,
4849	N2: DAG.getVectorIdxConstant(Val: i, DL));
4850	if (N->getNumOperands() == 1)
4851	Ops[i] = DAG.getNode(Opcode, DL, VT: EltVT, Operand: Val);
4852	else
4853	Ops[i] = DAG.getNode(Opcode, DL, VT: EltVT, N1: Val, N2: N->getOperand(Num: 1), Flags);
4854	}
4855
4856	return DAG.getBuildVector(VT: WidenVT, DL, Ops);
4857	}
4858
4859	SDValue DAGTypeLegalizer::WidenVecRes_FP_TO_XINT_SAT(SDNode *N) {
4860	SDLoc dl(N);
4861	EVT WidenVT = TLI.getTypeToTransformTo(Context&: *DAG.getContext(), VT: N->getValueType(ResNo: 0));
4862	ElementCount WidenNumElts = WidenVT.getVectorElementCount();
4863
4864	SDValue Src = N->getOperand(Num: 0);
4865	EVT SrcVT = Src.getValueType();
4866
4867	// Also widen the input.
4868	if (getTypeAction(VT: SrcVT) == TargetLowering::TypeWidenVector) {
4869	Src = GetWidenedVector(Op: Src);
4870	SrcVT = Src.getValueType();
4871	}
4872
4873	// Input and output not widened to the same size, give up.
4874	if (WidenNumElts != SrcVT.getVectorElementCount())
4875	return DAG.UnrollVectorOp(N, ResNE: WidenNumElts.getKnownMinValue());
4876
4877	return DAG.getNode(Opcode: N->getOpcode(), DL: dl, VT: WidenVT, N1: Src, N2: N->getOperand(Num: 1));
4878	}
4879
4880	SDValue DAGTypeLegalizer::WidenVecRes_XRINT(SDNode *N) {
4881	SDLoc dl(N);
4882	EVT WidenVT = TLI.getTypeToTransformTo(Context&: *DAG.getContext(), VT: N->getValueType(ResNo: 0));
4883	ElementCount WidenNumElts = WidenVT.getVectorElementCount();
4884
4885	SDValue Src = N->getOperand(Num: 0);
4886	EVT SrcVT = Src.getValueType();
4887
4888	// Also widen the input.
4889	if (getTypeAction(VT: SrcVT) == TargetLowering::TypeWidenVector) {
4890	Src = GetWidenedVector(Op: Src);
4891	SrcVT = Src.getValueType();
4892	}
4893
4894	// Input and output not widened to the same size, give up.
4895	if (WidenNumElts != SrcVT.getVectorElementCount())
4896	return DAG.UnrollVectorOp(N, ResNE: WidenNumElts.getKnownMinValue());
4897
4898	if (N->getNumOperands() == 1)
4899	return DAG.getNode(Opcode: N->getOpcode(), DL: dl, VT: WidenVT, Operand: Src);
4900
4901	assert(N->getNumOperands() == 3 && "Unexpected number of operands!");
4902	assert(N->isVPOpcode() && "Expected VP opcode");
4903
4904	SDValue Mask =
4905	GetWidenedMask(Mask: N->getOperand(Num: 1), EC: WidenVT.getVectorElementCount());
4906	return DAG.getNode(Opcode: N->getOpcode(), DL: dl, VT: WidenVT, N1: Src, N2: Mask, N3: N->getOperand(Num: 2));
4907	}
4908
4909	SDValue DAGTypeLegalizer::WidenVecRes_Convert_StrictFP(SDNode *N) {
4910	SDValue InOp = N->getOperand(Num: 1);
4911	SDLoc DL(N);
4912	SmallVector<SDValue, 4> NewOps(N->op_begin(), N->op_end());
4913
4914	EVT WidenVT = TLI.getTypeToTransformTo(Context&: *DAG.getContext(), VT: N->getValueType(ResNo: 0));
4915	unsigned WidenNumElts = WidenVT.getVectorNumElements();
4916
4917	EVT InVT = InOp.getValueType();
4918	EVT InEltVT = InVT.getVectorElementType();
4919
4920	unsigned Opcode = N->getOpcode();
4921
4922	// FIXME: Optimizations need to be implemented here.
4923
4924	// Otherwise unroll into some nasty scalar code and rebuild the vector.
4925	EVT EltVT = WidenVT.getVectorElementType();
4926	std::array<EVT, 2> EltVTs = {{EltVT, MVT::Other}};
4927	SmallVector<SDValue, 16> Ops(WidenNumElts, DAG.getUNDEF(VT: EltVT));
4928	SmallVector<SDValue, 32> OpChains;
4929	// Use the original element count so we don't do more scalar opts than
4930	// necessary.
4931	unsigned MinElts = N->getValueType(ResNo: 0).getVectorNumElements();
4932	for (unsigned i=0; i < MinElts; ++i) {
4933	NewOps[1] = DAG.getNode(Opcode: ISD::EXTRACT_VECTOR_ELT, DL, VT: InEltVT, N1: InOp,
4934	N2: DAG.getVectorIdxConstant(Val: i, DL));
4935	Ops[i] = DAG.getNode(Opcode, DL, ResultTys: EltVTs, Ops: NewOps);
4936	OpChains.push_back(Elt: Ops[i].getValue(R: 1));
4937	}
4938	SDValue NewChain = DAG.getNode(ISD::TokenFactor, DL, MVT::Other, OpChains);
4939	ReplaceValueWith(From: SDValue(N, 1), To: NewChain);
4940
4941	return DAG.getBuildVector(VT: WidenVT, DL, Ops);
4942	}
4943
4944	SDValue DAGTypeLegalizer::WidenVecRes_EXTEND_VECTOR_INREG(SDNode *N) {
4945	unsigned Opcode = N->getOpcode();
4946	SDValue InOp = N->getOperand(Num: 0);
4947	SDLoc DL(N);
4948
4949	EVT WidenVT = TLI.getTypeToTransformTo(Context&: *DAG.getContext(), VT: N->getValueType(ResNo: 0));
4950	EVT WidenSVT = WidenVT.getVectorElementType();
4951	unsigned WidenNumElts = WidenVT.getVectorNumElements();
4952
4953	EVT InVT = InOp.getValueType();
4954	EVT InSVT = InVT.getVectorElementType();
4955	unsigned InVTNumElts = InVT.getVectorNumElements();
4956
4957	if (getTypeAction(VT: InVT) == TargetLowering::TypeWidenVector) {
4958	InOp = GetWidenedVector(Op: InOp);
4959	InVT = InOp.getValueType();
4960	if (InVT.getSizeInBits() == WidenVT.getSizeInBits()) {
4961	switch (Opcode) {
4962	case ISD::ANY_EXTEND_VECTOR_INREG:
4963	case ISD::SIGN_EXTEND_VECTOR_INREG:
4964	case ISD::ZERO_EXTEND_VECTOR_INREG:
4965	return DAG.getNode(Opcode, DL, VT: WidenVT, Operand: InOp);
4966	}
4967	}
4968	}
4969
4970	// Unroll, extend the scalars and rebuild the vector.
4971	SmallVector<SDValue, 16> Ops;
4972	for (unsigned i = 0, e = std::min(a: InVTNumElts, b: WidenNumElts); i != e; ++i) {
4973	SDValue Val = DAG.getNode(Opcode: ISD::EXTRACT_VECTOR_ELT, DL, VT: InSVT, N1: InOp,
4974	N2: DAG.getVectorIdxConstant(Val: i, DL));
4975	switch (Opcode) {
4976	case ISD::ANY_EXTEND_VECTOR_INREG:
4977	Val = DAG.getNode(Opcode: ISD::ANY_EXTEND, DL, VT: WidenSVT, Operand: Val);
4978	break;
4979	case ISD::SIGN_EXTEND_VECTOR_INREG:
4980	Val = DAG.getNode(Opcode: ISD::SIGN_EXTEND, DL, VT: WidenSVT, Operand: Val);
4981	break;
4982	case ISD::ZERO_EXTEND_VECTOR_INREG:
4983	Val = DAG.getNode(Opcode: ISD::ZERO_EXTEND, DL, VT: WidenSVT, Operand: Val);
4984	break;
4985	default:
4986	llvm_unreachable("A *_EXTEND_VECTOR_INREG node was expected");
4987	}
4988	Ops.push_back(Elt: Val);
4989	}
4990
4991	while (Ops.size() != WidenNumElts)
4992	Ops.push_back(Elt: DAG.getUNDEF(VT: WidenSVT));
4993
4994	return DAG.getBuildVector(VT: WidenVT, DL, Ops);
4995	}
4996
4997	SDValue DAGTypeLegalizer::WidenVecRes_FCOPYSIGN(SDNode *N) {
4998	// If this is an FCOPYSIGN with same input types, we can treat it as a
4999	// normal (can trap) binary op.
5000	if (N->getOperand(Num: 0).getValueType() == N->getOperand(Num: 1).getValueType())
5001	return WidenVecRes_BinaryCanTrap(N);
5002
5003	// If the types are different, fall back to unrolling.
5004	EVT WidenVT = TLI.getTypeToTransformTo(Context&: *DAG.getContext(), VT: N->getValueType(ResNo: 0));
5005	return DAG.UnrollVectorOp(N, ResNE: WidenVT.getVectorNumElements());
5006	}
5007
5008	/// Result and first source operand are different scalar types, but must have
5009	/// the same number of elements. There is an additional control argument which
5010	/// should be passed through unchanged.
5011	SDValue DAGTypeLegalizer::WidenVecRes_UnarySameEltsWithScalarArg(SDNode *N) {
5012	SDValue FpValue = N->getOperand(Num: 0);
5013	EVT WidenVT = TLI.getTypeToTransformTo(Context&: *DAG.getContext(), VT: N->getValueType(ResNo: 0));
5014	if (getTypeAction(VT: FpValue.getValueType()) != TargetLowering::TypeWidenVector)
5015	return DAG.UnrollVectorOp(N, ResNE: WidenVT.getVectorNumElements());
5016	SDValue Arg = GetWidenedVector(Op: FpValue);
5017	return DAG.getNode(Opcode: N->getOpcode(), DL: SDLoc(N), VT: WidenVT, Ops: {Arg, N->getOperand(Num: 1)},
5018	Flags: N->getFlags());
5019	}
5020
5021	SDValue DAGTypeLegalizer::WidenVecRes_ExpOp(SDNode *N) {
5022	EVT WidenVT = TLI.getTypeToTransformTo(Context&: *DAG.getContext(), VT: N->getValueType(ResNo: 0));
5023	SDValue InOp = GetWidenedVector(Op: N->getOperand(Num: 0));
5024	SDValue RHS = N->getOperand(Num: 1);
5025	SDValue ExpOp = RHS.getValueType().isVector() ? GetWidenedVector(Op: RHS) : RHS;
5026
5027	return DAG.getNode(Opcode: N->getOpcode(), DL: SDLoc(N), VT: WidenVT, N1: InOp, N2: ExpOp);
5028	}
5029
5030	SDValue DAGTypeLegalizer::WidenVecRes_Unary(SDNode *N) {
5031	// Unary op widening.
5032	EVT WidenVT = TLI.getTypeToTransformTo(Context&: *DAG.getContext(), VT: N->getValueType(ResNo: 0));
5033	SDValue InOp = GetWidenedVector(Op: N->getOperand(Num: 0));
5034	if (N->getNumOperands() == 1)
5035	return DAG.getNode(Opcode: N->getOpcode(), DL: SDLoc(N), VT: WidenVT, Operand: InOp, Flags: N->getFlags());
5036
5037	assert(N->getNumOperands() == 3 && "Unexpected number of operands!");
5038	assert(N->isVPOpcode() && "Expected VP opcode");
5039
5040	SDValue Mask =
5041	GetWidenedMask(Mask: N->getOperand(Num: 1), EC: WidenVT.getVectorElementCount());
5042	return DAG.getNode(Opcode: N->getOpcode(), DL: SDLoc(N), VT: WidenVT,
5043	Ops: {InOp, Mask, N->getOperand(Num: 2)});
5044	}
5045
5046	SDValue DAGTypeLegalizer::WidenVecRes_InregOp(SDNode *N) {
5047	EVT WidenVT = TLI.getTypeToTransformTo(Context&: *DAG.getContext(), VT: N->getValueType(ResNo: 0));
5048	EVT ExtVT = EVT::getVectorVT(Context&: *DAG.getContext(),
5049	VT: cast<VTSDNode>(Val: N->getOperand(Num: 1))->getVT()
5050	.getVectorElementType(),
5051	NumElements: WidenVT.getVectorNumElements());
5052	SDValue WidenLHS = GetWidenedVector(Op: N->getOperand(Num: 0));
5053	return DAG.getNode(Opcode: N->getOpcode(), DL: SDLoc(N),
5054	VT: WidenVT, N1: WidenLHS, N2: DAG.getValueType(ExtVT));
5055	}
5056
5057	SDValue DAGTypeLegalizer::WidenVecRes_MERGE_VALUES(SDNode *N, unsigned ResNo) {
5058	SDValue WidenVec = DisintegrateMERGE_VALUES(N, ResNo);
5059	return GetWidenedVector(Op: WidenVec);
5060	}
5061
5062	SDValue DAGTypeLegalizer::WidenVecRes_BITCAST(SDNode *N) {
5063	SDValue InOp = N->getOperand(Num: 0);
5064	EVT InVT = InOp.getValueType();
5065	EVT VT = N->getValueType(ResNo: 0);
5066	EVT WidenVT = TLI.getTypeToTransformTo(Context&: *DAG.getContext(), VT);
5067	SDLoc dl(N);
5068
5069	switch (getTypeAction(VT: InVT)) {
5070	case TargetLowering::TypeLegal:
5071	break;
5072	case TargetLowering::TypeScalarizeScalableVector:
5073	report_fatal_error(reason: "Scalarization of scalable vectors is not supported.");
5074	case TargetLowering::TypePromoteInteger: {
5075	// If the incoming type is a vector that is being promoted, then
5076	// we know that the elements are arranged differently and that we
5077	// must perform the conversion using a stack slot.
5078	if (InVT.isVector())
5079	break;
5080
5081	// If the InOp is promoted to the same size, convert it. Otherwise,
5082	// fall out of the switch and widen the promoted input.
5083	SDValue NInOp = GetPromotedInteger(Op: InOp);
5084	EVT NInVT = NInOp.getValueType();
5085	if (WidenVT.bitsEq(VT: NInVT)) {
5086	// For big endian targets we need to shift the input integer or the
5087	// interesting bits will end up at the wrong place.
5088	if (DAG.getDataLayout().isBigEndian()) {
5089	unsigned ShiftAmt = NInVT.getSizeInBits() - InVT.getSizeInBits();
5090	EVT ShiftAmtTy = TLI.getShiftAmountTy(LHSTy: NInVT, DL: DAG.getDataLayout());
5091	assert(ShiftAmt < WidenVT.getSizeInBits() && "Too large shift amount!");
5092	NInOp = DAG.getNode(Opcode: ISD::SHL, DL: dl, VT: NInVT, N1: NInOp,
5093	N2: DAG.getConstant(Val: ShiftAmt, DL: dl, VT: ShiftAmtTy));
5094	}
5095	return DAG.getNode(Opcode: ISD::BITCAST, DL: dl, VT: WidenVT, Operand: NInOp);
5096	}
5097	InOp = NInOp;
5098	InVT = NInVT;
5099	break;
5100	}
5101	case TargetLowering::TypeSoftenFloat:
5102	case TargetLowering::TypePromoteFloat:
5103	case TargetLowering::TypeSoftPromoteHalf:
5104	case TargetLowering::TypeExpandInteger:
5105	case TargetLowering::TypeExpandFloat:
5106	case TargetLowering::TypeScalarizeVector:
5107	case TargetLowering::TypeSplitVector:
5108	break;
5109	case TargetLowering::TypeWidenVector:
5110	// If the InOp is widened to the same size, convert it. Otherwise, fall
5111	// out of the switch and widen the widened input.
5112	InOp = GetWidenedVector(Op: InOp);
5113	InVT = InOp.getValueType();
5114	if (WidenVT.bitsEq(VT: InVT))
5115	// The input widens to the same size. Convert to the widen value.
5116	return DAG.getNode(Opcode: ISD::BITCAST, DL: dl, VT: WidenVT, Operand: InOp);
5117	break;
5118	}
5119
5120	unsigned WidenSize = WidenVT.getSizeInBits();
5121	unsigned InSize = InVT.getSizeInBits();
5122	unsigned InScalarSize = InVT.getScalarSizeInBits();
5123	// x86mmx is not an acceptable vector element type, so don't try.
5124	if (WidenSize % InScalarSize == 0 && InVT != MVT::x86mmx) {
5125	// Determine new input vector type. The new input vector type will use
5126	// the same element type (if its a vector) or use the input type as a
5127	// vector. It is the same size as the type to widen to.
5128	EVT NewInVT;
5129	unsigned NewNumParts = WidenSize / InSize;
5130	if (InVT.isVector()) {
5131	EVT InEltVT = InVT.getVectorElementType();
5132	NewInVT = EVT::getVectorVT(Context&: *DAG.getContext(), VT: InEltVT,
5133	NumElements: WidenSize / InEltVT.getSizeInBits());
5134	} else {
5135	// For big endian systems, using the promoted input scalar type
5136	// to produce the scalar_to_vector would put the desired bits into
5137	// the least significant byte(s) of the wider element zero. This
5138	// will mean that the users of the result vector are using incorrect
5139	// bits. Use the original input type instead. Although either input
5140	// type can be used on little endian systems, for consistency we
5141	// use the original type there as well.
5142	EVT OrigInVT = N->getOperand(Num: 0).getValueType();
5143	NewNumParts = WidenSize / OrigInVT.getSizeInBits();
5144	NewInVT = EVT::getVectorVT(Context&: *DAG.getContext(), VT: OrigInVT, NumElements: NewNumParts);
5145	}
5146
5147	if (TLI.isTypeLegal(VT: NewInVT)) {
5148	SDValue NewVec;
5149	if (InVT.isVector()) {
5150	// Because the result and the input are different vector types, widening
5151	// the result could create a legal type but widening the input might
5152	// make it an illegal type that might lead to repeatedly splitting the
5153	// input and then widening it. To avoid this, we widen the input only if
5154	// it results in a legal type.
5155	if (WidenSize % InSize == 0) {
5156	SmallVector<SDValue, 16> Ops(NewNumParts, DAG.getUNDEF(VT: InVT));
5157	Ops[0] = InOp;
5158
5159	NewVec = DAG.getNode(Opcode: ISD::CONCAT_VECTORS, DL: dl, VT: NewInVT, Ops);
5160	} else {
5161	SmallVector<SDValue, 16> Ops;
5162	DAG.ExtractVectorElements(Op: InOp, Args&: Ops);
5163	Ops.append(NumInputs: WidenSize / InScalarSize - Ops.size(),
5164	Elt: DAG.getUNDEF(VT: InVT.getVectorElementType()));
5165
5166	NewVec = DAG.getNode(Opcode: ISD::BUILD_VECTOR, DL: dl, VT: NewInVT, Ops);
5167	}
5168	} else {
5169	NewVec = DAG.getNode(Opcode: ISD::SCALAR_TO_VECTOR, DL: dl, VT: NewInVT, Operand: InOp);
5170	}
5171	return DAG.getNode(Opcode: ISD::BITCAST, DL: dl, VT: WidenVT, Operand: NewVec);
5172	}
5173	}
5174
5175	return CreateStackStoreLoad(Op: InOp, DestVT: WidenVT);
5176	}
5177
5178	SDValue DAGTypeLegalizer::WidenVecRes_BUILD_VECTOR(SDNode *N) {
5179	SDLoc dl(N);
5180	// Build a vector with undefined for the new nodes.
5181	EVT VT = N->getValueType(ResNo: 0);
5182
5183	// Integer BUILD_VECTOR operands may be larger than the node's vector element
5184	// type. The UNDEFs need to have the same type as the existing operands.
5185	EVT EltVT = N->getOperand(Num: 0).getValueType();
5186	unsigned NumElts = VT.getVectorNumElements();
5187
5188	EVT WidenVT = TLI.getTypeToTransformTo(Context&: *DAG.getContext(), VT);
5189	unsigned WidenNumElts = WidenVT.getVectorNumElements();
5190
5191	SmallVector<SDValue, 16> NewOps(N->op_begin(), N->op_end());
5192	assert(WidenNumElts >= NumElts && "Shrinking vector instead of widening!");
5193	NewOps.append(NumInputs: WidenNumElts - NumElts, Elt: DAG.getUNDEF(VT: EltVT));
5194
5195	return DAG.getBuildVector(VT: WidenVT, DL: dl, Ops: NewOps);
5196	}
5197
5198	SDValue DAGTypeLegalizer::WidenVecRes_CONCAT_VECTORS(SDNode *N) {
5199	EVT InVT = N->getOperand(Num: 0).getValueType();
5200	EVT WidenVT = TLI.getTypeToTransformTo(Context&: *DAG.getContext(), VT: N->getValueType(ResNo: 0));
5201	SDLoc dl(N);
5202	unsigned NumOperands = N->getNumOperands();
5203
5204	bool InputWidened = false; // Indicates we need to widen the input.
5205	if (getTypeAction(VT: InVT) != TargetLowering::TypeWidenVector) {
5206	unsigned WidenNumElts = WidenVT.getVectorMinNumElements();
5207	unsigned NumInElts = InVT.getVectorMinNumElements();
5208	if (WidenNumElts % NumInElts == 0) {
5209	// Add undef vectors to widen to correct length.
5210	unsigned NumConcat = WidenNumElts / NumInElts;
5211	SDValue UndefVal = DAG.getUNDEF(VT: InVT);
5212	SmallVector<SDValue, 16> Ops(NumConcat);
5213	for (unsigned i=0; i < NumOperands; ++i)
5214	Ops[i] = N->getOperand(Num: i);
5215	for (unsigned i = NumOperands; i != NumConcat; ++i)
5216	Ops[i] = UndefVal;
5217	return DAG.getNode(Opcode: ISD::CONCAT_VECTORS, DL: dl, VT: WidenVT, Ops);
5218	}
5219	} else {
5220	InputWidened = true;
5221	if (WidenVT == TLI.getTypeToTransformTo(Context&: *DAG.getContext(), VT: InVT)) {
5222	// The inputs and the result are widen to the same value.
5223	unsigned i;
5224	for (i=1; i < NumOperands; ++i)
5225	if (!N->getOperand(Num: i).isUndef())
5226	break;
5227
5228	if (i == NumOperands)
5229	// Everything but the first operand is an UNDEF so just return the
5230	// widened first operand.
5231	return GetWidenedVector(Op: N->getOperand(Num: 0));
5232
5233	if (NumOperands == 2) {
5234	assert(!WidenVT.isScalableVector() &&
5235	"Cannot use vector shuffles to widen CONCAT_VECTOR result");
5236	unsigned WidenNumElts = WidenVT.getVectorNumElements();
5237	unsigned NumInElts = InVT.getVectorNumElements();
5238
5239	// Replace concat of two operands with a shuffle.
5240	SmallVector<int, 16> MaskOps(WidenNumElts, -1);
5241	for (unsigned i = 0; i < NumInElts; ++i) {
5242	MaskOps[i] = i;
5243	MaskOps[i + NumInElts] = i + WidenNumElts;
5244	}
5245	return DAG.getVectorShuffle(VT: WidenVT, dl,
5246	N1: GetWidenedVector(Op: N->getOperand(Num: 0)),
5247	N2: GetWidenedVector(Op: N->getOperand(Num: 1)),
5248	Mask: MaskOps);
5249	}
5250	}
5251	}
5252
5253	assert(!WidenVT.isScalableVector() &&
5254	"Cannot use build vectors to widen CONCAT_VECTOR result");
5255	unsigned WidenNumElts = WidenVT.getVectorNumElements();
5256	unsigned NumInElts = InVT.getVectorNumElements();
5257
5258	// Fall back to use extracts and build vector.
5259	EVT EltVT = WidenVT.getVectorElementType();
5260	SmallVector<SDValue, 16> Ops(WidenNumElts);
5261	unsigned Idx = 0;
5262	for (unsigned i=0; i < NumOperands; ++i) {
5263	SDValue InOp = N->getOperand(Num: i);
5264	if (InputWidened)
5265	InOp = GetWidenedVector(Op: InOp);
5266	for (unsigned j = 0; j < NumInElts; ++j)
5267	Ops[Idx++] = DAG.getNode(Opcode: ISD::EXTRACT_VECTOR_ELT, DL: dl, VT: EltVT, N1: InOp,
5268	N2: DAG.getVectorIdxConstant(Val: j, DL: dl));
5269	}
5270	SDValue UndefVal = DAG.getUNDEF(VT: EltVT);
5271	for (; Idx < WidenNumElts; ++Idx)
5272	Ops[Idx] = UndefVal;
5273	return DAG.getBuildVector(VT: WidenVT, DL: dl, Ops);
5274	}
5275
5276	SDValue DAGTypeLegalizer::WidenVecRes_INSERT_SUBVECTOR(SDNode *N) {
5277	EVT VT = N->getValueType(ResNo: 0);
5278	EVT WidenVT = TLI.getTypeToTransformTo(Context&: *DAG.getContext(), VT);
5279	SDValue InOp1 = GetWidenedVector(Op: N->getOperand(Num: 0));
5280	SDValue InOp2 = N->getOperand(Num: 1);
5281	SDValue Idx = N->getOperand(Num: 2);
5282	SDLoc dl(N);
5283	return DAG.getNode(Opcode: ISD::INSERT_SUBVECTOR, DL: dl, VT: WidenVT, N1: InOp1, N2: InOp2, N3: Idx);
5284	}
5285
5286	SDValue DAGTypeLegalizer::WidenVecRes_EXTRACT_SUBVECTOR(SDNode *N) {
5287	EVT VT = N->getValueType(ResNo: 0);
5288	EVT EltVT = VT.getVectorElementType();
5289	EVT WidenVT = TLI.getTypeToTransformTo(Context&: *DAG.getContext(), VT);
5290	SDValue InOp = N->getOperand(Num: 0);
5291	SDValue Idx = N->getOperand(Num: 1);
5292	SDLoc dl(N);
5293
5294	auto InOpTypeAction = getTypeAction(VT: InOp.getValueType());
5295	if (InOpTypeAction == TargetLowering::TypeWidenVector)
5296	InOp = GetWidenedVector(Op: InOp);
5297
5298	EVT InVT = InOp.getValueType();
5299
5300	// Check if we can just return the input vector after widening.
5301	uint64_t IdxVal = Idx->getAsZExtVal();
5302	if (IdxVal == 0 && InVT == WidenVT)
5303	return InOp;
5304
5305	// Check if we can extract from the vector.
5306	unsigned WidenNumElts = WidenVT.getVectorMinNumElements();
5307	unsigned InNumElts = InVT.getVectorMinNumElements();
5308	unsigned VTNumElts = VT.getVectorMinNumElements();
5309	assert(IdxVal % VTNumElts == 0 &&
5310	"Expected Idx to be a multiple of subvector minimum vector length");
5311	if (IdxVal % WidenNumElts == 0 && IdxVal + WidenNumElts < InNumElts)
5312	return DAG.getNode(Opcode: ISD::EXTRACT_SUBVECTOR, DL: dl, VT: WidenVT, N1: InOp, N2: Idx);
5313
5314	if (VT.isScalableVector()) {
5315	// Try to split the operation up into smaller extracts and concat the
5316	// results together, e.g.
5317	// nxv6i64 extract_subvector(nxv12i64, 6)
5318	// <->
5319	// nxv8i64 concat(
5320	// nxv2i64 extract_subvector(nxv16i64, 6)
5321	// nxv2i64 extract_subvector(nxv16i64, 8)
5322	// nxv2i64 extract_subvector(nxv16i64, 10)
5323	// undef)
5324	unsigned GCD = std::gcd(m: VTNumElts, n: WidenNumElts);
5325	assert((IdxVal % GCD) == 0 && "Expected Idx to be a multiple of the broken "
5326	"down type's element count");
5327	EVT PartVT = EVT::getVectorVT(Context&: *DAG.getContext(), VT: EltVT,
5328	EC: ElementCount::getScalable(MinVal: GCD));
5329	// Avoid recursion around e.g. nxv1i8.
5330	if (getTypeAction(VT: PartVT) != TargetLowering::TypeWidenVector) {
5331	SmallVector<SDValue> Parts;
5332	unsigned I = 0;
5333	for (; I < VTNumElts / GCD; ++I)
5334	Parts.push_back(
5335	Elt: DAG.getNode(Opcode: ISD::EXTRACT_SUBVECTOR, DL: dl, VT: PartVT, N1: InOp,
5336	N2: DAG.getVectorIdxConstant(Val: IdxVal + I * GCD, DL: dl)));
5337	for (; I < WidenNumElts / GCD; ++I)
5338	Parts.push_back(Elt: DAG.getUNDEF(VT: PartVT));
5339
5340	return DAG.getNode(Opcode: ISD::CONCAT_VECTORS, DL: dl, VT: WidenVT, Ops: Parts);
5341	}
5342
5343	report_fatal_error(reason: "Don't know how to widen the result of "
5344	"EXTRACT_SUBVECTOR for scalable vectors");
5345	}
5346
5347	// We could try widening the input to the right length but for now, extract
5348	// the original elements, fill the rest with undefs and build a vector.
5349	SmallVector<SDValue, 16> Ops(WidenNumElts);
5350	unsigned i;
5351	for (i = 0; i < VTNumElts; ++i)
5352	Ops[i] = DAG.getNode(Opcode: ISD::EXTRACT_VECTOR_ELT, DL: dl, VT: EltVT, N1: InOp,
5353	N2: DAG.getVectorIdxConstant(Val: IdxVal + i, DL: dl));
5354
5355	SDValue UndefVal = DAG.getUNDEF(VT: EltVT);
5356	for (; i < WidenNumElts; ++i)
5357	Ops[i] = UndefVal;
5358	return DAG.getBuildVector(VT: WidenVT, DL: dl, Ops);
5359	}
5360
5361	SDValue DAGTypeLegalizer::WidenVecRes_AssertZext(SDNode *N) {
5362	SDValue InOp = ModifyToType(
5363	InOp: N->getOperand(Num: 0),
5364	NVT: TLI.getTypeToTransformTo(Context&: *DAG.getContext(), VT: N->getValueType(ResNo: 0)), FillWithZeroes: true);
5365	return DAG.getNode(Opcode: ISD::AssertZext, DL: SDLoc(N), VT: InOp.getValueType(), N1: InOp,
5366	N2: N->getOperand(Num: 1));
5367	}
5368
5369	SDValue DAGTypeLegalizer::WidenVecRes_INSERT_VECTOR_ELT(SDNode *N) {
5370	SDValue InOp = GetWidenedVector(Op: N->getOperand(Num: 0));
5371	return DAG.getNode(Opcode: ISD::INSERT_VECTOR_ELT, DL: SDLoc(N),
5372	VT: InOp.getValueType(), N1: InOp,
5373	N2: N->getOperand(Num: 1), N3: N->getOperand(Num: 2));
5374	}
5375
5376	SDValue DAGTypeLegalizer::WidenVecRes_LOAD(SDNode *N) {
5377	LoadSDNode *LD = cast<LoadSDNode>(Val: N);
5378	ISD::LoadExtType ExtType = LD->getExtensionType();
5379
5380	// A vector must always be stored in memory as-is, i.e. without any padding
5381	// between the elements, since various code depend on it, e.g. in the
5382	// handling of a bitcast of a vector type to int, which may be done with a
5383	// vector store followed by an integer load. A vector that does not have
5384	// elements that are byte-sized must therefore be stored as an integer
5385	// built out of the extracted vector elements.
5386	if (!LD->getMemoryVT().isByteSized()) {
5387	SDValue Value, NewChain;
5388	std::tie(args&: Value, args&: NewChain) = TLI.scalarizeVectorLoad(LD, DAG);
5389	ReplaceValueWith(From: SDValue(LD, 0), To: Value);
5390	ReplaceValueWith(From: SDValue(LD, 1), To: NewChain);
5391	return SDValue();
5392	}
5393
5394	// Generate a vector-predicated load if it is custom/legal on the target. To
5395	// avoid possible recursion, only do this if the widened mask type is legal.
5396	// FIXME: Not all targets may support EVL in VP_LOAD. These will have been
5397	// removed from the IR by the ExpandVectorPredication pass but we're
5398	// reintroducing them here.
5399	EVT LdVT = LD->getMemoryVT();
5400	EVT WideVT = TLI.getTypeToTransformTo(Context&: *DAG.getContext(), VT: LdVT);
5401	EVT WideMaskVT = EVT::getVectorVT(*DAG.getContext(), MVT::i1,
5402	WideVT.getVectorElementCount());
5403	if (ExtType == ISD::NON_EXTLOAD &&
5404	TLI.isOperationLegalOrCustom(Op: ISD::VP_LOAD, VT: WideVT) &&
5405	TLI.isTypeLegal(VT: WideMaskVT)) {
5406	SDLoc DL(N);
5407	SDValue Mask = DAG.getAllOnesConstant(DL, VT: WideMaskVT);
5408	SDValue EVL = DAG.getElementCount(DL, VT: TLI.getVPExplicitVectorLengthTy(),
5409	EC: LdVT.getVectorElementCount());
5410	const auto *MMO = LD->getMemOperand();
5411	SDValue NewLoad =
5412	DAG.getLoadVP(VT: WideVT, dl: DL, Chain: LD->getChain(), Ptr: LD->getBasePtr(), Mask, EVL,
5413	PtrInfo: MMO->getPointerInfo(), Alignment: MMO->getAlign(), MMOFlags: MMO->getFlags(),
5414	AAInfo: MMO->getAAInfo());
5415
5416	// Modified the chain - switch anything that used the old chain to use
5417	// the new one.
5418	ReplaceValueWith(From: SDValue(N, 1), To: NewLoad.getValue(R: 1));
5419
5420	return NewLoad;
5421	}
5422
5423	SDValue Result;
5424	SmallVector<SDValue, 16> LdChain; // Chain for the series of load
5425	if (ExtType != ISD::NON_EXTLOAD)
5426	Result = GenWidenVectorExtLoads(LdChain, LD, ExtType);
5427	else
5428	Result = GenWidenVectorLoads(LdChain, LD);
5429
5430	if (Result) {
5431	// If we generate a single load, we can use that for the chain. Otherwise,
5432	// build a factor node to remember the multiple loads are independent and
5433	// chain to that.
5434	SDValue NewChain;
5435	if (LdChain.size() == 1)
5436	NewChain = LdChain[0];
5437	else
5438	NewChain = DAG.getNode(ISD::TokenFactor, SDLoc(LD), MVT::Other, LdChain);
5439
5440	// Modified the chain - switch anything that used the old chain to use
5441	// the new one.
5442	ReplaceValueWith(From: SDValue(N, 1), To: NewChain);
5443
5444	return Result;
5445	}
5446
5447	report_fatal_error(reason: "Unable to widen vector load");
5448	}
5449
5450	SDValue DAGTypeLegalizer::WidenVecRes_VP_LOAD(VPLoadSDNode *N) {
5451	EVT WidenVT = TLI.getTypeToTransformTo(Context&: *DAG.getContext(), VT: N->getValueType(ResNo: 0));
5452	SDValue Mask = N->getMask();
5453	SDValue EVL = N->getVectorLength();
5454	ISD::LoadExtType ExtType = N->getExtensionType();
5455	SDLoc dl(N);
5456
5457	// The mask should be widened as well
5458	assert(getTypeAction(Mask.getValueType()) ==
5459	TargetLowering::TypeWidenVector &&
5460	"Unable to widen binary VP op");
5461	Mask = GetWidenedVector(Op: Mask);
5462	assert(Mask.getValueType().getVectorElementCount() ==
5463	TLI.getTypeToTransformTo(*DAG.getContext(), Mask.getValueType())
5464	.getVectorElementCount() &&
5465	"Unable to widen vector load");
5466
5467	SDValue Res =
5468	DAG.getLoadVP(AM: N->getAddressingMode(), ExtType, VT: WidenVT, dl, Chain: N->getChain(),
5469	Ptr: N->getBasePtr(), Offset: N->getOffset(), Mask, EVL,
5470	MemVT: N->getMemoryVT(), MMO: N->getMemOperand(), IsExpanding: N->isExpandingLoad());
5471	// Legalize the chain result - switch anything that used the old chain to
5472	// use the new one.
5473	ReplaceValueWith(From: SDValue(N, 1), To: Res.getValue(R: 1));
5474	return Res;
5475	}
5476
5477	SDValue DAGTypeLegalizer::WidenVecRes_VP_STRIDED_LOAD(VPStridedLoadSDNode *N) {
5478	SDLoc DL(N);
5479
5480	// The mask should be widened as well
5481	SDValue Mask = N->getMask();
5482	assert(getTypeAction(Mask.getValueType()) ==
5483	TargetLowering::TypeWidenVector &&
5484	"Unable to widen VP strided load");
5485	Mask = GetWidenedVector(Op: Mask);
5486
5487	EVT WidenVT = TLI.getTypeToTransformTo(Context&: *DAG.getContext(), VT: N->getValueType(ResNo: 0));
5488	assert(Mask.getValueType().getVectorElementCount() ==
5489	WidenVT.getVectorElementCount() &&
5490	"Data and mask vectors should have the same number of elements");
5491
5492	SDValue Res = DAG.getStridedLoadVP(
5493	AM: N->getAddressingMode(), ExtType: N->getExtensionType(), VT: WidenVT, DL, Chain: N->getChain(),
5494	Ptr: N->getBasePtr(), Offset: N->getOffset(), Stride: N->getStride(), Mask,
5495	EVL: N->getVectorLength(), MemVT: N->getMemoryVT(), MMO: N->getMemOperand(),
5496	IsExpanding: N->isExpandingLoad());
5497
5498	// Legalize the chain result - switch anything that used the old chain to
5499	// use the new one.
5500	ReplaceValueWith(From: SDValue(N, 1), To: Res.getValue(R: 1));
5501	return Res;
5502	}
5503
5504	SDValue DAGTypeLegalizer::WidenVecRes_MLOAD(MaskedLoadSDNode *N) {
5505
5506	EVT WidenVT = TLI.getTypeToTransformTo(Context&: *DAG.getContext(),VT: N->getValueType(ResNo: 0));
5507	SDValue Mask = N->getMask();
5508	EVT MaskVT = Mask.getValueType();
5509	SDValue PassThru = GetWidenedVector(Op: N->getPassThru());
5510	ISD::LoadExtType ExtType = N->getExtensionType();
5511	SDLoc dl(N);
5512
5513	// The mask should be widened as well
5514	EVT WideMaskVT = EVT::getVectorVT(Context&: *DAG.getContext(),
5515	VT: MaskVT.getVectorElementType(),
5516	NumElements: WidenVT.getVectorNumElements());
5517	Mask = ModifyToType(InOp: Mask, NVT: WideMaskVT, FillWithZeroes: true);
5518
5519	SDValue Res = DAG.getMaskedLoad(
5520	VT: WidenVT, dl, Chain: N->getChain(), Base: N->getBasePtr(), Offset: N->getOffset(), Mask,
5521	Src0: PassThru, MemVT: N->getMemoryVT(), MMO: N->getMemOperand(), AM: N->getAddressingMode(),
5522	ExtType, IsExpanding: N->isExpandingLoad());
5523	// Legalize the chain result - switch anything that used the old chain to
5524	// use the new one.
5525	ReplaceValueWith(From: SDValue(N, 1), To: Res.getValue(R: 1));
5526	return Res;
5527	}
5528
5529	SDValue DAGTypeLegalizer::WidenVecRes_MGATHER(MaskedGatherSDNode *N) {
5530
5531	EVT WideVT = TLI.getTypeToTransformTo(Context&: *DAG.getContext(), VT: N->getValueType(ResNo: 0));
5532	SDValue Mask = N->getMask();
5533	EVT MaskVT = Mask.getValueType();
5534	SDValue PassThru = GetWidenedVector(Op: N->getPassThru());
5535	SDValue Scale = N->getScale();
5536	unsigned NumElts = WideVT.getVectorNumElements();
5537	SDLoc dl(N);
5538
5539	// The mask should be widened as well
5540	EVT WideMaskVT = EVT::getVectorVT(Context&: *DAG.getContext(),
5541	VT: MaskVT.getVectorElementType(),
5542	NumElements: WideVT.getVectorNumElements());
5543	Mask = ModifyToType(InOp: Mask, NVT: WideMaskVT, FillWithZeroes: true);
5544
5545	// Widen the Index operand
5546	SDValue Index = N->getIndex();
5547	EVT WideIndexVT = EVT::getVectorVT(Context&: *DAG.getContext(),
5548	VT: Index.getValueType().getScalarType(),
5549	NumElements: NumElts);
5550	Index = ModifyToType(InOp: Index, NVT: WideIndexVT);
5551	SDValue Ops[] = { N->getChain(), PassThru, Mask, N->getBasePtr(), Index,
5552	Scale };
5553
5554	// Widen the MemoryType
5555	EVT WideMemVT = EVT::getVectorVT(Context&: *DAG.getContext(),
5556	VT: N->getMemoryVT().getScalarType(), NumElements: NumElts);
5557	SDValue Res = DAG.getMaskedGather(DAG.getVTList(WideVT, MVT::Other),
5558	WideMemVT, dl, Ops, N->getMemOperand(),
5559	N->getIndexType(), N->getExtensionType());
5560
5561	// Legalize the chain result - switch anything that used the old chain to
5562	// use the new one.
5563	ReplaceValueWith(From: SDValue(N, 1), To: Res.getValue(R: 1));
5564	return Res;
5565	}
5566
5567	SDValue DAGTypeLegalizer::WidenVecRes_VP_GATHER(VPGatherSDNode *N) {
5568	EVT WideVT = TLI.getTypeToTransformTo(Context&: *DAG.getContext(), VT: N->getValueType(ResNo: 0));
5569	SDValue Mask = N->getMask();
5570	SDValue Scale = N->getScale();
5571	ElementCount WideEC = WideVT.getVectorElementCount();
5572	SDLoc dl(N);
5573
5574	SDValue Index = GetWidenedVector(Op: N->getIndex());
5575	EVT WideMemVT = EVT::getVectorVT(Context&: *DAG.getContext(),
5576	VT: N->getMemoryVT().getScalarType(), EC: WideEC);
5577	Mask = GetWidenedMask(Mask, EC: WideEC);
5578
5579	SDValue Ops[] = {N->getChain(), N->getBasePtr(), Index, Scale,
5580	Mask, N->getVectorLength()};
5581	SDValue Res = DAG.getGatherVP(DAG.getVTList(WideVT, MVT::Other), WideMemVT,
5582	dl, Ops, N->getMemOperand(), N->getIndexType());
5583
5584	// Legalize the chain result - switch anything that used the old chain to
5585	// use the new one.
5586	ReplaceValueWith(From: SDValue(N, 1), To: Res.getValue(R: 1));
5587	return Res;
5588	}
5589
5590	SDValue DAGTypeLegalizer::WidenVecRes_ScalarOp(SDNode *N) {
5591	EVT WidenVT = TLI.getTypeToTransformTo(Context&: *DAG.getContext(), VT: N->getValueType(ResNo: 0));
5592	return DAG.getNode(Opcode: N->getOpcode(), DL: SDLoc(N), VT: WidenVT, Operand: N->getOperand(Num: 0));
5593	}
5594
5595	// Return true is this is a SETCC node or a strict version of it.
5596	static inline bool isSETCCOp(unsigned Opcode) {
5597	switch (Opcode) {
5598	case ISD::SETCC:
5599	case ISD::STRICT_FSETCC:
5600	case ISD::STRICT_FSETCCS:
5601	return true;
5602	}
5603	return false;
5604	}
5605
5606	// Return true if this is a node that could have two SETCCs as operands.
5607	static inline bool isLogicalMaskOp(unsigned Opcode) {
5608	switch (Opcode) {
5609	case ISD::AND:
5610	case ISD::OR:
5611	case ISD::XOR:
5612	return true;
5613	}
5614	return false;
5615	}
5616
5617	// If N is a SETCC or a strict variant of it, return the type
5618	// of the compare operands.
5619	static inline EVT getSETCCOperandType(SDValue N) {
5620	unsigned OpNo = N->isStrictFPOpcode() ? 1 : 0;
5621	return N->getOperand(Num: OpNo).getValueType();
5622	}
5623
5624	// This is used just for the assert in convertMask(). Check that this either
5625	// a SETCC or a previously handled SETCC by convertMask().
5626	#ifndef NDEBUG
5627	static inline bool isSETCCorConvertedSETCC(SDValue N) {
5628	if (N.getOpcode() == ISD::EXTRACT_SUBVECTOR)
5629	N = N.getOperand(i: 0);
5630	else if (N.getOpcode() == ISD::CONCAT_VECTORS) {
5631	for (unsigned i = 1; i < N->getNumOperands(); ++i)
5632	if (!N->getOperand(Num: i)->isUndef())
5633	return false;
5634	N = N.getOperand(i: 0);
5635	}
5636
5637	if (N.getOpcode() == ISD::TRUNCATE)
5638	N = N.getOperand(i: 0);
5639	else if (N.getOpcode() == ISD::SIGN_EXTEND)
5640	N = N.getOperand(i: 0);
5641
5642	if (isLogicalMaskOp(Opcode: N.getOpcode()))
5643	return isSETCCorConvertedSETCC(N: N.getOperand(i: 0)) &&
5644	isSETCCorConvertedSETCC(N: N.getOperand(i: 1));
5645
5646	return (isSETCCOp(Opcode: N.getOpcode()) ||
5647	ISD::isBuildVectorOfConstantSDNodes(N: N.getNode()));
5648	}
5649	#endif
5650
5651	// Return a mask of vector type MaskVT to replace InMask. Also adjust MaskVT
5652	// to ToMaskVT if needed with vector extension or truncation.
5653	SDValue DAGTypeLegalizer::convertMask(SDValue InMask, EVT MaskVT,
5654	EVT ToMaskVT) {
5655	// Currently a SETCC or a AND/OR/XOR with two SETCCs are handled.
5656	// FIXME: This code seems to be too restrictive, we might consider
5657	// generalizing it or dropping it.
5658	assert(isSETCCorConvertedSETCC(InMask) && "Unexpected mask argument.");
5659
5660	// Make a new Mask node, with a legal result VT.
5661	SDValue Mask;
5662	SmallVector<SDValue, 4> Ops;
5663	for (unsigned i = 0, e = InMask->getNumOperands(); i < e; ++i)
5664	Ops.push_back(Elt: InMask->getOperand(Num: i));
5665	if (InMask->isStrictFPOpcode()) {
5666	Mask = DAG.getNode(InMask->getOpcode(), SDLoc(InMask),
5667	{ MaskVT, MVT::Other }, Ops);
5668	ReplaceValueWith(From: InMask.getValue(R: 1), To: Mask.getValue(R: 1));
5669	}
5670	else
5671	Mask = DAG.getNode(Opcode: InMask->getOpcode(), DL: SDLoc(InMask), VT: MaskVT, Ops);
5672
5673	// If MaskVT has smaller or bigger elements than ToMaskVT, a vector sign
5674	// extend or truncate is needed.
5675	LLVMContext &Ctx = *DAG.getContext();
5676	unsigned MaskScalarBits = MaskVT.getScalarSizeInBits();
5677	unsigned ToMaskScalBits = ToMaskVT.getScalarSizeInBits();
5678	if (MaskScalarBits < ToMaskScalBits) {
5679	EVT ExtVT = EVT::getVectorVT(Context&: Ctx, VT: ToMaskVT.getVectorElementType(),
5680	NumElements: MaskVT.getVectorNumElements());
5681	Mask = DAG.getNode(Opcode: ISD::SIGN_EXTEND, DL: SDLoc(Mask), VT: ExtVT, Operand: Mask);
5682	} else if (MaskScalarBits > ToMaskScalBits) {
5683	EVT TruncVT = EVT::getVectorVT(Context&: Ctx, VT: ToMaskVT.getVectorElementType(),
5684	NumElements: MaskVT.getVectorNumElements());
5685	Mask = DAG.getNode(Opcode: ISD::TRUNCATE, DL: SDLoc(Mask), VT: TruncVT, Operand: Mask);
5686	}
5687
5688	assert(Mask->getValueType(0).getScalarSizeInBits() ==
5689	ToMaskVT.getScalarSizeInBits() &&
5690	"Mask should have the right element size by now.");
5691
5692	// Adjust Mask to the right number of elements.
5693	unsigned CurrMaskNumEls = Mask->getValueType(ResNo: 0).getVectorNumElements();
5694	if (CurrMaskNumEls > ToMaskVT.getVectorNumElements()) {
5695	SDValue ZeroIdx = DAG.getVectorIdxConstant(Val: 0, DL: SDLoc(Mask));
5696	Mask = DAG.getNode(Opcode: ISD::EXTRACT_SUBVECTOR, DL: SDLoc(Mask), VT: ToMaskVT, N1: Mask,
5697	N2: ZeroIdx);
5698	} else if (CurrMaskNumEls < ToMaskVT.getVectorNumElements()) {
5699	unsigned NumSubVecs = (ToMaskVT.getVectorNumElements() / CurrMaskNumEls);
5700	EVT SubVT = Mask->getValueType(ResNo: 0);
5701	SmallVector<SDValue, 16> SubOps(NumSubVecs, DAG.getUNDEF(VT: SubVT));
5702	SubOps[0] = Mask;
5703	Mask = DAG.getNode(Opcode: ISD::CONCAT_VECTORS, DL: SDLoc(Mask), VT: ToMaskVT, Ops: SubOps);
5704	}
5705
5706	assert((Mask->getValueType(0) == ToMaskVT) &&
5707	"A mask of ToMaskVT should have been produced by now.");
5708
5709	return Mask;
5710	}
5711
5712	// This method tries to handle some special cases for the vselect mask
5713	// and if needed adjusting the mask vector type to match that of the VSELECT.
5714	// Without it, many cases end up with scalarization of the SETCC, with many
5715	// unnecessary instructions.
5716	SDValue DAGTypeLegalizer::WidenVSELECTMask(SDNode *N) {
5717	LLVMContext &Ctx = *DAG.getContext();
5718	SDValue Cond = N->getOperand(Num: 0);
5719
5720	if (N->getOpcode() != ISD::VSELECT)
5721	return SDValue();
5722
5723	if (!isSETCCOp(Opcode: Cond->getOpcode()) && !isLogicalMaskOp(Opcode: Cond->getOpcode()))
5724	return SDValue();
5725
5726	// If this is a splitted VSELECT that was previously already handled, do
5727	// nothing.
5728	EVT CondVT = Cond->getValueType(ResNo: 0);
5729	if (CondVT.getScalarSizeInBits() != 1)
5730	return SDValue();
5731
5732	EVT VSelVT = N->getValueType(ResNo: 0);
5733
5734	// This method can't handle scalable vector types.
5735	// FIXME: This support could be added in the future.
5736	if (VSelVT.isScalableVector())
5737	return SDValue();
5738
5739	// Only handle vector types which are a power of 2.
5740	if (!isPowerOf2_64(Value: VSelVT.getSizeInBits()))
5741	return SDValue();
5742
5743	// Don't touch if this will be scalarized.
5744	EVT FinalVT = VSelVT;
5745	while (getTypeAction(VT: FinalVT) == TargetLowering::TypeSplitVector)
5746	FinalVT = FinalVT.getHalfNumVectorElementsVT(Context&: Ctx);
5747
5748	if (FinalVT.getVectorNumElements() == 1)
5749	return SDValue();
5750
5751	// If there is support for an i1 vector mask, don't touch.
5752	if (isSETCCOp(Opcode: Cond.getOpcode())) {
5753	EVT SetCCOpVT = getSETCCOperandType(N: Cond);
5754	while (TLI.getTypeAction(Context&: Ctx, VT: SetCCOpVT) != TargetLowering::TypeLegal)
5755	SetCCOpVT = TLI.getTypeToTransformTo(Context&: Ctx, VT: SetCCOpVT);
5756	EVT SetCCResVT = getSetCCResultType(VT: SetCCOpVT);
5757	if (SetCCResVT.getScalarSizeInBits() == 1)
5758	return SDValue();
5759	} else if (CondVT.getScalarType() == MVT::i1) {
5760	// If there is support for an i1 vector mask (or only scalar i1 conditions),
5761	// don't touch.
5762	while (TLI.getTypeAction(Context&: Ctx, VT: CondVT) != TargetLowering::TypeLegal)
5763	CondVT = TLI.getTypeToTransformTo(Context&: Ctx, VT: CondVT);
5764
5765	if (CondVT.getScalarType() == MVT::i1)
5766	return SDValue();
5767	}
5768
5769	// Widen the vselect result type if needed.
5770	if (getTypeAction(VT: VSelVT) == TargetLowering::TypeWidenVector)
5771	VSelVT = TLI.getTypeToTransformTo(Context&: Ctx, VT: VSelVT);
5772
5773	// The mask of the VSELECT should have integer elements.
5774	EVT ToMaskVT = VSelVT;
5775	if (!ToMaskVT.getScalarType().isInteger())
5776	ToMaskVT = ToMaskVT.changeVectorElementTypeToInteger();
5777
5778	SDValue Mask;
5779	if (isSETCCOp(Opcode: Cond->getOpcode())) {
5780	EVT MaskVT = getSetCCResultType(VT: getSETCCOperandType(N: Cond));
5781	Mask = convertMask(InMask: Cond, MaskVT, ToMaskVT);
5782	} else if (isLogicalMaskOp(Opcode: Cond->getOpcode()) &&
5783	isSETCCOp(Opcode: Cond->getOperand(Num: 0).getOpcode()) &&
5784	isSETCCOp(Opcode: Cond->getOperand(Num: 1).getOpcode())) {
5785	// Cond is (AND/OR/XOR (SETCC, SETCC))
5786	SDValue SETCC0 = Cond->getOperand(Num: 0);
5787	SDValue SETCC1 = Cond->getOperand(Num: 1);
5788	EVT VT0 = getSetCCResultType(VT: getSETCCOperandType(N: SETCC0));
5789	EVT VT1 = getSetCCResultType(VT: getSETCCOperandType(N: SETCC1));
5790	unsigned ScalarBits0 = VT0.getScalarSizeInBits();
5791	unsigned ScalarBits1 = VT1.getScalarSizeInBits();
5792	unsigned ScalarBits_ToMask = ToMaskVT.getScalarSizeInBits();
5793	EVT MaskVT;
5794	// If the two SETCCs have different VTs, either extend/truncate one of
5795	// them to the other "towards" ToMaskVT, or truncate one and extend the
5796	// other to ToMaskVT.
5797	if (ScalarBits0 != ScalarBits1) {
5798	EVT NarrowVT = ((ScalarBits0 < ScalarBits1) ? VT0 : VT1);
5799	EVT WideVT = ((NarrowVT == VT0) ? VT1 : VT0);
5800	if (ScalarBits_ToMask >= WideVT.getScalarSizeInBits())
5801	MaskVT = WideVT;
5802	else if (ScalarBits_ToMask <= NarrowVT.getScalarSizeInBits())
5803	MaskVT = NarrowVT;
5804	else
5805	MaskVT = ToMaskVT;
5806	} else
5807	// If the two SETCCs have the same VT, don't change it.
5808	MaskVT = VT0;
5809
5810	// Make new SETCCs and logical nodes.
5811	SETCC0 = convertMask(InMask: SETCC0, MaskVT: VT0, ToMaskVT: MaskVT);
5812	SETCC1 = convertMask(InMask: SETCC1, MaskVT: VT1, ToMaskVT: MaskVT);
5813	Cond = DAG.getNode(Opcode: Cond->getOpcode(), DL: SDLoc(Cond), VT: MaskVT, N1: SETCC0, N2: SETCC1);
5814
5815	// Convert the logical op for VSELECT if needed.
5816	Mask = convertMask(InMask: Cond, MaskVT, ToMaskVT);
5817	} else
5818	return SDValue();
5819
5820	return Mask;
5821	}
5822
5823	SDValue DAGTypeLegalizer::WidenVecRes_Select(SDNode *N) {
5824	EVT WidenVT = TLI.getTypeToTransformTo(Context&: *DAG.getContext(), VT: N->getValueType(ResNo: 0));
5825	ElementCount WidenEC = WidenVT.getVectorElementCount();
5826
5827	SDValue Cond1 = N->getOperand(Num: 0);
5828	EVT CondVT = Cond1.getValueType();
5829	unsigned Opcode = N->getOpcode();
5830	if (CondVT.isVector()) {
5831	if (SDValue WideCond = WidenVSELECTMask(N)) {
5832	SDValue InOp1 = GetWidenedVector(Op: N->getOperand(Num: 1));
5833	SDValue InOp2 = GetWidenedVector(Op: N->getOperand(Num: 2));
5834	assert(InOp1.getValueType() == WidenVT && InOp2.getValueType() == WidenVT);
5835	return DAG.getNode(Opcode, DL: SDLoc(N), VT: WidenVT, N1: WideCond, N2: InOp1, N3: InOp2);
5836	}
5837
5838	EVT CondEltVT = CondVT.getVectorElementType();
5839	EVT CondWidenVT = EVT::getVectorVT(Context&: *DAG.getContext(), VT: CondEltVT, EC: WidenEC);
5840	if (getTypeAction(VT: CondVT) == TargetLowering::TypeWidenVector)
5841	Cond1 = GetWidenedVector(Op: Cond1);
5842
5843	// If we have to split the condition there is no point in widening the
5844	// select. This would result in an cycle of widening the select ->
5845	// widening the condition operand -> splitting the condition operand ->
5846	// splitting the select -> widening the select. Instead split this select
5847	// further and widen the resulting type.
5848	if (getTypeAction(VT: CondVT) == TargetLowering::TypeSplitVector) {
5849	SDValue SplitSelect = SplitVecOp_VSELECT(N, OpNo: 0);
5850	SDValue Res = ModifyToType(InOp: SplitSelect, NVT: WidenVT);
5851	return Res;
5852	}
5853
5854	if (Cond1.getValueType() != CondWidenVT)
5855	Cond1 = ModifyToType(InOp: Cond1, NVT: CondWidenVT);
5856	}
5857
5858	SDValue InOp1 = GetWidenedVector(Op: N->getOperand(Num: 1));
5859	SDValue InOp2 = GetWidenedVector(Op: N->getOperand(Num: 2));
5860	assert(InOp1.getValueType() == WidenVT && InOp2.getValueType() == WidenVT);
5861	if (Opcode == ISD::VP_SELECT || Opcode == ISD::VP_MERGE)
5862	return DAG.getNode(Opcode, DL: SDLoc(N), VT: WidenVT, N1: Cond1, N2: InOp1, N3: InOp2,
5863	N4: N->getOperand(Num: 3));
5864	return DAG.getNode(Opcode, DL: SDLoc(N), VT: WidenVT, N1: Cond1, N2: InOp1, N3: InOp2);
5865	}
5866
5867	SDValue DAGTypeLegalizer::WidenVecRes_SELECT_CC(SDNode *N) {
5868	SDValue InOp1 = GetWidenedVector(Op: N->getOperand(Num: 2));
5869	SDValue InOp2 = GetWidenedVector(Op: N->getOperand(Num: 3));
5870	return DAG.getNode(Opcode: ISD::SELECT_CC, DL: SDLoc(N),
5871	VT: InOp1.getValueType(), N1: N->getOperand(Num: 0),
5872	N2: N->getOperand(Num: 1), N3: InOp1, N4: InOp2, N5: N->getOperand(Num: 4));
5873	}
5874
5875	SDValue DAGTypeLegalizer::WidenVecRes_UNDEF(SDNode *N) {
5876	EVT WidenVT = TLI.getTypeToTransformTo(Context&: *DAG.getContext(), VT: N->getValueType(ResNo: 0));
5877	return DAG.getUNDEF(VT: WidenVT);
5878	}
5879
5880	SDValue DAGTypeLegalizer::WidenVecRes_VECTOR_SHUFFLE(ShuffleVectorSDNode *N) {
5881	EVT VT = N->getValueType(ResNo: 0);
5882	SDLoc dl(N);
5883
5884	EVT WidenVT = TLI.getTypeToTransformTo(Context&: *DAG.getContext(), VT);
5885	unsigned NumElts = VT.getVectorNumElements();
5886	unsigned WidenNumElts = WidenVT.getVectorNumElements();
5887
5888	SDValue InOp1 = GetWidenedVector(Op: N->getOperand(Num: 0));
5889	SDValue InOp2 = GetWidenedVector(Op: N->getOperand(Num: 1));
5890
5891	// Adjust mask based on new input vector length.
5892	SmallVector<int, 16> NewMask;
5893	for (unsigned i = 0; i != NumElts; ++i) {
5894	int Idx = N->getMaskElt(Idx: i);
5895	if (Idx < (int)NumElts)
5896	NewMask.push_back(Elt: Idx);
5897	else
5898	NewMask.push_back(Elt: Idx - NumElts + WidenNumElts);
5899	}
5900	for (unsigned i = NumElts; i != WidenNumElts; ++i)
5901	NewMask.push_back(Elt: -1);
5902	return DAG.getVectorShuffle(VT: WidenVT, dl, N1: InOp1, N2: InOp2, Mask: NewMask);
5903	}
5904
5905	SDValue DAGTypeLegalizer::WidenVecRes_VECTOR_REVERSE(SDNode *N) {
5906	EVT VT = N->getValueType(ResNo: 0);
5907	EVT EltVT = VT.getVectorElementType();
5908	SDLoc dl(N);
5909
5910	EVT WidenVT = TLI.getTypeToTransformTo(Context&: *DAG.getContext(), VT);
5911	SDValue OpValue = GetWidenedVector(Op: N->getOperand(Num: 0));
5912	assert(WidenVT == OpValue.getValueType() && "Unexpected widened vector type");
5913
5914	SDValue ReverseVal = DAG.getNode(Opcode: ISD::VECTOR_REVERSE, DL: dl, VT: WidenVT, Operand: OpValue);
5915	unsigned WidenNumElts = WidenVT.getVectorMinNumElements();
5916	unsigned VTNumElts = VT.getVectorMinNumElements();
5917	unsigned IdxVal = WidenNumElts - VTNumElts;
5918
5919	if (VT.isScalableVector()) {
5920	// Try to split the 'Widen ReverseVal' into smaller extracts and concat the
5921	// results together, e.g.(nxv6i64 -> nxv8i64)
5922	// nxv8i64 vector_reverse
5923	// <->
5924	// nxv8i64 concat(
5925	// nxv2i64 extract_subvector(nxv8i64, 2)
5926	// nxv2i64 extract_subvector(nxv8i64, 4)
5927	// nxv2i64 extract_subvector(nxv8i64, 6)
5928	// nxv2i64 undef)
5929
5930	unsigned GCD = std::gcd(m: VTNumElts, n: WidenNumElts);
5931	EVT PartVT = EVT::getVectorVT(Context&: *DAG.getContext(), VT: EltVT,
5932	EC: ElementCount::getScalable(MinVal: GCD));
5933	assert((IdxVal % GCD) == 0 && "Expected Idx to be a multiple of the broken "
5934	"down type's element count");
5935	SmallVector<SDValue> Parts;
5936	unsigned i = 0;
5937	for (; i < VTNumElts / GCD; ++i)
5938	Parts.push_back(
5939	Elt: DAG.getNode(Opcode: ISD::EXTRACT_SUBVECTOR, DL: dl, VT: PartVT, N1: ReverseVal,
5940	N2: DAG.getVectorIdxConstant(Val: IdxVal + i * GCD, DL: dl)));
5941	for (; i < WidenNumElts / GCD; ++i)
5942	Parts.push_back(Elt: DAG.getUNDEF(VT: PartVT));
5943
5944	return DAG.getNode(Opcode: ISD::CONCAT_VECTORS, DL: dl, VT: WidenVT, Ops: Parts);
5945	}
5946
5947	// Use VECTOR_SHUFFLE to combine new vector from 'ReverseVal' for
5948	// fixed-vectors.
5949	SmallVector<int, 16> Mask;
5950	for (unsigned i = 0; i != VTNumElts; ++i) {
5951	Mask.push_back(Elt: IdxVal + i);
5952	}
5953	for (unsigned i = VTNumElts; i != WidenNumElts; ++i)
5954	Mask.push_back(Elt: -1);
5955
5956	return DAG.getVectorShuffle(VT: WidenVT, dl, N1: ReverseVal, N2: DAG.getUNDEF(VT: WidenVT),
5957	Mask);
5958	}
5959
5960	SDValue DAGTypeLegalizer::WidenVecRes_SETCC(SDNode *N) {
5961	assert(N->getValueType(0).isVector() &&
5962	N->getOperand(0).getValueType().isVector() &&
5963	"Operands must be vectors");
5964	EVT WidenVT = TLI.getTypeToTransformTo(Context&: *DAG.getContext(), VT: N->getValueType(ResNo: 0));
5965	ElementCount WidenEC = WidenVT.getVectorElementCount();
5966
5967	SDValue InOp1 = N->getOperand(Num: 0);
5968	EVT InVT = InOp1.getValueType();
5969	assert(InVT.isVector() && "can not widen non-vector type");
5970	EVT WidenInVT =
5971	EVT::getVectorVT(Context&: *DAG.getContext(), VT: InVT.getVectorElementType(), EC: WidenEC);
5972
5973	// The input and output types often differ here, and it could be that while
5974	// we'd prefer to widen the result type, the input operands have been split.
5975	// In this case, we also need to split the result of this node as well.
5976	if (getTypeAction(VT: InVT) == TargetLowering::TypeSplitVector) {
5977	SDValue SplitVSetCC = SplitVecOp_VSETCC(N);
5978	SDValue Res = ModifyToType(InOp: SplitVSetCC, NVT: WidenVT);
5979	return Res;
5980	}
5981
5982	// If the inputs also widen, handle them directly. Otherwise widen by hand.
5983	SDValue InOp2 = N->getOperand(Num: 1);
5984	if (getTypeAction(VT: InVT) == TargetLowering::TypeWidenVector) {
5985	InOp1 = GetWidenedVector(Op: InOp1);
5986	InOp2 = GetWidenedVector(Op: InOp2);
5987	} else {
5988	InOp1 = DAG.WidenVector(N: InOp1, DL: SDLoc(N));
5989	InOp2 = DAG.WidenVector(N: InOp2, DL: SDLoc(N));
5990	}
5991
5992	// Assume that the input and output will be widen appropriately. If not,
5993	// we will have to unroll it at some point.
5994	assert(InOp1.getValueType() == WidenInVT &&
5995	InOp2.getValueType() == WidenInVT &&
5996	"Input not widened to expected type!");
5997	(void)WidenInVT;
5998	if (N->getOpcode() == ISD::VP_SETCC) {
5999	SDValue Mask =
6000	GetWidenedMask(Mask: N->getOperand(Num: 3), EC: WidenVT.getVectorElementCount());
6001	return DAG.getNode(Opcode: ISD::VP_SETCC, DL: SDLoc(N), VT: WidenVT, N1: InOp1, N2: InOp2,
6002	N3: N->getOperand(Num: 2), N4: Mask, N5: N->getOperand(Num: 4));
6003	}
6004	return DAG.getNode(Opcode: ISD::SETCC, DL: SDLoc(N), VT: WidenVT, N1: InOp1, N2: InOp2,
6005	N3: N->getOperand(Num: 2));
6006	}
6007
6008	SDValue DAGTypeLegalizer::WidenVecRes_STRICT_FSETCC(SDNode *N) {
6009	assert(N->getValueType(0).isVector() &&
6010	N->getOperand(1).getValueType().isVector() &&
6011	"Operands must be vectors");
6012	EVT VT = N->getValueType(ResNo: 0);
6013	EVT WidenVT = TLI.getTypeToTransformTo(Context&: *DAG.getContext(), VT);
6014	unsigned WidenNumElts = WidenVT.getVectorNumElements();
6015	unsigned NumElts = VT.getVectorNumElements();
6016	EVT EltVT = VT.getVectorElementType();
6017
6018	SDLoc dl(N);
6019	SDValue Chain = N->getOperand(Num: 0);
6020	SDValue LHS = N->getOperand(Num: 1);
6021	SDValue RHS = N->getOperand(Num: 2);
6022	SDValue CC = N->getOperand(Num: 3);
6023	EVT TmpEltVT = LHS.getValueType().getVectorElementType();
6024
6025	// Fully unroll and reassemble.
6026	SmallVector<SDValue, 8> Scalars(WidenNumElts, DAG.getUNDEF(VT: EltVT));
6027	SmallVector<SDValue, 8> Chains(NumElts);
6028	for (unsigned i = 0; i != NumElts; ++i) {
6029	SDValue LHSElem = DAG.getNode(Opcode: ISD::EXTRACT_VECTOR_ELT, DL: dl, VT: TmpEltVT, N1: LHS,
6030	N2: DAG.getVectorIdxConstant(Val: i, DL: dl));
6031	SDValue RHSElem = DAG.getNode(Opcode: ISD::EXTRACT_VECTOR_ELT, DL: dl, VT: TmpEltVT, N1: RHS,
6032	N2: DAG.getVectorIdxConstant(Val: i, DL: dl));
6033
6034	Scalars[i] = DAG.getNode(N->getOpcode(), dl, {MVT::i1, MVT::Other},
6035	{Chain, LHSElem, RHSElem, CC});
6036	Chains[i] = Scalars[i].getValue(R: 1);
6037	Scalars[i] = DAG.getSelect(DL: dl, VT: EltVT, Cond: Scalars[i],
6038	LHS: DAG.getBoolConstant(V: true, DL: dl, VT: EltVT, OpVT: VT),
6039	RHS: DAG.getBoolConstant(V: false, DL: dl, VT: EltVT, OpVT: VT));
6040	}
6041
6042	SDValue NewChain = DAG.getNode(ISD::TokenFactor, dl, MVT::Other, Chains);
6043	ReplaceValueWith(From: SDValue(N, 1), To: NewChain);
6044
6045	return DAG.getBuildVector(VT: WidenVT, DL: dl, Ops: Scalars);
6046	}
6047
6048	//===----------------------------------------------------------------------===//
6049	// Widen Vector Operand
6050	//===----------------------------------------------------------------------===//
6051	bool DAGTypeLegalizer::WidenVectorOperand(SDNode *N, unsigned OpNo) {
6052	LLVM_DEBUG(dbgs() << "Widen node operand " << OpNo << ": "; N->dump(&DAG));
6053	SDValue Res = SDValue();
6054
6055	// See if the target wants to custom widen this node.
6056	if (CustomLowerNode(N, VT: N->getOperand(Num: OpNo).getValueType(), LegalizeResult: false))
6057	return false;
6058
6059	switch (N->getOpcode()) {
6060	default:
6061	#ifndef NDEBUG
6062	dbgs() << "WidenVectorOperand op #" << OpNo << ": ";
6063	N->dump(G: &DAG);
6064	dbgs() << "\n";
6065	#endif
6066	report_fatal_error(reason: "Do not know how to widen this operator's operand!");
6067
6068	case ISD::BITCAST: Res = WidenVecOp_BITCAST(N); break;
6069	case ISD::CONCAT_VECTORS: Res = WidenVecOp_CONCAT_VECTORS(N); break;
6070	case ISD::INSERT_SUBVECTOR: Res = WidenVecOp_INSERT_SUBVECTOR(N); break;
6071	case ISD::EXTRACT_SUBVECTOR: Res = WidenVecOp_EXTRACT_SUBVECTOR(N); break;
6072	case ISD::EXTRACT_VECTOR_ELT: Res = WidenVecOp_EXTRACT_VECTOR_ELT(N); break;
6073	case ISD::STORE: Res = WidenVecOp_STORE(N); break;
6074	case ISD::VP_STORE: Res = WidenVecOp_VP_STORE(N, OpNo); break;
6075	case ISD::EXPERIMENTAL_VP_STRIDED_STORE:
6076	Res = WidenVecOp_VP_STRIDED_STORE(N, OpNo);
6077	break;
6078	case ISD::ANY_EXTEND_VECTOR_INREG:
6079	case ISD::SIGN_EXTEND_VECTOR_INREG:
6080	case ISD::ZERO_EXTEND_VECTOR_INREG:
6081	Res = WidenVecOp_EXTEND_VECTOR_INREG(N);
6082	break;
6083	case ISD::MSTORE: Res = WidenVecOp_MSTORE(N, OpNo); break;
6084	case ISD::MGATHER: Res = WidenVecOp_MGATHER(N, OpNo); break;
6085	case ISD::MSCATTER: Res = WidenVecOp_MSCATTER(N, OpNo); break;
6086	case ISD::VP_SCATTER: Res = WidenVecOp_VP_SCATTER(N, OpNo); break;
6087	case ISD::SETCC: Res = WidenVecOp_SETCC(N); break;
6088	case ISD::STRICT_FSETCC:
6089	case ISD::STRICT_FSETCCS: Res = WidenVecOp_STRICT_FSETCC(N); break;
6090	case ISD::VSELECT: Res = WidenVecOp_VSELECT(N); break;
6091	case ISD::FLDEXP:
6092	case ISD::FCOPYSIGN:
6093	case ISD::LRINT:
6094	case ISD::LLRINT:
6095	Res = WidenVecOp_UnrollVectorOp(N);
6096	break;
6097	case ISD::IS_FPCLASS: Res = WidenVecOp_IS_FPCLASS(N); break;
6098
6099	case ISD::ANY_EXTEND:
6100	case ISD::SIGN_EXTEND:
6101	case ISD::ZERO_EXTEND:
6102	Res = WidenVecOp_EXTEND(N);
6103	break;
6104
6105	case ISD::FP_EXTEND:
6106	case ISD::STRICT_FP_EXTEND:
6107	case ISD::FP_ROUND:
6108	case ISD::STRICT_FP_ROUND:
6109	case ISD::FP_TO_SINT:
6110	case ISD::STRICT_FP_TO_SINT:
6111	case ISD::FP_TO_UINT:
6112	case ISD::STRICT_FP_TO_UINT:
6113	case ISD::SINT_TO_FP:
6114	case ISD::STRICT_SINT_TO_FP:
6115	case ISD::UINT_TO_FP:
6116	case ISD::STRICT_UINT_TO_FP:
6117	case ISD::TRUNCATE:
6118	Res = WidenVecOp_Convert(N);
6119	break;
6120
6121	case ISD::FP_TO_SINT_SAT:
6122	case ISD::FP_TO_UINT_SAT:
6123	Res = WidenVecOp_FP_TO_XINT_SAT(N);
6124	break;
6125
6126	case ISD::VECREDUCE_FADD:
6127	case ISD::VECREDUCE_FMUL:
6128	case ISD::VECREDUCE_ADD:
6129	case ISD::VECREDUCE_MUL:
6130	case ISD::VECREDUCE_AND:
6131	case ISD::VECREDUCE_OR:
6132	case ISD::VECREDUCE_XOR:
6133	case ISD::VECREDUCE_SMAX:
6134	case ISD::VECREDUCE_SMIN:
6135	case ISD::VECREDUCE_UMAX:
6136	case ISD::VECREDUCE_UMIN:
6137	case ISD::VECREDUCE_FMAX:
6138	case ISD::VECREDUCE_FMIN:
6139	case ISD::VECREDUCE_FMAXIMUM:
6140	case ISD::VECREDUCE_FMINIMUM:
6141	Res = WidenVecOp_VECREDUCE(N);
6142	break;
6143	case ISD::VECREDUCE_SEQ_FADD:
6144	case ISD::VECREDUCE_SEQ_FMUL:
6145	Res = WidenVecOp_VECREDUCE_SEQ(N);
6146	break;
6147	case ISD::VP_REDUCE_FADD:
6148	case ISD::VP_REDUCE_SEQ_FADD:
6149	case ISD::VP_REDUCE_FMUL:
6150	case ISD::VP_REDUCE_SEQ_FMUL:
6151	case ISD::VP_REDUCE_ADD:
6152	case ISD::VP_REDUCE_MUL:
6153	case ISD::VP_REDUCE_AND:
6154	case ISD::VP_REDUCE_OR:
6155	case ISD::VP_REDUCE_XOR:
6156	case ISD::VP_REDUCE_SMAX:
6157	case ISD::VP_REDUCE_SMIN:
6158	case ISD::VP_REDUCE_UMAX:
6159	case ISD::VP_REDUCE_UMIN:
6160	case ISD::VP_REDUCE_FMAX:
6161	case ISD::VP_REDUCE_FMIN:
6162	Res = WidenVecOp_VP_REDUCE(N);
6163	break;
6164	}
6165
6166	// If Res is null, the sub-method took care of registering the result.
6167	if (!Res.getNode()) return false;
6168
6169	// If the result is N, the sub-method updated N in place. Tell the legalizer
6170	// core about this.
6171	if (Res.getNode() == N)
6172	return true;
6173
6174
6175	if (N->isStrictFPOpcode())
6176	assert(Res.getValueType() == N->getValueType(0) && N->getNumValues() == 2 &&
6177	"Invalid operand expansion");
6178	else
6179	assert(Res.getValueType() == N->getValueType(0) && N->getNumValues() == 1 &&
6180	"Invalid operand expansion");
6181
6182	ReplaceValueWith(From: SDValue(N, 0), To: Res);
6183	return false;
6184	}
6185
6186	SDValue DAGTypeLegalizer::WidenVecOp_EXTEND(SDNode *N) {
6187	SDLoc DL(N);
6188	EVT VT = N->getValueType(ResNo: 0);
6189
6190	SDValue InOp = N->getOperand(Num: 0);
6191	assert(getTypeAction(InOp.getValueType()) ==
6192	TargetLowering::TypeWidenVector &&
6193	"Unexpected type action");
6194	InOp = GetWidenedVector(Op: InOp);
6195	assert(VT.getVectorNumElements() <
6196	InOp.getValueType().getVectorNumElements() &&
6197	"Input wasn't widened!");
6198
6199	// We may need to further widen the operand until it has the same total
6200	// vector size as the result.
6201	EVT InVT = InOp.getValueType();
6202	if (InVT.getSizeInBits() != VT.getSizeInBits()) {
6203	EVT InEltVT = InVT.getVectorElementType();
6204	for (EVT FixedVT : MVT::vector_valuetypes()) {
6205	EVT FixedEltVT = FixedVT.getVectorElementType();
6206	if (TLI.isTypeLegal(FixedVT) &&
6207	FixedVT.getSizeInBits() == VT.getSizeInBits() &&
6208	FixedEltVT == InEltVT) {
6209	assert(FixedVT.getVectorNumElements() >= VT.getVectorNumElements() &&
6210	"Not enough elements in the fixed type for the operand!");
6211	assert(FixedVT.getVectorNumElements() != InVT.getVectorNumElements() &&
6212	"We can't have the same type as we started with!");
6213	if (FixedVT.getVectorNumElements() > InVT.getVectorNumElements())
6214	InOp = DAG.getNode(ISD::INSERT_SUBVECTOR, DL, FixedVT,
6215	DAG.getUNDEF(FixedVT), InOp,
6216	DAG.getVectorIdxConstant(0, DL));
6217	else
6218	InOp = DAG.getNode(ISD::EXTRACT_SUBVECTOR, DL, FixedVT, InOp,
6219	DAG.getVectorIdxConstant(0, DL));
6220	break;
6221	}
6222	}
6223	InVT = InOp.getValueType();
6224	if (InVT.getSizeInBits() != VT.getSizeInBits())
6225	// We couldn't find a legal vector type that was a widening of the input
6226	// and could be extended in-register to the result type, so we have to
6227	// scalarize.
6228	return WidenVecOp_Convert(N);
6229	}
6230
6231	// Use special DAG nodes to represent the operation of extending the
6232	// low lanes.
6233	switch (N->getOpcode()) {
6234	default:
6235	llvm_unreachable("Extend legalization on extend operation!");
6236	case ISD::ANY_EXTEND:
6237	return DAG.getNode(Opcode: ISD::ANY_EXTEND_VECTOR_INREG, DL, VT, Operand: InOp);
6238	case ISD::SIGN_EXTEND:
6239	return DAG.getNode(Opcode: ISD::SIGN_EXTEND_VECTOR_INREG, DL, VT, Operand: InOp);
6240	case ISD::ZERO_EXTEND:
6241	return DAG.getNode(Opcode: ISD::ZERO_EXTEND_VECTOR_INREG, DL, VT, Operand: InOp);
6242	}
6243	}
6244
6245	SDValue DAGTypeLegalizer::WidenVecOp_UnrollVectorOp(SDNode *N) {
6246	// The result (and first input) is legal, but the second input is illegal.
6247	// We can't do much to fix that, so just unroll and let the extracts off of
6248	// the second input be widened as needed later.
6249	return DAG.UnrollVectorOp(N);
6250	}
6251
6252	SDValue DAGTypeLegalizer::WidenVecOp_IS_FPCLASS(SDNode *N) {
6253	SDLoc DL(N);
6254	EVT ResultVT = N->getValueType(ResNo: 0);
6255	SDValue Test = N->getOperand(Num: 1);
6256	SDValue WideArg = GetWidenedVector(Op: N->getOperand(Num: 0));
6257
6258	// Process this node similarly to SETCC.
6259	EVT WideResultVT = getSetCCResultType(VT: WideArg.getValueType());
6260	if (ResultVT.getScalarType() == MVT::i1)
6261	WideResultVT = EVT::getVectorVT(*DAG.getContext(), MVT::i1,
6262	WideResultVT.getVectorNumElements());
6263
6264	SDValue WideNode = DAG.getNode(Opcode: ISD::IS_FPCLASS, DL, VT: WideResultVT,
6265	Ops: {WideArg, Test}, Flags: N->getFlags());
6266
6267	// Extract the needed results from the result vector.
6268	EVT ResVT =
6269	EVT::getVectorVT(Context&: *DAG.getContext(), VT: WideResultVT.getVectorElementType(),
6270	NumElements: ResultVT.getVectorNumElements());
6271	SDValue CC = DAG.getNode(Opcode: ISD::EXTRACT_SUBVECTOR, DL, VT: ResVT, N1: WideNode,
6272	N2: DAG.getVectorIdxConstant(Val: 0, DL));
6273
6274	EVT OpVT = N->getOperand(Num: 0).getValueType();
6275	ISD::NodeType ExtendCode =
6276	TargetLowering::getExtendForContent(Content: TLI.getBooleanContents(Type: OpVT));
6277	return DAG.getNode(Opcode: ExtendCode, DL, VT: ResultVT, Operand: CC);
6278	}
6279
6280	SDValue DAGTypeLegalizer::WidenVecOp_Convert(SDNode *N) {
6281	// Since the result is legal and the input is illegal.
6282	EVT VT = N->getValueType(ResNo: 0);
6283	EVT EltVT = VT.getVectorElementType();
6284	SDLoc dl(N);
6285	SDValue InOp = N->getOperand(Num: N->isStrictFPOpcode() ? 1 : 0);
6286	assert(getTypeAction(InOp.getValueType()) ==
6287	TargetLowering::TypeWidenVector &&
6288	"Unexpected type action");
6289	InOp = GetWidenedVector(Op: InOp);
6290	EVT InVT = InOp.getValueType();
6291	unsigned Opcode = N->getOpcode();
6292
6293	// See if a widened result type would be legal, if so widen the node.
6294	// FIXME: This isn't safe for StrictFP. Other optimization here is needed.
6295	EVT WideVT = EVT::getVectorVT(Context&: *DAG.getContext(), VT: EltVT,
6296	EC: InVT.getVectorElementCount());
6297	if (TLI.isTypeLegal(VT: WideVT) && !N->isStrictFPOpcode()) {
6298	SDValue Res;
6299	if (N->isStrictFPOpcode()) {
6300	if (Opcode == ISD::STRICT_FP_ROUND)
6301	Res = DAG.getNode(Opcode, dl, { WideVT, MVT::Other },
6302	{ N->getOperand(0), InOp, N->getOperand(2) });
6303	else
6304	Res = DAG.getNode(Opcode, dl, { WideVT, MVT::Other },
6305	{ N->getOperand(0), InOp });
6306	// Legalize the chain result - switch anything that used the old chain to
6307	// use the new one.
6308	ReplaceValueWith(From: SDValue(N, 1), To: Res.getValue(R: 1));
6309	} else {
6310	if (Opcode == ISD::FP_ROUND)
6311	Res = DAG.getNode(Opcode, DL: dl, VT: WideVT, N1: InOp, N2: N->getOperand(Num: 1));
6312	else
6313	Res = DAG.getNode(Opcode, DL: dl, VT: WideVT, Operand: InOp);
6314	}
6315	return DAG.getNode(Opcode: ISD::EXTRACT_SUBVECTOR, DL: dl, VT, N1: Res,
6316	N2: DAG.getVectorIdxConstant(Val: 0, DL: dl));
6317	}
6318
6319	EVT InEltVT = InVT.getVectorElementType();
6320
6321	// Unroll the convert into some scalar code and create a nasty build vector.
6322	unsigned NumElts = VT.getVectorNumElements();
6323	SmallVector<SDValue, 16> Ops(NumElts);
6324	if (N->isStrictFPOpcode()) {
6325	SmallVector<SDValue, 4> NewOps(N->op_begin(), N->op_end());
6326	SmallVector<SDValue, 32> OpChains;
6327	for (unsigned i=0; i < NumElts; ++i) {
6328	NewOps[1] = DAG.getNode(Opcode: ISD::EXTRACT_VECTOR_ELT, DL: dl, VT: InEltVT, N1: InOp,
6329	N2: DAG.getVectorIdxConstant(Val: i, DL: dl));
6330	Ops[i] = DAG.getNode(Opcode, dl, { EltVT, MVT::Other }, NewOps);
6331	OpChains.push_back(Elt: Ops[i].getValue(R: 1));
6332	}
6333	SDValue NewChain = DAG.getNode(ISD::TokenFactor, dl, MVT::Other, OpChains);
6334	ReplaceValueWith(From: SDValue(N, 1), To: NewChain);
6335	} else {
6336	for (unsigned i = 0; i < NumElts; ++i)
6337	Ops[i] = DAG.getNode(Opcode, DL: dl, VT: EltVT,
6338	Operand: DAG.getNode(Opcode: ISD::EXTRACT_VECTOR_ELT, DL: dl, VT: InEltVT,
6339	N1: InOp, N2: DAG.getVectorIdxConstant(Val: i, DL: dl)));
6340	}
6341
6342	return DAG.getBuildVector(VT, DL: dl, Ops);
6343	}
6344
6345	SDValue DAGTypeLegalizer::WidenVecOp_FP_TO_XINT_SAT(SDNode *N) {
6346	EVT DstVT = N->getValueType(ResNo: 0);
6347	SDValue Src = GetWidenedVector(Op: N->getOperand(Num: 0));
6348	EVT SrcVT = Src.getValueType();
6349	ElementCount WideNumElts = SrcVT.getVectorElementCount();
6350	SDLoc dl(N);
6351
6352	// See if a widened result type would be legal, if so widen the node.
6353	EVT WideDstVT = EVT::getVectorVT(Context&: *DAG.getContext(),
6354	VT: DstVT.getVectorElementType(), EC: WideNumElts);
6355	if (TLI.isTypeLegal(VT: WideDstVT)) {
6356	SDValue Res =
6357	DAG.getNode(Opcode: N->getOpcode(), DL: dl, VT: WideDstVT, N1: Src, N2: N->getOperand(Num: 1));
6358	return DAG.getNode(
6359	Opcode: ISD::EXTRACT_SUBVECTOR, DL: dl, VT: DstVT, N1: Res,
6360	N2: DAG.getConstant(Val: 0, DL: dl, VT: TLI.getVectorIdxTy(DL: DAG.getDataLayout())));
6361	}
6362
6363	// Give up and unroll.
6364	return DAG.UnrollVectorOp(N);
6365	}
6366
6367	SDValue DAGTypeLegalizer::WidenVecOp_BITCAST(SDNode *N) {
6368	EVT VT = N->getValueType(ResNo: 0);
6369	SDValue InOp = GetWidenedVector(Op: N->getOperand(Num: 0));
6370	EVT InWidenVT = InOp.getValueType();
6371	SDLoc dl(N);
6372
6373	// Check if we can convert between two legal vector types and extract.
6374	TypeSize InWidenSize = InWidenVT.getSizeInBits();
6375	TypeSize Size = VT.getSizeInBits();
6376	// x86mmx is not an acceptable vector element type, so don't try.
6377	if (!VT.isVector() && VT != MVT::x86mmx &&
6378	InWidenSize.hasKnownScalarFactor(Size)) {
6379	unsigned NewNumElts = InWidenSize.getKnownScalarFactor(RHS: Size);
6380	EVT NewVT = EVT::getVectorVT(Context&: *DAG.getContext(), VT, NumElements: NewNumElts);
6381	if (TLI.isTypeLegal(VT: NewVT)) {
6382	SDValue BitOp = DAG.getNode(Opcode: ISD::BITCAST, DL: dl, VT: NewVT, Operand: InOp);
6383	return DAG.getNode(Opcode: ISD::EXTRACT_VECTOR_ELT, DL: dl, VT, N1: BitOp,
6384	N2: DAG.getVectorIdxConstant(Val: 0, DL: dl));
6385	}
6386	}
6387
6388	// Handle a case like bitcast v12i8 -> v3i32. Normally that would get widened
6389	// to v16i8 -> v4i32, but for a target where v3i32 is legal but v12i8 is not,
6390	// we end up here. Handling the case here with EXTRACT_SUBVECTOR avoids
6391	// having to copy via memory.
6392	if (VT.isVector()) {
6393	EVT EltVT = VT.getVectorElementType();
6394	unsigned EltSize = EltVT.getFixedSizeInBits();
6395	if (InWidenSize.isKnownMultipleOf(RHS: EltSize)) {
6396	ElementCount NewNumElts =
6397	(InWidenVT.getVectorElementCount() * InWidenVT.getScalarSizeInBits())
6398	.divideCoefficientBy(RHS: EltSize);
6399	EVT NewVT = EVT::getVectorVT(Context&: *DAG.getContext(), VT: EltVT, EC: NewNumElts);
6400	if (TLI.isTypeLegal(VT: NewVT)) {
6401	SDValue BitOp = DAG.getNode(Opcode: ISD::BITCAST, DL: dl, VT: NewVT, Operand: InOp);
6402	return DAG.getNode(Opcode: ISD::EXTRACT_SUBVECTOR, DL: dl, VT, N1: BitOp,
6403	N2: DAG.getVectorIdxConstant(Val: 0, DL: dl));
6404	}
6405	}
6406	}
6407
6408	return CreateStackStoreLoad(Op: InOp, DestVT: VT);
6409	}
6410
6411	SDValue DAGTypeLegalizer::WidenVecOp_CONCAT_VECTORS(SDNode *N) {
6412	EVT VT = N->getValueType(ResNo: 0);
6413	EVT EltVT = VT.getVectorElementType();
6414	EVT InVT = N->getOperand(Num: 0).getValueType();
6415	SDLoc dl(N);
6416
6417	// If the widen width for this operand is the same as the width of the concat
6418	// and all but the first operand is undef, just use the widened operand.
6419	unsigned NumOperands = N->getNumOperands();
6420	if (VT == TLI.getTypeToTransformTo(Context&: *DAG.getContext(), VT: InVT)) {
6421	unsigned i;
6422	for (i = 1; i < NumOperands; ++i)
6423	if (!N->getOperand(Num: i).isUndef())
6424	break;
6425
6426	if (i == NumOperands)
6427	return GetWidenedVector(Op: N->getOperand(Num: 0));
6428	}
6429
6430	// Otherwise, fall back to a nasty build vector.
6431	unsigned NumElts = VT.getVectorNumElements();
6432	SmallVector<SDValue, 16> Ops(NumElts);
6433
6434	unsigned NumInElts = InVT.getVectorNumElements();
6435
6436	unsigned Idx = 0;
6437	for (unsigned i=0; i < NumOperands; ++i) {
6438	SDValue InOp = N->getOperand(Num: i);
6439	assert(getTypeAction(InOp.getValueType()) ==
6440	TargetLowering::TypeWidenVector &&
6441	"Unexpected type action");
6442	InOp = GetWidenedVector(Op: InOp);
6443	for (unsigned j = 0; j < NumInElts; ++j)
6444	Ops[Idx++] = DAG.getNode(Opcode: ISD::EXTRACT_VECTOR_ELT, DL: dl, VT: EltVT, N1: InOp,
6445	N2: DAG.getVectorIdxConstant(Val: j, DL: dl));
6446	}
6447	return DAG.getBuildVector(VT, DL: dl, Ops);
6448	}
6449
6450	SDValue DAGTypeLegalizer::WidenVecOp_INSERT_SUBVECTOR(SDNode *N) {
6451	EVT VT = N->getValueType(ResNo: 0);
6452	SDValue SubVec = N->getOperand(Num: 1);
6453	SDValue InVec = N->getOperand(Num: 0);
6454
6455	if (getTypeAction(VT: SubVec.getValueType()) == TargetLowering::TypeWidenVector)
6456	SubVec = GetWidenedVector(Op: SubVec);
6457
6458	EVT SubVT = SubVec.getValueType();
6459
6460	// Whether or not all the elements of the widened SubVec will be inserted into
6461	// valid indices of VT.
6462	bool IndicesValid = false;
6463	// If we statically know that VT can fit SubVT, the indices are valid.
6464	if (VT.knownBitsGE(VT: SubVT))
6465	IndicesValid = true;
6466	else if (VT.isScalableVector() && SubVT.isFixedLengthVector()) {
6467	// Otherwise, if we're inserting a fixed vector into a scalable vector and
6468	// we know the minimum vscale we can work out if it's valid ourselves.
6469	Attribute Attr = DAG.getMachineFunction().getFunction().getFnAttribute(
6470	Attribute::VScaleRange);
6471	if (Attr.isValid()) {
6472	unsigned VScaleMin = Attr.getVScaleRangeMin();
6473	if (VT.getSizeInBits().getKnownMinValue() * VScaleMin >=
6474	SubVT.getFixedSizeInBits())
6475	IndicesValid = true;
6476	}
6477	}
6478
6479	// We need to make sure that the indices are still valid, otherwise we might
6480	// widen what was previously well-defined to something undefined.
6481	if (IndicesValid && InVec.isUndef() && N->getConstantOperandVal(Num: 2) == 0)
6482	return DAG.getNode(Opcode: ISD::INSERT_SUBVECTOR, DL: SDLoc(N), VT, N1: InVec, N2: SubVec,
6483	N3: N->getOperand(Num: 2));
6484
6485	report_fatal_error(reason: "Don't know how to widen the operands for "
6486	"INSERT_SUBVECTOR");
6487	}
6488
6489	SDValue DAGTypeLegalizer::WidenVecOp_EXTRACT_SUBVECTOR(SDNode *N) {
6490	SDValue InOp = GetWidenedVector(Op: N->getOperand(Num: 0));
6491	return DAG.getNode(Opcode: ISD::EXTRACT_SUBVECTOR, DL: SDLoc(N),
6492	VT: N->getValueType(ResNo: 0), N1: InOp, N2: N->getOperand(Num: 1));
6493	}
6494
6495	SDValue DAGTypeLegalizer::WidenVecOp_EXTRACT_VECTOR_ELT(SDNode *N) {
6496	SDValue InOp = GetWidenedVector(Op: N->getOperand(Num: 0));
6497	return DAG.getNode(Opcode: ISD::EXTRACT_VECTOR_ELT, DL: SDLoc(N),
6498	VT: N->getValueType(ResNo: 0), N1: InOp, N2: N->getOperand(Num: 1));
6499	}
6500
6501	SDValue DAGTypeLegalizer::WidenVecOp_EXTEND_VECTOR_INREG(SDNode *N) {
6502	SDValue InOp = GetWidenedVector(Op: N->getOperand(Num: 0));
6503	return DAG.getNode(Opcode: N->getOpcode(), DL: SDLoc(N), VT: N->getValueType(ResNo: 0), Operand: InOp);
6504	}
6505
6506	SDValue DAGTypeLegalizer::WidenVecOp_STORE(SDNode *N) {
6507	// We have to widen the value, but we want only to store the original
6508	// vector type.
6509	StoreSDNode *ST = cast<StoreSDNode>(Val: N);
6510
6511	if (!ST->getMemoryVT().getScalarType().isByteSized())
6512	return TLI.scalarizeVectorStore(ST, DAG);
6513
6514	if (ST->isTruncatingStore())
6515	return TLI.scalarizeVectorStore(ST, DAG);
6516
6517	// Generate a vector-predicated store if it is custom/legal on the target.
6518	// To avoid possible recursion, only do this if the widened mask type is
6519	// legal.
6520	// FIXME: Not all targets may support EVL in VP_STORE. These will have been
6521	// removed from the IR by the ExpandVectorPredication pass but we're
6522	// reintroducing them here.
6523	SDValue StVal = ST->getValue();
6524	EVT StVT = StVal.getValueType();
6525	EVT WideVT = TLI.getTypeToTransformTo(Context&: *DAG.getContext(), VT: StVT);
6526	EVT WideMaskVT = EVT::getVectorVT(*DAG.getContext(), MVT::i1,
6527	WideVT.getVectorElementCount());
6528
6529	if (TLI.isOperationLegalOrCustom(Op: ISD::VP_STORE, VT: WideVT) &&
6530	TLI.isTypeLegal(VT: WideMaskVT)) {
6531	// Widen the value.
6532	SDLoc DL(N);
6533	StVal = GetWidenedVector(Op: StVal);
6534	SDValue Mask = DAG.getAllOnesConstant(DL, VT: WideMaskVT);
6535	SDValue EVL = DAG.getElementCount(DL, VT: TLI.getVPExplicitVectorLengthTy(),
6536	EC: StVT.getVectorElementCount());
6537	return DAG.getStoreVP(Chain: ST->getChain(), dl: DL, Val: StVal, Ptr: ST->getBasePtr(),
6538	Offset: DAG.getUNDEF(VT: ST->getBasePtr().getValueType()), Mask,
6539	EVL, MemVT: StVT, MMO: ST->getMemOperand(),
6540	AM: ST->getAddressingMode());
6541	}
6542
6543	SmallVector<SDValue, 16> StChain;
6544	if (GenWidenVectorStores(StChain, ST)) {
6545	if (StChain.size() == 1)
6546	return StChain[0];
6547
6548	return DAG.getNode(ISD::TokenFactor, SDLoc(ST), MVT::Other, StChain);
6549	}
6550
6551	report_fatal_error(reason: "Unable to widen vector store");
6552	}
6553
6554	SDValue DAGTypeLegalizer::WidenVecOp_VP_STORE(SDNode *N, unsigned OpNo) {
6555	assert((OpNo == 1 || OpNo == 3) &&
6556	"Can widen only data or mask operand of vp_store");
6557	VPStoreSDNode *ST = cast<VPStoreSDNode>(Val: N);
6558	SDValue Mask = ST->getMask();
6559	SDValue StVal = ST->getValue();
6560	SDLoc dl(N);
6561
6562	if (OpNo == 1) {
6563	// Widen the value.
6564	StVal = GetWidenedVector(Op: StVal);
6565
6566	// We only handle the case where the mask needs widening to an
6567	// identically-sized type as the vector inputs.
6568	assert(getTypeAction(Mask.getValueType()) ==
6569	TargetLowering::TypeWidenVector &&
6570	"Unable to widen VP store");
6571	Mask = GetWidenedVector(Op: Mask);
6572	} else {
6573	Mask = GetWidenedVector(Op: Mask);
6574
6575	// We only handle the case where the stored value needs widening to an
6576	// identically-sized type as the mask.
6577	assert(getTypeAction(StVal.getValueType()) ==
6578	TargetLowering::TypeWidenVector &&
6579	"Unable to widen VP store");
6580	StVal = GetWidenedVector(Op: StVal);
6581	}
6582
6583	assert(Mask.getValueType().getVectorElementCount() ==
6584	StVal.getValueType().getVectorElementCount() &&
6585	"Mask and data vectors should have the same number of elements");
6586	return DAG.getStoreVP(Chain: ST->getChain(), dl, Val: StVal, Ptr: ST->getBasePtr(),
6587	Offset: ST->getOffset(), Mask, EVL: ST->getVectorLength(),
6588	MemVT: ST->getMemoryVT(), MMO: ST->getMemOperand(),
6589	AM: ST->getAddressingMode(), IsTruncating: ST->isTruncatingStore(),
6590	IsCompressing: ST->isCompressingStore());
6591	}
6592
6593	SDValue DAGTypeLegalizer::WidenVecOp_VP_STRIDED_STORE(SDNode *N,
6594	unsigned OpNo) {
6595	assert((OpNo == 1 || OpNo == 4) &&
6596	"Can widen only data or mask operand of vp_strided_store");
6597	VPStridedStoreSDNode *SST = cast<VPStridedStoreSDNode>(Val: N);
6598	SDValue Mask = SST->getMask();
6599	SDValue StVal = SST->getValue();
6600	SDLoc DL(N);
6601
6602	if (OpNo == 1)
6603	assert(getTypeAction(Mask.getValueType()) ==
6604	TargetLowering::TypeWidenVector &&
6605	"Unable to widen VP strided store");
6606	else
6607	assert(getTypeAction(StVal.getValueType()) ==
6608	TargetLowering::TypeWidenVector &&
6609	"Unable to widen VP strided store");
6610
6611	StVal = GetWidenedVector(Op: StVal);
6612	Mask = GetWidenedVector(Op: Mask);
6613
6614	assert(StVal.getValueType().getVectorElementCount() ==
6615	Mask.getValueType().getVectorElementCount() &&
6616	"Data and mask vectors should have the same number of elements");
6617
6618	return DAG.getStridedStoreVP(
6619	Chain: SST->getChain(), DL, Val: StVal, Ptr: SST->getBasePtr(), Offset: SST->getOffset(),
6620	Stride: SST->getStride(), Mask, EVL: SST->getVectorLength(), MemVT: SST->getMemoryVT(),
6621	MMO: SST->getMemOperand(), AM: SST->getAddressingMode(), IsTruncating: SST->isTruncatingStore(),
6622	IsCompressing: SST->isCompressingStore());
6623	}
6624
6625	SDValue DAGTypeLegalizer::WidenVecOp_MSTORE(SDNode *N, unsigned OpNo) {
6626	assert((OpNo == 1 || OpNo == 4) &&
6627	"Can widen only data or mask operand of mstore");
6628	MaskedStoreSDNode *MST = cast<MaskedStoreSDNode>(Val: N);
6629	SDValue Mask = MST->getMask();
6630	EVT MaskVT = Mask.getValueType();
6631	SDValue StVal = MST->getValue();
6632	SDLoc dl(N);
6633
6634	if (OpNo == 1) {
6635	// Widen the value.
6636	StVal = GetWidenedVector(Op: StVal);
6637
6638	// The mask should be widened as well.
6639	EVT WideVT = StVal.getValueType();
6640	EVT WideMaskVT = EVT::getVectorVT(Context&: *DAG.getContext(),
6641	VT: MaskVT.getVectorElementType(),
6642	NumElements: WideVT.getVectorNumElements());
6643	Mask = ModifyToType(InOp: Mask, NVT: WideMaskVT, FillWithZeroes: true);
6644	} else {
6645	// Widen the mask.
6646	EVT WideMaskVT = TLI.getTypeToTransformTo(Context&: *DAG.getContext(), VT: MaskVT);
6647	Mask = ModifyToType(InOp: Mask, NVT: WideMaskVT, FillWithZeroes: true);
6648
6649	EVT ValueVT = StVal.getValueType();
6650	EVT WideVT = EVT::getVectorVT(Context&: *DAG.getContext(),
6651	VT: ValueVT.getVectorElementType(),
6652	NumElements: WideMaskVT.getVectorNumElements());
6653	StVal = ModifyToType(InOp: StVal, NVT: WideVT);
6654	}
6655
6656	assert(Mask.getValueType().getVectorNumElements() ==
6657	StVal.getValueType().getVectorNumElements() &&
6658	"Mask and data vectors should have the same number of elements");
6659	return DAG.getMaskedStore(Chain: MST->getChain(), dl, Val: StVal, Base: MST->getBasePtr(),
6660	Offset: MST->getOffset(), Mask, MemVT: MST->getMemoryVT(),
6661	MMO: MST->getMemOperand(), AM: MST->getAddressingMode(),
6662	IsTruncating: false, IsCompressing: MST->isCompressingStore());
6663	}
6664
6665	SDValue DAGTypeLegalizer::WidenVecOp_MGATHER(SDNode *N, unsigned OpNo) {
6666	assert(OpNo == 4 && "Can widen only the index of mgather");
6667	auto *MG = cast<MaskedGatherSDNode>(Val: N);
6668	SDValue DataOp = MG->getPassThru();
6669	SDValue Mask = MG->getMask();
6670	SDValue Scale = MG->getScale();
6671
6672	// Just widen the index. It's allowed to have extra elements.
6673	SDValue Index = GetWidenedVector(Op: MG->getIndex());
6674
6675	SDLoc dl(N);
6676	SDValue Ops[] = {MG->getChain(), DataOp, Mask, MG->getBasePtr(), Index,
6677	Scale};
6678	SDValue Res = DAG.getMaskedGather(VTs: MG->getVTList(), MemVT: MG->getMemoryVT(), dl, Ops,
6679	MMO: MG->getMemOperand(), IndexType: MG->getIndexType(),
6680	ExtTy: MG->getExtensionType());
6681	ReplaceValueWith(From: SDValue(N, 1), To: Res.getValue(R: 1));
6682	ReplaceValueWith(From: SDValue(N, 0), To: Res.getValue(R: 0));
6683	return SDValue();
6684	}
6685
6686	SDValue DAGTypeLegalizer::WidenVecOp_MSCATTER(SDNode *N, unsigned OpNo) {
6687	MaskedScatterSDNode *MSC = cast<MaskedScatterSDNode>(Val: N);
6688	SDValue DataOp = MSC->getValue();
6689	SDValue Mask = MSC->getMask();
6690	SDValue Index = MSC->getIndex();
6691	SDValue Scale = MSC->getScale();
6692	EVT WideMemVT = MSC->getMemoryVT();
6693
6694	if (OpNo == 1) {
6695	DataOp = GetWidenedVector(Op: DataOp);
6696	unsigned NumElts = DataOp.getValueType().getVectorNumElements();
6697
6698	// Widen index.
6699	EVT IndexVT = Index.getValueType();
6700	EVT WideIndexVT = EVT::getVectorVT(Context&: *DAG.getContext(),
6701	VT: IndexVT.getVectorElementType(), NumElements: NumElts);
6702	Index = ModifyToType(InOp: Index, NVT: WideIndexVT);
6703
6704	// The mask should be widened as well.
6705	EVT MaskVT = Mask.getValueType();
6706	EVT WideMaskVT = EVT::getVectorVT(Context&: *DAG.getContext(),
6707	VT: MaskVT.getVectorElementType(), NumElements: NumElts);
6708	Mask = ModifyToType(InOp: Mask, NVT: WideMaskVT, FillWithZeroes: true);
6709
6710	// Widen the MemoryType
6711	WideMemVT = EVT::getVectorVT(Context&: *DAG.getContext(),
6712	VT: MSC->getMemoryVT().getScalarType(), NumElements: NumElts);
6713	} else if (OpNo == 4) {
6714	// Just widen the index. It's allowed to have extra elements.
6715	Index = GetWidenedVector(Op: Index);
6716	} else
6717	llvm_unreachable("Can't widen this operand of mscatter");
6718
6719	SDValue Ops[] = {MSC->getChain(), DataOp, Mask, MSC->getBasePtr(), Index,
6720	Scale};
6721	return DAG.getMaskedScatter(DAG.getVTList(MVT::Other), WideMemVT, SDLoc(N),
6722	Ops, MSC->getMemOperand(), MSC->getIndexType(),
6723	MSC->isTruncatingStore());
6724	}
6725
6726	SDValue DAGTypeLegalizer::WidenVecOp_VP_SCATTER(SDNode *N, unsigned OpNo) {
6727	VPScatterSDNode *VPSC = cast<VPScatterSDNode>(Val: N);
6728	SDValue DataOp = VPSC->getValue();
6729	SDValue Mask = VPSC->getMask();
6730	SDValue Index = VPSC->getIndex();
6731	SDValue Scale = VPSC->getScale();
6732	EVT WideMemVT = VPSC->getMemoryVT();
6733
6734	if (OpNo == 1) {
6735	DataOp = GetWidenedVector(Op: DataOp);
6736	Index = GetWidenedVector(Op: Index);
6737	const auto WideEC = DataOp.getValueType().getVectorElementCount();
6738	Mask = GetWidenedMask(Mask, EC: WideEC);
6739	WideMemVT = EVT::getVectorVT(Context&: *DAG.getContext(),
6740	VT: VPSC->getMemoryVT().getScalarType(), EC: WideEC);
6741	} else if (OpNo == 3) {
6742	// Just widen the index. It's allowed to have extra elements.
6743	Index = GetWidenedVector(Op: Index);
6744	} else
6745	llvm_unreachable("Can't widen this operand of VP_SCATTER");
6746
6747	SDValue Ops[] = {
6748	VPSC->getChain(), DataOp, VPSC->getBasePtr(), Index, Scale, Mask,
6749	VPSC->getVectorLength()};
6750	return DAG.getScatterVP(DAG.getVTList(MVT::Other), WideMemVT, SDLoc(N), Ops,
6751	VPSC->getMemOperand(), VPSC->getIndexType());
6752	}
6753
6754	SDValue DAGTypeLegalizer::WidenVecOp_SETCC(SDNode *N) {
6755	SDValue InOp0 = GetWidenedVector(Op: N->getOperand(Num: 0));
6756	SDValue InOp1 = GetWidenedVector(Op: N->getOperand(Num: 1));
6757	SDLoc dl(N);
6758	EVT VT = N->getValueType(ResNo: 0);
6759
6760	// WARNING: In this code we widen the compare instruction with garbage.
6761	// This garbage may contain denormal floats which may be slow. Is this a real
6762	// concern ? Should we zero the unused lanes if this is a float compare ?
6763
6764	// Get a new SETCC node to compare the newly widened operands.
6765	// Only some of the compared elements are legal.
6766	EVT SVT = getSetCCResultType(VT: InOp0.getValueType());
6767	// The result type is legal, if its vXi1, keep vXi1 for the new SETCC.
6768	if (VT.getScalarType() == MVT::i1)
6769	SVT = EVT::getVectorVT(*DAG.getContext(), MVT::i1,
6770	SVT.getVectorElementCount());
6771
6772	SDValue WideSETCC = DAG.getNode(Opcode: ISD::SETCC, DL: SDLoc(N),
6773	VT: SVT, N1: InOp0, N2: InOp1, N3: N->getOperand(Num: 2));
6774
6775	// Extract the needed results from the result vector.
6776	EVT ResVT = EVT::getVectorVT(Context&: *DAG.getContext(),
6777	VT: SVT.getVectorElementType(),
6778	EC: VT.getVectorElementCount());
6779	SDValue CC = DAG.getNode(Opcode: ISD::EXTRACT_SUBVECTOR, DL: dl, VT: ResVT, N1: WideSETCC,
6780	N2: DAG.getVectorIdxConstant(Val: 0, DL: dl));
6781
6782	EVT OpVT = N->getOperand(Num: 0).getValueType();
6783	ISD::NodeType ExtendCode =
6784	TargetLowering::getExtendForContent(Content: TLI.getBooleanContents(Type: OpVT));
6785	return DAG.getNode(Opcode: ExtendCode, DL: dl, VT, Operand: CC);
6786	}
6787
6788	SDValue DAGTypeLegalizer::WidenVecOp_STRICT_FSETCC(SDNode *N) {
6789	SDValue Chain = N->getOperand(Num: 0);
6790	SDValue LHS = GetWidenedVector(Op: N->getOperand(Num: 1));
6791	SDValue RHS = GetWidenedVector(Op: N->getOperand(Num: 2));
6792	SDValue CC = N->getOperand(Num: 3);
6793	SDLoc dl(N);
6794
6795	EVT VT = N->getValueType(ResNo: 0);
6796	EVT EltVT = VT.getVectorElementType();
6797	EVT TmpEltVT = LHS.getValueType().getVectorElementType();
6798	unsigned NumElts = VT.getVectorNumElements();
6799
6800	// Unroll into a build vector.
6801	SmallVector<SDValue, 8> Scalars(NumElts);
6802	SmallVector<SDValue, 8> Chains(NumElts);
6803
6804	for (unsigned i = 0; i != NumElts; ++i) {
6805	SDValue LHSElem = DAG.getNode(Opcode: ISD::EXTRACT_VECTOR_ELT, DL: dl, VT: TmpEltVT, N1: LHS,
6806	N2: DAG.getVectorIdxConstant(Val: i, DL: dl));
6807	SDValue RHSElem = DAG.getNode(Opcode: ISD::EXTRACT_VECTOR_ELT, DL: dl, VT: TmpEltVT, N1: RHS,
6808	N2: DAG.getVectorIdxConstant(Val: i, DL: dl));
6809
6810	Scalars[i] = DAG.getNode(N->getOpcode(), dl, {MVT::i1, MVT::Other},
6811	{Chain, LHSElem, RHSElem, CC});
6812	Chains[i] = Scalars[i].getValue(R: 1);
6813	Scalars[i] = DAG.getSelect(DL: dl, VT: EltVT, Cond: Scalars[i],
6814	LHS: DAG.getBoolConstant(V: true, DL: dl, VT: EltVT, OpVT: VT),
6815	RHS: DAG.getBoolConstant(V: false, DL: dl, VT: EltVT, OpVT: VT));
6816	}
6817
6818	SDValue NewChain = DAG.getNode(ISD::TokenFactor, dl, MVT::Other, Chains);
6819	ReplaceValueWith(From: SDValue(N, 1), To: NewChain);
6820
6821	return DAG.getBuildVector(VT, DL: dl, Ops: Scalars);
6822	}
6823
6824	SDValue DAGTypeLegalizer::WidenVecOp_VECREDUCE(SDNode *N) {
6825	SDLoc dl(N);
6826	SDValue Op = GetWidenedVector(Op: N->getOperand(Num: 0));
6827	EVT OrigVT = N->getOperand(Num: 0).getValueType();
6828	EVT WideVT = Op.getValueType();
6829	EVT ElemVT = OrigVT.getVectorElementType();
6830	SDNodeFlags Flags = N->getFlags();
6831
6832	unsigned Opc = N->getOpcode();
6833	unsigned BaseOpc = ISD::getVecReduceBaseOpcode(VecReduceOpcode: Opc);
6834	SDValue NeutralElem = DAG.getNeutralElement(Opcode: BaseOpc, DL: dl, VT: ElemVT, Flags);
6835	assert(NeutralElem && "Neutral element must exist");
6836
6837	// Pad the vector with the neutral element.
6838	unsigned OrigElts = OrigVT.getVectorMinNumElements();
6839	unsigned WideElts = WideVT.getVectorMinNumElements();
6840
6841	if (WideVT.isScalableVector()) {
6842	unsigned GCD = std::gcd(m: OrigElts, n: WideElts);
6843	EVT SplatVT = EVT::getVectorVT(Context&: *DAG.getContext(), VT: ElemVT,
6844	EC: ElementCount::getScalable(MinVal: GCD));
6845	SDValue SplatNeutral = DAG.getSplatVector(VT: SplatVT, DL: dl, Op: NeutralElem);
6846	for (unsigned Idx = OrigElts; Idx < WideElts; Idx = Idx + GCD)
6847	Op = DAG.getNode(Opcode: ISD::INSERT_SUBVECTOR, DL: dl, VT: WideVT, N1: Op, N2: SplatNeutral,
6848	N3: DAG.getVectorIdxConstant(Val: Idx, DL: dl));
6849	return DAG.getNode(Opcode: Opc, DL: dl, VT: N->getValueType(ResNo: 0), Operand: Op, Flags);
6850	}
6851
6852	for (unsigned Idx = OrigElts; Idx < WideElts; Idx++)
6853	Op = DAG.getNode(Opcode: ISD::INSERT_VECTOR_ELT, DL: dl, VT: WideVT, N1: Op, N2: NeutralElem,
6854	N3: DAG.getVectorIdxConstant(Val: Idx, DL: dl));
6855
6856	return DAG.getNode(Opcode: Opc, DL: dl, VT: N->getValueType(ResNo: 0), Operand: Op, Flags);
6857	}
6858
6859	SDValue DAGTypeLegalizer::WidenVecOp_VECREDUCE_SEQ(SDNode *N) {
6860	SDLoc dl(N);
6861	SDValue AccOp = N->getOperand(Num: 0);
6862	SDValue VecOp = N->getOperand(Num: 1);
6863	SDValue Op = GetWidenedVector(Op: VecOp);
6864
6865	EVT OrigVT = VecOp.getValueType();
6866	EVT WideVT = Op.getValueType();
6867	EVT ElemVT = OrigVT.getVectorElementType();
6868	SDNodeFlags Flags = N->getFlags();
6869
6870	unsigned Opc = N->getOpcode();
6871	unsigned BaseOpc = ISD::getVecReduceBaseOpcode(VecReduceOpcode: Opc);
6872	SDValue NeutralElem = DAG.getNeutralElement(Opcode: BaseOpc, DL: dl, VT: ElemVT, Flags);
6873
6874	// Pad the vector with the neutral element.
6875	unsigned OrigElts = OrigVT.getVectorMinNumElements();
6876	unsigned WideElts = WideVT.getVectorMinNumElements();
6877
6878	if (WideVT.isScalableVector()) {
6879	unsigned GCD = std::gcd(m: OrigElts, n: WideElts);
6880	EVT SplatVT = EVT::getVectorVT(Context&: *DAG.getContext(), VT: ElemVT,
6881	EC: ElementCount::getScalable(MinVal: GCD));
6882	SDValue SplatNeutral = DAG.getSplatVector(VT: SplatVT, DL: dl, Op: NeutralElem);
6883	for (unsigned Idx = OrigElts; Idx < WideElts; Idx = Idx + GCD)
6884	Op = DAG.getNode(Opcode: ISD::INSERT_SUBVECTOR, DL: dl, VT: WideVT, N1: Op, N2: SplatNeutral,
6885	N3: DAG.getVectorIdxConstant(Val: Idx, DL: dl));
6886	return DAG.getNode(Opcode: Opc, DL: dl, VT: N->getValueType(ResNo: 0), N1: AccOp, N2: Op, Flags);
6887	}
6888
6889	for (unsigned Idx = OrigElts; Idx < WideElts; Idx++)
6890	Op = DAG.getNode(Opcode: ISD::INSERT_VECTOR_ELT, DL: dl, VT: WideVT, N1: Op, N2: NeutralElem,
6891	N3: DAG.getVectorIdxConstant(Val: Idx, DL: dl));
6892
6893	return DAG.getNode(Opcode: Opc, DL: dl, VT: N->getValueType(ResNo: 0), N1: AccOp, N2: Op, Flags);
6894	}
6895
6896	SDValue DAGTypeLegalizer::WidenVecOp_VP_REDUCE(SDNode *N) {
6897	assert(N->isVPOpcode() && "Expected VP opcode");
6898
6899	SDLoc dl(N);
6900	SDValue Op = GetWidenedVector(Op: N->getOperand(Num: 1));
6901	SDValue Mask = GetWidenedMask(Mask: N->getOperand(Num: 2),
6902	EC: Op.getValueType().getVectorElementCount());
6903
6904	return DAG.getNode(Opcode: N->getOpcode(), DL: dl, VT: N->getValueType(ResNo: 0),
6905	Ops: {N->getOperand(Num: 0), Op, Mask, N->getOperand(Num: 3)},
6906	Flags: N->getFlags());
6907	}
6908
6909	SDValue DAGTypeLegalizer::WidenVecOp_VSELECT(SDNode *N) {
6910	// This only gets called in the case that the left and right inputs and
6911	// result are of a legal odd vector type, and the condition is illegal i1 of
6912	// the same odd width that needs widening.
6913	EVT VT = N->getValueType(ResNo: 0);
6914	assert(VT.isVector() && !VT.isPow2VectorType() && isTypeLegal(VT));
6915
6916	SDValue Cond = GetWidenedVector(Op: N->getOperand(Num: 0));
6917	SDValue LeftIn = DAG.WidenVector(N: N->getOperand(Num: 1), DL: SDLoc(N));
6918	SDValue RightIn = DAG.WidenVector(N: N->getOperand(Num: 2), DL: SDLoc(N));
6919	SDLoc DL(N);
6920
6921	SDValue Select = DAG.getNode(Opcode: N->getOpcode(), DL, VT: LeftIn.getValueType(), N1: Cond,
6922	N2: LeftIn, N3: RightIn);
6923	return DAG.getNode(Opcode: ISD::EXTRACT_SUBVECTOR, DL, VT, N1: Select,
6924	N2: DAG.getVectorIdxConstant(Val: 0, DL));
6925	}
6926
6927	//===----------------------------------------------------------------------===//
6928	// Vector Widening Utilities
6929	//===----------------------------------------------------------------------===//
6930
6931	// Utility function to find the type to chop up a widen vector for load/store
6932	// TLI: Target lowering used to determine legal types.
6933	// Width: Width left need to load/store.
6934	// WidenVT: The widen vector type to load to/store from
6935	// Align: If 0, don't allow use of a wider type
6936	// WidenEx: If Align is not 0, the amount additional we can load/store from.
6937
6938	static std::optional<EVT> findMemType(SelectionDAG &DAG,
6939	const TargetLowering &TLI, unsigned Width,
6940	EVT WidenVT, unsigned Align = 0,
6941	unsigned WidenEx = 0) {
6942	EVT WidenEltVT = WidenVT.getVectorElementType();
6943	const bool Scalable = WidenVT.isScalableVector();
6944	unsigned WidenWidth = WidenVT.getSizeInBits().getKnownMinValue();
6945	unsigned WidenEltWidth = WidenEltVT.getSizeInBits();
6946	unsigned AlignInBits = Align*8;
6947
6948	// If we have one element to load/store, return it.
6949	EVT RetVT = WidenEltVT;
6950	if (!Scalable && Width == WidenEltWidth)
6951	return RetVT;
6952
6953	// Don't bother looking for an integer type if the vector is scalable, skip
6954	// to vector types.
6955	if (!Scalable) {
6956	// See if there is larger legal integer than the element type to load/store.
6957	for (EVT MemVT : reverse(MVT::integer_valuetypes())) {
6958	unsigned MemVTWidth = MemVT.getSizeInBits();
6959	if (MemVT.getSizeInBits() <= WidenEltWidth)
6960	break;
6961	auto Action = TLI.getTypeAction(*DAG.getContext(), MemVT);
6962	if ((Action == TargetLowering::TypeLegal ||
6963	Action == TargetLowering::TypePromoteInteger) &&
6964	(WidenWidth % MemVTWidth) == 0 &&
6965	isPowerOf2_32(WidenWidth / MemVTWidth) &&
6966	(MemVTWidth <= Width ||
6967	(Align!=0 && MemVTWidth<=AlignInBits && MemVTWidth<=Width+WidenEx))) {
6968	if (MemVTWidth == WidenWidth)
6969	return MemVT;
6970	RetVT = MemVT;
6971	break;
6972	}
6973	}
6974	}
6975
6976	// See if there is a larger vector type to load/store that has the same vector
6977	// element type and is evenly divisible with the WidenVT.
6978	for (EVT MemVT : reverse(MVT::vector_valuetypes())) {
6979	// Skip vector MVTs which don't match the scalable property of WidenVT.
6980	if (Scalable != MemVT.isScalableVector())
6981	continue;
6982	unsigned MemVTWidth = MemVT.getSizeInBits().getKnownMinValue();
6983	auto Action = TLI.getTypeAction(*DAG.getContext(), MemVT);
6984	if ((Action == TargetLowering::TypeLegal ||
6985	Action == TargetLowering::TypePromoteInteger) &&
6986	WidenEltVT == MemVT.getVectorElementType() &&
6987	(WidenWidth % MemVTWidth) == 0 &&
6988	isPowerOf2_32(WidenWidth / MemVTWidth) &&
6989	(MemVTWidth <= Width ||
6990	(Align!=0 && MemVTWidth<=AlignInBits && MemVTWidth<=Width+WidenEx))) {
6991	if (RetVT.getFixedSizeInBits() < MemVTWidth || MemVT == WidenVT)
6992	return MemVT;
6993	}
6994	}
6995
6996	// Using element-wise loads and stores for widening operations is not
6997	// supported for scalable vectors
6998	if (Scalable)
6999	return std::nullopt;
7000
7001	return RetVT;
7002	}
7003
7004	// Builds a vector type from scalar loads
7005	// VecTy: Resulting Vector type
7006	// LDOps: Load operators to build a vector type
7007	// [Start,End) the list of loads to use.
7008	static SDValue BuildVectorFromScalar(SelectionDAG& DAG, EVT VecTy,
7009	SmallVectorImpl<SDValue> &LdOps,
7010	unsigned Start, unsigned End) {
7011	SDLoc dl(LdOps[Start]);
7012	EVT LdTy = LdOps[Start].getValueType();
7013	unsigned Width = VecTy.getSizeInBits();
7014	unsigned NumElts = Width / LdTy.getSizeInBits();
7015	EVT NewVecVT = EVT::getVectorVT(Context&: *DAG.getContext(), VT: LdTy, NumElements: NumElts);
7016
7017	unsigned Idx = 1;
7018	SDValue VecOp = DAG.getNode(Opcode: ISD::SCALAR_TO_VECTOR, DL: dl, VT: NewVecVT,Operand: LdOps[Start]);
7019
7020	for (unsigned i = Start + 1; i != End; ++i) {
7021	EVT NewLdTy = LdOps[i].getValueType();
7022	if (NewLdTy != LdTy) {
7023	NumElts = Width / NewLdTy.getSizeInBits();
7024	NewVecVT = EVT::getVectorVT(Context&: *DAG.getContext(), VT: NewLdTy, NumElements: NumElts);
7025	VecOp = DAG.getNode(Opcode: ISD::BITCAST, DL: dl, VT: NewVecVT, Operand: VecOp);
7026	// Readjust position and vector position based on new load type.
7027	Idx = Idx * LdTy.getSizeInBits() / NewLdTy.getSizeInBits();
7028	LdTy = NewLdTy;
7029	}
7030	VecOp = DAG.getNode(Opcode: ISD::INSERT_VECTOR_ELT, DL: dl, VT: NewVecVT, N1: VecOp, N2: LdOps[i],
7031	N3: DAG.getVectorIdxConstant(Val: Idx++, DL: dl));
7032	}
7033	return DAG.getNode(Opcode: ISD::BITCAST, DL: dl, VT: VecTy, Operand: VecOp);
7034	}
7035
7036	SDValue DAGTypeLegalizer::GenWidenVectorLoads(SmallVectorImpl<SDValue> &LdChain,
7037	LoadSDNode *LD) {
7038	// The strategy assumes that we can efficiently load power-of-two widths.
7039	// The routine chops the vector into the largest vector loads with the same
7040	// element type or scalar loads and then recombines it to the widen vector
7041	// type.
7042	EVT WidenVT = TLI.getTypeToTransformTo(Context&: *DAG.getContext(),VT: LD->getValueType(ResNo: 0));
7043	EVT LdVT = LD->getMemoryVT();
7044	SDLoc dl(LD);
7045	assert(LdVT.isVector() && WidenVT.isVector());
7046	assert(LdVT.isScalableVector() == WidenVT.isScalableVector());
7047	assert(LdVT.getVectorElementType() == WidenVT.getVectorElementType());
7048
7049	// Load information
7050	SDValue Chain = LD->getChain();
7051	SDValue BasePtr = LD->getBasePtr();
7052	MachineMemOperand::Flags MMOFlags = LD->getMemOperand()->getFlags();
7053	AAMDNodes AAInfo = LD->getAAInfo();
7054
7055	TypeSize LdWidth = LdVT.getSizeInBits();
7056	TypeSize WidenWidth = WidenVT.getSizeInBits();
7057	TypeSize WidthDiff = WidenWidth - LdWidth;
7058	// Allow wider loads if they are sufficiently aligned to avoid memory faults
7059	// and if the original load is simple.
7060	unsigned LdAlign =
7061	(!LD->isSimple() || LdVT.isScalableVector()) ? 0 : LD->getAlign().value();
7062
7063	// Find the vector type that can load from.
7064	std::optional<EVT> FirstVT =
7065	findMemType(DAG, TLI, Width: LdWidth.getKnownMinValue(), WidenVT, Align: LdAlign,
7066	WidenEx: WidthDiff.getKnownMinValue());
7067
7068	if (!FirstVT)
7069	return SDValue();
7070
7071	SmallVector<EVT, 8> MemVTs;
7072	TypeSize FirstVTWidth = FirstVT->getSizeInBits();
7073
7074	// Unless we're able to load in one instruction we must work out how to load
7075	// the remainder.
7076	if (!TypeSize::isKnownLE(LHS: LdWidth, RHS: FirstVTWidth)) {
7077	std::optional<EVT> NewVT = FirstVT;
7078	TypeSize RemainingWidth = LdWidth;
7079	TypeSize NewVTWidth = FirstVTWidth;
7080	do {
7081	RemainingWidth -= NewVTWidth;
7082	if (TypeSize::isKnownLT(LHS: RemainingWidth, RHS: NewVTWidth)) {
7083	// The current type we are using is too large. Find a better size.
7084	NewVT = findMemType(DAG, TLI, Width: RemainingWidth.getKnownMinValue(),
7085	WidenVT, Align: LdAlign, WidenEx: WidthDiff.getKnownMinValue());
7086	if (!NewVT)
7087	return SDValue();
7088	NewVTWidth = NewVT->getSizeInBits();
7089	}
7090	MemVTs.push_back(Elt: *NewVT);
7091	} while (TypeSize::isKnownGT(LHS: RemainingWidth, RHS: NewVTWidth));
7092	}
7093
7094	SDValue LdOp = DAG.getLoad(VT: *FirstVT, dl, Chain, Ptr: BasePtr, PtrInfo: LD->getPointerInfo(),
7095	Alignment: LD->getOriginalAlign(), MMOFlags, AAInfo);
7096	LdChain.push_back(Elt: LdOp.getValue(R: 1));
7097
7098	// Check if we can load the element with one instruction.
7099	if (MemVTs.empty()) {
7100	assert(TypeSize::isKnownLE(LdWidth, FirstVTWidth));
7101	if (!FirstVT->isVector()) {
7102	unsigned NumElts =
7103	WidenWidth.getFixedValue() / FirstVTWidth.getFixedValue();
7104	EVT NewVecVT = EVT::getVectorVT(Context&: *DAG.getContext(), VT: *FirstVT, NumElements: NumElts);
7105	SDValue VecOp = DAG.getNode(Opcode: ISD::SCALAR_TO_VECTOR, DL: dl, VT: NewVecVT, Operand: LdOp);
7106	return DAG.getNode(Opcode: ISD::BITCAST, DL: dl, VT: WidenVT, Operand: VecOp);
7107	}
7108	if (FirstVT == WidenVT)
7109	return LdOp;
7110
7111	// TODO: We don't currently have any tests that exercise this code path.
7112	assert(WidenWidth.getFixedValue() % FirstVTWidth.getFixedValue() == 0);
7113	unsigned NumConcat =
7114	WidenWidth.getFixedValue() / FirstVTWidth.getFixedValue();
7115	SmallVector<SDValue, 16> ConcatOps(NumConcat);
7116	SDValue UndefVal = DAG.getUNDEF(VT: *FirstVT);
7117	ConcatOps[0] = LdOp;
7118	for (unsigned i = 1; i != NumConcat; ++i)
7119	ConcatOps[i] = UndefVal;
7120	return DAG.getNode(Opcode: ISD::CONCAT_VECTORS, DL: dl, VT: WidenVT, Ops: ConcatOps);
7121	}
7122
7123	// Load vector by using multiple loads from largest vector to scalar.
7124	SmallVector<SDValue, 16> LdOps;
7125	LdOps.push_back(Elt: LdOp);
7126
7127	uint64_t ScaledOffset = 0;
7128	MachinePointerInfo MPI = LD->getPointerInfo();
7129
7130	// First incremement past the first load.
7131	IncrementPointer(N: cast<LoadSDNode>(Val&: LdOp), MemVT: *FirstVT, MPI, Ptr&: BasePtr,
7132	ScaledOffset: &ScaledOffset);
7133
7134	for (EVT MemVT : MemVTs) {
7135	Align NewAlign = ScaledOffset == 0
7136	? LD->getOriginalAlign()
7137	: commonAlignment(A: LD->getAlign(), Offset: ScaledOffset);
7138	SDValue L =
7139	DAG.getLoad(VT: MemVT, dl, Chain, Ptr: BasePtr, PtrInfo: MPI, Alignment: NewAlign, MMOFlags, AAInfo);
7140
7141	LdOps.push_back(Elt: L);
7142	LdChain.push_back(Elt: L.getValue(R: 1));
7143	IncrementPointer(N: cast<LoadSDNode>(Val&: L), MemVT, MPI, Ptr&: BasePtr, ScaledOffset: &ScaledOffset);
7144	}
7145
7146	// Build the vector from the load operations.
7147	unsigned End = LdOps.size();
7148	if (!LdOps[0].getValueType().isVector())
7149	// All the loads are scalar loads.
7150	return BuildVectorFromScalar(DAG, VecTy: WidenVT, LdOps, Start: 0, End);
7151
7152	// If the load contains vectors, build the vector using concat vector.
7153	// All of the vectors used to load are power-of-2, and the scalar loads can be
7154	// combined to make a power-of-2 vector.
7155	SmallVector<SDValue, 16> ConcatOps(End);
7156	int i = End - 1;
7157	int Idx = End;
7158	EVT LdTy = LdOps[i].getValueType();
7159	// First, combine the scalar loads to a vector.
7160	if (!LdTy.isVector()) {
7161	for (--i; i >= 0; --i) {
7162	LdTy = LdOps[i].getValueType();
7163	if (LdTy.isVector())
7164	break;
7165	}
7166	ConcatOps[--Idx] = BuildVectorFromScalar(DAG, VecTy: LdTy, LdOps, Start: i + 1, End);
7167	}
7168
7169	ConcatOps[--Idx] = LdOps[i];
7170	for (--i; i >= 0; --i) {
7171	EVT NewLdTy = LdOps[i].getValueType();
7172	if (NewLdTy != LdTy) {
7173	// Create a larger vector.
7174	TypeSize LdTySize = LdTy.getSizeInBits();
7175	TypeSize NewLdTySize = NewLdTy.getSizeInBits();
7176	assert(NewLdTySize.isScalable() == LdTySize.isScalable() &&
7177	NewLdTySize.isKnownMultipleOf(LdTySize.getKnownMinValue()));
7178	unsigned NumOps =
7179	NewLdTySize.getKnownMinValue() / LdTySize.getKnownMinValue();
7180	SmallVector<SDValue, 16> WidenOps(NumOps);
7181	unsigned j = 0;
7182	for (; j != End-Idx; ++j)
7183	WidenOps[j] = ConcatOps[Idx+j];
7184	for (; j != NumOps; ++j)
7185	WidenOps[j] = DAG.getUNDEF(VT: LdTy);
7186
7187	ConcatOps[End-1] = DAG.getNode(Opcode: ISD::CONCAT_VECTORS, DL: dl, VT: NewLdTy,
7188	Ops: WidenOps);
7189	Idx = End - 1;
7190	LdTy = NewLdTy;
7191	}
7192	ConcatOps[--Idx] = LdOps[i];
7193	}
7194
7195	if (WidenWidth == LdTy.getSizeInBits() * (End - Idx))
7196	return DAG.getNode(Opcode: ISD::CONCAT_VECTORS, DL: dl, VT: WidenVT,
7197	Ops: ArrayRef(&ConcatOps[Idx], End - Idx));
7198
7199	// We need to fill the rest with undefs to build the vector.
7200	unsigned NumOps =
7201	WidenWidth.getKnownMinValue() / LdTy.getSizeInBits().getKnownMinValue();
7202	SmallVector<SDValue, 16> WidenOps(NumOps);
7203	SDValue UndefVal = DAG.getUNDEF(VT: LdTy);
7204	{
7205	unsigned i = 0;
7206	for (; i != End-Idx; ++i)
7207	WidenOps[i] = ConcatOps[Idx+i];
7208	for (; i != NumOps; ++i)
7209	WidenOps[i] = UndefVal;
7210	}
7211	return DAG.getNode(Opcode: ISD::CONCAT_VECTORS, DL: dl, VT: WidenVT, Ops: WidenOps);
7212	}
7213
7214	SDValue
7215	DAGTypeLegalizer::GenWidenVectorExtLoads(SmallVectorImpl<SDValue> &LdChain,
7216	LoadSDNode *LD,
7217	ISD::LoadExtType ExtType) {
7218	// For extension loads, it may not be more efficient to chop up the vector
7219	// and then extend it. Instead, we unroll the load and build a new vector.
7220	EVT WidenVT = TLI.getTypeToTransformTo(Context&: *DAG.getContext(),VT: LD->getValueType(ResNo: 0));
7221	EVT LdVT = LD->getMemoryVT();
7222	SDLoc dl(LD);
7223	assert(LdVT.isVector() && WidenVT.isVector());
7224	assert(LdVT.isScalableVector() == WidenVT.isScalableVector());
7225
7226	// Load information
7227	SDValue Chain = LD->getChain();
7228	SDValue BasePtr = LD->getBasePtr();
7229	MachineMemOperand::Flags MMOFlags = LD->getMemOperand()->getFlags();
7230	AAMDNodes AAInfo = LD->getAAInfo();
7231
7232	if (LdVT.isScalableVector())
7233	report_fatal_error(reason: "Generating widen scalable extending vector loads is "
7234	"not yet supported");
7235
7236	EVT EltVT = WidenVT.getVectorElementType();
7237	EVT LdEltVT = LdVT.getVectorElementType();
7238	unsigned NumElts = LdVT.getVectorNumElements();
7239
7240	// Load each element and widen.
7241	unsigned WidenNumElts = WidenVT.getVectorNumElements();
7242	SmallVector<SDValue, 16> Ops(WidenNumElts);
7243	unsigned Increment = LdEltVT.getSizeInBits() / 8;
7244	Ops[0] =
7245	DAG.getExtLoad(ExtType, dl, VT: EltVT, Chain, Ptr: BasePtr, PtrInfo: LD->getPointerInfo(),
7246	MemVT: LdEltVT, Alignment: LD->getOriginalAlign(), MMOFlags, AAInfo);
7247	LdChain.push_back(Elt: Ops[0].getValue(R: 1));
7248	unsigned i = 0, Offset = Increment;
7249	for (i=1; i < NumElts; ++i, Offset += Increment) {
7250	SDValue NewBasePtr =
7251	DAG.getObjectPtrOffset(SL: dl, Ptr: BasePtr, Offset: TypeSize::getFixed(ExactSize: Offset));
7252	Ops[i] = DAG.getExtLoad(ExtType, dl, VT: EltVT, Chain, Ptr: NewBasePtr,
7253	PtrInfo: LD->getPointerInfo().getWithOffset(O: Offset), MemVT: LdEltVT,
7254	Alignment: LD->getOriginalAlign(), MMOFlags, AAInfo);
7255	LdChain.push_back(Elt: Ops[i].getValue(R: 1));
7256	}
7257
7258	// Fill the rest with undefs.
7259	SDValue UndefVal = DAG.getUNDEF(VT: EltVT);
7260	for (; i != WidenNumElts; ++i)
7261	Ops[i] = UndefVal;
7262
7263	return DAG.getBuildVector(VT: WidenVT, DL: dl, Ops);
7264	}
7265
7266	bool DAGTypeLegalizer::GenWidenVectorStores(SmallVectorImpl<SDValue> &StChain,
7267	StoreSDNode *ST) {
7268	// The strategy assumes that we can efficiently store power-of-two widths.
7269	// The routine chops the vector into the largest vector stores with the same
7270	// element type or scalar stores.
7271	SDValue Chain = ST->getChain();
7272	SDValue BasePtr = ST->getBasePtr();
7273	MachineMemOperand::Flags MMOFlags = ST->getMemOperand()->getFlags();
7274	AAMDNodes AAInfo = ST->getAAInfo();
7275	SDValue ValOp = GetWidenedVector(Op: ST->getValue());
7276	SDLoc dl(ST);
7277
7278	EVT StVT = ST->getMemoryVT();
7279	TypeSize StWidth = StVT.getSizeInBits();
7280	EVT ValVT = ValOp.getValueType();
7281	TypeSize ValWidth = ValVT.getSizeInBits();
7282	EVT ValEltVT = ValVT.getVectorElementType();
7283	unsigned ValEltWidth = ValEltVT.getFixedSizeInBits();
7284	assert(StVT.getVectorElementType() == ValEltVT);
7285	assert(StVT.isScalableVector() == ValVT.isScalableVector() &&
7286	"Mismatch between store and value types");
7287
7288	int Idx = 0; // current index to store
7289
7290	MachinePointerInfo MPI = ST->getPointerInfo();
7291	uint64_t ScaledOffset = 0;
7292
7293	// A breakdown of how to widen this vector store. Each element of the vector
7294	// is a memory VT combined with the number of times it is to be stored to,
7295	// e,g., v5i32 -> {{v2i32,2},{i32,1}}
7296	SmallVector<std::pair<EVT, unsigned>, 4> MemVTs;
7297
7298	while (StWidth.isNonZero()) {
7299	// Find the largest vector type we can store with.
7300	std::optional<EVT> NewVT =
7301	findMemType(DAG, TLI, Width: StWidth.getKnownMinValue(), WidenVT: ValVT);
7302	if (!NewVT)
7303	return false;
7304	MemVTs.push_back(Elt: {*NewVT, 0});
7305	TypeSize NewVTWidth = NewVT->getSizeInBits();
7306
7307	do {
7308	StWidth -= NewVTWidth;
7309	MemVTs.back().second++;
7310	} while (StWidth.isNonZero() && TypeSize::isKnownGE(LHS: StWidth, RHS: NewVTWidth));
7311	}
7312
7313	for (const auto &Pair : MemVTs) {
7314	EVT NewVT = Pair.first;
7315	unsigned Count = Pair.second;
7316	TypeSize NewVTWidth = NewVT.getSizeInBits();
7317
7318	if (NewVT.isVector()) {
7319	unsigned NumVTElts = NewVT.getVectorMinNumElements();
7320	do {
7321	Align NewAlign = ScaledOffset == 0
7322	? ST->getOriginalAlign()
7323	: commonAlignment(A: ST->getAlign(), Offset: ScaledOffset);
7324	SDValue EOp = DAG.getNode(Opcode: ISD::EXTRACT_SUBVECTOR, DL: dl, VT: NewVT, N1: ValOp,
7325	N2: DAG.getVectorIdxConstant(Val: Idx, DL: dl));
7326	SDValue PartStore = DAG.getStore(Chain, dl, Val: EOp, Ptr: BasePtr, PtrInfo: MPI, Alignment: NewAlign,
7327	MMOFlags, AAInfo);
7328	StChain.push_back(Elt: PartStore);
7329
7330	Idx += NumVTElts;
7331	IncrementPointer(N: cast<StoreSDNode>(Val&: PartStore), MemVT: NewVT, MPI, Ptr&: BasePtr,
7332	ScaledOffset: &ScaledOffset);
7333	} while (--Count);
7334	} else {
7335	// Cast the vector to the scalar type we can store.
7336	unsigned NumElts = ValWidth.getFixedValue() / NewVTWidth.getFixedValue();
7337	EVT NewVecVT = EVT::getVectorVT(Context&: *DAG.getContext(), VT: NewVT, NumElements: NumElts);
7338	SDValue VecOp = DAG.getNode(Opcode: ISD::BITCAST, DL: dl, VT: NewVecVT, Operand: ValOp);
7339	// Readjust index position based on new vector type.
7340	Idx = Idx * ValEltWidth / NewVTWidth.getFixedValue();
7341	do {
7342	SDValue EOp = DAG.getNode(Opcode: ISD::EXTRACT_VECTOR_ELT, DL: dl, VT: NewVT, N1: VecOp,
7343	N2: DAG.getVectorIdxConstant(Val: Idx++, DL: dl));
7344	SDValue PartStore =
7345	DAG.getStore(Chain, dl, Val: EOp, Ptr: BasePtr, PtrInfo: MPI, Alignment: ST->getOriginalAlign(),
7346	MMOFlags, AAInfo);
7347	StChain.push_back(Elt: PartStore);
7348
7349	IncrementPointer(N: cast<StoreSDNode>(Val&: PartStore), MemVT: NewVT, MPI, Ptr&: BasePtr);
7350	} while (--Count);
7351	// Restore index back to be relative to the original widen element type.
7352	Idx = Idx * NewVTWidth.getFixedValue() / ValEltWidth;
7353	}
7354	}
7355
7356	return true;
7357	}
7358
7359	/// Modifies a vector input (widen or narrows) to a vector of NVT. The
7360	/// input vector must have the same element type as NVT.
7361	/// FillWithZeroes specifies that the vector should be widened with zeroes.
7362	SDValue DAGTypeLegalizer::ModifyToType(SDValue InOp, EVT NVT,
7363	bool FillWithZeroes) {
7364	// Note that InOp might have been widened so it might already have
7365	// the right width or it might need be narrowed.
7366	EVT InVT = InOp.getValueType();
7367	assert(InVT.getVectorElementType() == NVT.getVectorElementType() &&
7368	"input and widen element type must match");
7369	assert(InVT.isScalableVector() == NVT.isScalableVector() &&
7370	"cannot modify scalable vectors in this way");
7371	SDLoc dl(InOp);
7372
7373	// Check if InOp already has the right width.
7374	if (InVT == NVT)
7375	return InOp;
7376
7377	ElementCount InEC = InVT.getVectorElementCount();
7378	ElementCount WidenEC = NVT.getVectorElementCount();
7379	if (WidenEC.hasKnownScalarFactor(RHS: InEC)) {
7380	unsigned NumConcat = WidenEC.getKnownScalarFactor(RHS: InEC);
7381	SmallVector<SDValue, 16> Ops(NumConcat);
7382	SDValue FillVal = FillWithZeroes ? DAG.getConstant(Val: 0, DL: dl, VT: InVT) :
7383	DAG.getUNDEF(VT: InVT);
7384	Ops[0] = InOp;
7385	for (unsigned i = 1; i != NumConcat; ++i)
7386	Ops[i] = FillVal;
7387
7388	return DAG.getNode(Opcode: ISD::CONCAT_VECTORS, DL: dl, VT: NVT, Ops);
7389	}
7390
7391	if (InEC.hasKnownScalarFactor(RHS: WidenEC))
7392	return DAG.getNode(Opcode: ISD::EXTRACT_SUBVECTOR, DL: dl, VT: NVT, N1: InOp,
7393	N2: DAG.getVectorIdxConstant(Val: 0, DL: dl));
7394
7395	assert(!InVT.isScalableVector() && !NVT.isScalableVector() &&
7396	"Scalable vectors should have been handled already.");
7397
7398	unsigned InNumElts = InEC.getFixedValue();
7399	unsigned WidenNumElts = WidenEC.getFixedValue();
7400
7401	// Fall back to extract and build (+ mask, if padding with zeros).
7402	SmallVector<SDValue, 16> Ops(WidenNumElts);
7403	EVT EltVT = NVT.getVectorElementType();
7404	unsigned MinNumElts = std::min(a: WidenNumElts, b: InNumElts);
7405	unsigned Idx;
7406	for (Idx = 0; Idx < MinNumElts; ++Idx)
7407	Ops[Idx] = DAG.getNode(Opcode: ISD::EXTRACT_VECTOR_ELT, DL: dl, VT: EltVT, N1: InOp,
7408	N2: DAG.getVectorIdxConstant(Val: Idx, DL: dl));
7409
7410	SDValue UndefVal = DAG.getUNDEF(VT: EltVT);
7411	for (; Idx < WidenNumElts; ++Idx)
7412	Ops[Idx] = UndefVal;
7413
7414	SDValue Widened = DAG.getBuildVector(VT: NVT, DL: dl, Ops);
7415	if (!FillWithZeroes)
7416	return Widened;
7417
7418	assert(NVT.isInteger() &&
7419	"We expect to never want to FillWithZeroes for non-integral types.");
7420
7421	SmallVector<SDValue, 16> MaskOps;
7422	MaskOps.append(NumInputs: MinNumElts, Elt: DAG.getAllOnesConstant(DL: dl, VT: EltVT));
7423	MaskOps.append(NumInputs: WidenNumElts - MinNumElts, Elt: DAG.getConstant(Val: 0, DL: dl, VT: EltVT));
7424
7425	return DAG.getNode(Opcode: ISD::AND, DL: dl, VT: NVT, N1: Widened,
7426	N2: DAG.getBuildVector(VT: NVT, DL: dl, Ops: MaskOps));
7427	}
7428