1	//===- LegalizeDAG.cpp - Implement SelectionDAG::Legalize -----------------===//
2	//
3	// Part of the LLVM Project, under the Apache License v2.0 with LLVM Exceptions.
4	// See https://llvm.org/LICENSE.txt for license information.
5	// SPDX-License-Identifier: Apache-2.0 WITH LLVM-exception
6	//
7	//===----------------------------------------------------------------------===//
8	//
9	// This file implements the SelectionDAG::Legalize method.
10	//
11	//===----------------------------------------------------------------------===//
12
13	#include "llvm/ADT/APFloat.h"
14	#include "llvm/ADT/APInt.h"
15	#include "llvm/ADT/ArrayRef.h"
16	#include "llvm/ADT/FloatingPointMode.h"
17	#include "llvm/ADT/SetVector.h"
18	#include "llvm/ADT/SmallPtrSet.h"
19	#include "llvm/ADT/SmallSet.h"
20	#include "llvm/ADT/SmallVector.h"
21	#include "llvm/Analysis/ConstantFolding.h"
22	#include "llvm/Analysis/TargetLibraryInfo.h"
23	#include "llvm/CodeGen/ISDOpcodes.h"
24	#include "llvm/CodeGen/MachineFrameInfo.h"
25	#include "llvm/CodeGen/MachineFunction.h"
26	#include "llvm/CodeGen/MachineJumpTableInfo.h"
27	#include "llvm/CodeGen/MachineMemOperand.h"
28	#include "llvm/CodeGen/RuntimeLibcalls.h"
29	#include "llvm/CodeGen/SelectionDAG.h"
30	#include "llvm/CodeGen/SelectionDAGNodes.h"
31	#include "llvm/CodeGen/TargetFrameLowering.h"
32	#include "llvm/CodeGen/TargetLowering.h"
33	#include "llvm/CodeGen/TargetSubtargetInfo.h"
34	#include "llvm/CodeGen/ValueTypes.h"
35	#include "llvm/CodeGenTypes/MachineValueType.h"
36	#include "llvm/IR/CallingConv.h"
37	#include "llvm/IR/Constants.h"
38	#include "llvm/IR/DataLayout.h"
39	#include "llvm/IR/DerivedTypes.h"
40	#include "llvm/IR/Function.h"
41	#include "llvm/IR/Metadata.h"
42	#include "llvm/IR/Type.h"
43	#include "llvm/Support/Casting.h"
44	#include "llvm/Support/Compiler.h"
45	#include "llvm/Support/Debug.h"
46	#include "llvm/Support/ErrorHandling.h"
47	#include "llvm/Support/MathExtras.h"
48	#include "llvm/Support/raw_ostream.h"
49	#include "llvm/Target/TargetMachine.h"
50	#include "llvm/Target/TargetOptions.h"
51	#include <cassert>
52	#include <cstdint>
53	#include <tuple>
54	#include <utility>
55
56	using namespace llvm;
57
58	#define DEBUG_TYPE "legalizedag"
59
60	namespace {
61
62	/// Keeps track of state when getting the sign of a floating-point value as an
63	/// integer.
64	struct FloatSignAsInt {
65	EVT FloatVT;
66	SDValue Chain;
67	SDValue FloatPtr;
68	SDValue IntPtr;
69	MachinePointerInfo IntPointerInfo;
70	MachinePointerInfo FloatPointerInfo;
71	SDValue IntValue;
72	APInt SignMask;
73	uint8_t SignBit;
74	};
75
76	//===----------------------------------------------------------------------===//
77	/// This takes an arbitrary SelectionDAG as input and
78	/// hacks on it until the target machine can handle it. This involves
79	/// eliminating value sizes the machine cannot handle (promoting small sizes to
80	/// large sizes or splitting up large values into small values) as well as
81	/// eliminating operations the machine cannot handle.
82	///
83	/// This code also does a small amount of optimization and recognition of idioms
84	/// as part of its processing. For example, if a target does not support a
85	/// 'setcc' instruction efficiently, but does support 'brcc' instruction, this
86	/// will attempt merge setcc and brc instructions into brcc's.
87	class SelectionDAGLegalize {
88	const TargetMachine &TM;
89	const TargetLowering &TLI;
90	SelectionDAG &DAG;
91
92	/// The set of nodes which have already been legalized. We hold a
93	/// reference to it in order to update as necessary on node deletion.
94	SmallPtrSetImpl<SDNode *> &LegalizedNodes;
95
96	/// A set of all the nodes updated during legalization.
97	SmallSetVector<SDNode *, 16> *UpdatedNodes;
98
99	EVT getSetCCResultType(EVT VT) const {
100	return TLI.getSetCCResultType(DL: DAG.getDataLayout(), Context&: *DAG.getContext(), VT);
101	}
102
103	// Libcall insertion helpers.
104
105	public:
106	SelectionDAGLegalize(SelectionDAG &DAG,
107	SmallPtrSetImpl<SDNode *> &LegalizedNodes,
108	SmallSetVector<SDNode *, 16> *UpdatedNodes = nullptr)
109	: TM(DAG.getTarget()), TLI(DAG.getTargetLoweringInfo()), DAG(DAG),
110	LegalizedNodes(LegalizedNodes), UpdatedNodes(UpdatedNodes) {}
111
112	/// Legalizes the given operation.
113	void LegalizeOp(SDNode *Node);
114
115	private:
116	SDValue OptimizeFloatStore(StoreSDNode *ST);
117
118	void LegalizeLoadOps(SDNode *Node);
119	void LegalizeStoreOps(SDNode *Node);
120
121	SDValue ExpandINSERT_VECTOR_ELT(SDValue Op);
122
123	/// Return a vector shuffle operation which
124	/// performs the same shuffe in terms of order or result bytes, but on a type
125	/// whose vector element type is narrower than the original shuffle type.
126	/// e.g. <v4i32> <0, 1, 0, 1> -> v8i16 <0, 1, 2, 3, 0, 1, 2, 3>
127	SDValue ShuffleWithNarrowerEltType(EVT NVT, EVT VT, const SDLoc &dl,
128	SDValue N1, SDValue N2,
129	ArrayRef<int> Mask) const;
130
131	std::pair<SDValue, SDValue> ExpandLibCall(RTLIB::Libcall LC, SDNode *Node,
132	TargetLowering::ArgListTy &&Args, bool isSigned);
133	std::pair<SDValue, SDValue> ExpandLibCall(RTLIB::Libcall LC, SDNode *Node, bool isSigned);
134
135	void ExpandFrexpLibCall(SDNode *Node, SmallVectorImpl<SDValue> &Results);
136	void ExpandFPLibCall(SDNode *Node, RTLIB::Libcall LC,
137	SmallVectorImpl<SDValue> &Results);
138	void ExpandFPLibCall(SDNode *Node, RTLIB::Libcall Call_F32,
139	RTLIB::Libcall Call_F64, RTLIB::Libcall Call_F80,
140	RTLIB::Libcall Call_F128,
141	RTLIB::Libcall Call_PPCF128,
142	SmallVectorImpl<SDValue> &Results);
143	SDValue ExpandIntLibCall(SDNode *Node, bool isSigned,
144	RTLIB::Libcall Call_I8,
145	RTLIB::Libcall Call_I16,
146	RTLIB::Libcall Call_I32,
147	RTLIB::Libcall Call_I64,
148	RTLIB::Libcall Call_I128);
149	void ExpandArgFPLibCall(SDNode *Node,
150	RTLIB::Libcall Call_F32, RTLIB::Libcall Call_F64,
151	RTLIB::Libcall Call_F80, RTLIB::Libcall Call_F128,
152	RTLIB::Libcall Call_PPCF128,
153	SmallVectorImpl<SDValue> &Results);
154	void ExpandDivRemLibCall(SDNode *Node, SmallVectorImpl<SDValue> &Results);
155	void ExpandSinCosLibCall(SDNode *Node, SmallVectorImpl<SDValue> &Results);
156
157	SDValue EmitStackConvert(SDValue SrcOp, EVT SlotVT, EVT DestVT,
158	const SDLoc &dl);
159	SDValue EmitStackConvert(SDValue SrcOp, EVT SlotVT, EVT DestVT,
160	const SDLoc &dl, SDValue ChainIn);
161	SDValue ExpandBUILD_VECTOR(SDNode *Node);
162	SDValue ExpandSPLAT_VECTOR(SDNode *Node);
163	SDValue ExpandSCALAR_TO_VECTOR(SDNode *Node);
164	void ExpandDYNAMIC_STACKALLOC(SDNode *Node,
165	SmallVectorImpl<SDValue> &Results);
166	void getSignAsIntValue(FloatSignAsInt &State, const SDLoc &DL,
167	SDValue Value) const;
168	SDValue modifySignAsInt(const FloatSignAsInt &State, const SDLoc &DL,
169	SDValue NewIntValue) const;
170	SDValue ExpandFCOPYSIGN(SDNode *Node) const;
171	SDValue ExpandFABS(SDNode *Node) const;
172	SDValue ExpandFNEG(SDNode *Node) const;
173	SDValue expandLdexp(SDNode *Node) const;
174	SDValue expandFrexp(SDNode *Node) const;
175
176	SDValue ExpandLegalINT_TO_FP(SDNode *Node, SDValue &Chain);
177	void PromoteLegalINT_TO_FP(SDNode *N, const SDLoc &dl,
178	SmallVectorImpl<SDValue> &Results);
179	void PromoteLegalFP_TO_INT(SDNode *N, const SDLoc &dl,
180	SmallVectorImpl<SDValue> &Results);
181	SDValue PromoteLegalFP_TO_INT_SAT(SDNode *Node, const SDLoc &dl);
182
183	SDValue ExpandPARITY(SDValue Op, const SDLoc &dl);
184
185	SDValue ExpandExtractFromVectorThroughStack(SDValue Op);
186	SDValue ExpandInsertToVectorThroughStack(SDValue Op);
187	SDValue ExpandVectorBuildThroughStack(SDNode* Node);
188
189	SDValue ExpandConstantFP(ConstantFPSDNode *CFP, bool UseCP);
190	SDValue ExpandConstant(ConstantSDNode *CP);
191
192	// if ExpandNode returns false, LegalizeOp falls back to ConvertNodeToLibcall
193	bool ExpandNode(SDNode *Node);
194	void ConvertNodeToLibcall(SDNode *Node);
195	void PromoteNode(SDNode *Node);
196
197	public:
198	// Node replacement helpers
199
200	void ReplacedNode(SDNode *N) {
201	LegalizedNodes.erase(Ptr: N);
202	if (UpdatedNodes)
203	UpdatedNodes->insert(X: N);
204	}
205
206	void ReplaceNode(SDNode *Old, SDNode *New) {
207	LLVM_DEBUG(dbgs() << " ... replacing: "; Old->dump(&DAG);
208	dbgs() << " with: "; New->dump(&DAG));
209
210	assert(Old->getNumValues() == New->getNumValues() &&
211	"Replacing one node with another that produces a different number "
212	"of values!");
213	DAG.ReplaceAllUsesWith(From: Old, To: New);
214	if (UpdatedNodes)
215	UpdatedNodes->insert(X: New);
216	ReplacedNode(N: Old);
217	}
218
219	void ReplaceNode(SDValue Old, SDValue New) {
220	LLVM_DEBUG(dbgs() << " ... replacing: "; Old->dump(&DAG);
221	dbgs() << " with: "; New->dump(&DAG));
222
223	DAG.ReplaceAllUsesWith(From: Old, To: New);
224	if (UpdatedNodes)
225	UpdatedNodes->insert(X: New.getNode());
226	ReplacedNode(N: Old.getNode());
227	}
228
229	void ReplaceNode(SDNode *Old, const SDValue *New) {
230	LLVM_DEBUG(dbgs() << " ... replacing: "; Old->dump(&DAG));
231
232	DAG.ReplaceAllUsesWith(From: Old, To: New);
233	for (unsigned i = 0, e = Old->getNumValues(); i != e; ++i) {
234	LLVM_DEBUG(dbgs() << (i == 0 ? " with: " : " and: ");
235	New[i]->dump(&DAG));
236	if (UpdatedNodes)
237	UpdatedNodes->insert(X: New[i].getNode());
238	}
239	ReplacedNode(N: Old);
240	}
241
242	void ReplaceNodeWithValue(SDValue Old, SDValue New) {
243	LLVM_DEBUG(dbgs() << " ... replacing: "; Old->dump(&DAG);
244	dbgs() << " with: "; New->dump(&DAG));
245
246	DAG.ReplaceAllUsesOfValueWith(From: Old, To: New);
247	if (UpdatedNodes)
248	UpdatedNodes->insert(X: New.getNode());
249	ReplacedNode(N: Old.getNode());
250	}
251	};
252
253	} // end anonymous namespace
254
255	// Helper function that generates an MMO that considers the alignment of the
256	// stack, and the size of the stack object
257	static MachineMemOperand *getStackAlignedMMO(SDValue StackPtr,
258	MachineFunction &MF,
259	bool isObjectScalable) {
260	auto &MFI = MF.getFrameInfo();
261	int FI = cast<FrameIndexSDNode>(Val&: StackPtr)->getIndex();
262	MachinePointerInfo PtrInfo = MachinePointerInfo::getFixedStack(MF, FI);
263	LocationSize ObjectSize = isObjectScalable
264	? LocationSize::beforeOrAfterPointer()
265	: LocationSize::precise(Value: MFI.getObjectSize(ObjectIdx: FI));
266	return MF.getMachineMemOperand(PtrInfo, F: MachineMemOperand::MOStore,
267	Size: ObjectSize, BaseAlignment: MFI.getObjectAlign(ObjectIdx: FI));
268	}
269
270	/// Return a vector shuffle operation which
271	/// performs the same shuffle in terms of order or result bytes, but on a type
272	/// whose vector element type is narrower than the original shuffle type.
273	/// e.g. <v4i32> <0, 1, 0, 1> -> v8i16 <0, 1, 2, 3, 0, 1, 2, 3>
274	SDValue SelectionDAGLegalize::ShuffleWithNarrowerEltType(
275	EVT NVT, EVT VT, const SDLoc &dl, SDValue N1, SDValue N2,
276	ArrayRef<int> Mask) const {
277	unsigned NumMaskElts = VT.getVectorNumElements();
278	unsigned NumDestElts = NVT.getVectorNumElements();
279	unsigned NumEltsGrowth = NumDestElts / NumMaskElts;
280
281	assert(NumEltsGrowth && "Cannot promote to vector type with fewer elts!");
282
283	if (NumEltsGrowth == 1)
284	return DAG.getVectorShuffle(VT: NVT, dl, N1, N2, Mask);
285
286	SmallVector<int, 8> NewMask;
287	for (unsigned i = 0; i != NumMaskElts; ++i) {
288	int Idx = Mask[i];
289	for (unsigned j = 0; j != NumEltsGrowth; ++j) {
290	if (Idx < 0)
291	NewMask.push_back(Elt: -1);
292	else
293	NewMask.push_back(Elt: Idx * NumEltsGrowth + j);
294	}
295	}
296	assert(NewMask.size() == NumDestElts && "Non-integer NumEltsGrowth?");
297	assert(TLI.isShuffleMaskLegal(NewMask, NVT) && "Shuffle not legal?");
298	return DAG.getVectorShuffle(VT: NVT, dl, N1, N2, Mask: NewMask);
299	}
300
301	/// Expands the ConstantFP node to an integer constant or
302	/// a load from the constant pool.
303	SDValue
304	SelectionDAGLegalize::ExpandConstantFP(ConstantFPSDNode *CFP, bool UseCP) {
305	bool Extend = false;
306	SDLoc dl(CFP);
307
308	// If a FP immediate is precise when represented as a float and if the
309	// target can do an extending load from float to double, we put it into
310	// the constant pool as a float, even if it's is statically typed as a
311	// double. This shrinks FP constants and canonicalizes them for targets where
312	// an FP extending load is the same cost as a normal load (such as on the x87
313	// fp stack or PPC FP unit).
314	EVT VT = CFP->getValueType(ResNo: 0);
315	ConstantFP *LLVMC = const_cast<ConstantFP*>(CFP->getConstantFPValue());
316	if (!UseCP) {
317	assert((VT == MVT::f64 || VT == MVT::f32) && "Invalid type expansion");
318	return DAG.getConstant(LLVMC->getValueAPF().bitcastToAPInt(), dl,
319	(VT == MVT::f64) ? MVT::i64 : MVT::i32);
320	}
321
322	APFloat APF = CFP->getValueAPF();
323	EVT OrigVT = VT;
324	EVT SVT = VT;
325
326	// We don't want to shrink SNaNs. Converting the SNaN back to its real type
327	// can cause it to be changed into a QNaN on some platforms (e.g. on SystemZ).
328	if (!APF.isSignaling()) {
329	while (SVT != MVT::f32 && SVT != MVT::f16 && SVT != MVT::bf16) {
330	SVT = (MVT::SimpleValueType)(SVT.getSimpleVT().SimpleTy - 1);
331	if (ConstantFPSDNode::isValueValidForType(VT: SVT, Val: APF) &&
332	// Only do this if the target has a native EXTLOAD instruction from
333	// smaller type.
334	TLI.isLoadExtLegal(ExtType: ISD::EXTLOAD, ValVT: OrigVT, MemVT: SVT) &&
335	TLI.ShouldShrinkFPConstant(OrigVT)) {
336	Type *SType = SVT.getTypeForEVT(Context&: *DAG.getContext());
337	LLVMC = cast<ConstantFP>(Val: ConstantFoldCastOperand(
338	Opcode: Instruction::FPTrunc, C: LLVMC, DestTy: SType, DL: DAG.getDataLayout()));
339	VT = SVT;
340	Extend = true;
341	}
342	}
343	}
344
345	SDValue CPIdx =
346	DAG.getConstantPool(C: LLVMC, VT: TLI.getPointerTy(DL: DAG.getDataLayout()));
347	Align Alignment = cast<ConstantPoolSDNode>(Val&: CPIdx)->getAlign();
348	if (Extend) {
349	SDValue Result = DAG.getExtLoad(
350	ExtType: ISD::EXTLOAD, dl, VT: OrigVT, Chain: DAG.getEntryNode(), Ptr: CPIdx,
351	PtrInfo: MachinePointerInfo::getConstantPool(MF&: DAG.getMachineFunction()), MemVT: VT,
352	Alignment);
353	return Result;
354	}
355	SDValue Result = DAG.getLoad(
356	VT: OrigVT, dl, Chain: DAG.getEntryNode(), Ptr: CPIdx,
357	PtrInfo: MachinePointerInfo::getConstantPool(MF&: DAG.getMachineFunction()), Alignment);
358	return Result;
359	}
360
361	/// Expands the Constant node to a load from the constant pool.
362	SDValue SelectionDAGLegalize::ExpandConstant(ConstantSDNode *CP) {
363	SDLoc dl(CP);
364	EVT VT = CP->getValueType(ResNo: 0);
365	SDValue CPIdx = DAG.getConstantPool(C: CP->getConstantIntValue(),
366	VT: TLI.getPointerTy(DL: DAG.getDataLayout()));
367	Align Alignment = cast<ConstantPoolSDNode>(Val&: CPIdx)->getAlign();
368	SDValue Result = DAG.getLoad(
369	VT, dl, Chain: DAG.getEntryNode(), Ptr: CPIdx,
370	PtrInfo: MachinePointerInfo::getConstantPool(MF&: DAG.getMachineFunction()), Alignment);
371	return Result;
372	}
373
374	SDValue SelectionDAGLegalize::ExpandINSERT_VECTOR_ELT(SDValue Op) {
375	SDValue Vec = Op.getOperand(i: 0);
376	SDValue Val = Op.getOperand(i: 1);
377	SDValue Idx = Op.getOperand(i: 2);
378	SDLoc dl(Op);
379
380	if (ConstantSDNode *InsertPos = dyn_cast<ConstantSDNode>(Val&: Idx)) {
381	// SCALAR_TO_VECTOR requires that the type of the value being inserted
382	// match the element type of the vector being created, except for
383	// integers in which case the inserted value can be over width.
384	EVT EltVT = Vec.getValueType().getVectorElementType();
385	if (Val.getValueType() == EltVT ||
386	(EltVT.isInteger() && Val.getValueType().bitsGE(VT: EltVT))) {
387	SDValue ScVec = DAG.getNode(Opcode: ISD::SCALAR_TO_VECTOR, DL: dl,
388	VT: Vec.getValueType(), Operand: Val);
389
390	unsigned NumElts = Vec.getValueType().getVectorNumElements();
391	// We generate a shuffle of InVec and ScVec, so the shuffle mask
392	// should be 0,1,2,3,4,5... with the appropriate element replaced with
393	// elt 0 of the RHS.
394	SmallVector<int, 8> ShufOps;
395	for (unsigned i = 0; i != NumElts; ++i)
396	ShufOps.push_back(Elt: i != InsertPos->getZExtValue() ? i : NumElts);
397
398	return DAG.getVectorShuffle(VT: Vec.getValueType(), dl, N1: Vec, N2: ScVec, Mask: ShufOps);
399	}
400	}
401	return ExpandInsertToVectorThroughStack(Op);
402	}
403
404	SDValue SelectionDAGLegalize::OptimizeFloatStore(StoreSDNode* ST) {
405	if (!ISD::isNormalStore(N: ST))
406	return SDValue();
407
408	LLVM_DEBUG(dbgs() << "Optimizing float store operations\n");
409	// Turn 'store float 1.0, Ptr' -> 'store int 0x12345678, Ptr'
410	// FIXME: move this to the DAG Combiner! Note that we can't regress due
411	// to phase ordering between legalized code and the dag combiner. This
412	// probably means that we need to integrate dag combiner and legalizer
413	// together.
414	// We generally can't do this one for long doubles.
415	SDValue Chain = ST->getChain();
416	SDValue Ptr = ST->getBasePtr();
417	SDValue Value = ST->getValue();
418	MachineMemOperand::Flags MMOFlags = ST->getMemOperand()->getFlags();
419	AAMDNodes AAInfo = ST->getAAInfo();
420	SDLoc dl(ST);
421
422	// Don't optimise TargetConstantFP
423	if (Value.getOpcode() == ISD::TargetConstantFP)
424	return SDValue();
425
426	if (ConstantFPSDNode *CFP = dyn_cast<ConstantFPSDNode>(Val&: Value)) {
427	if (CFP->getValueType(ResNo: 0) == MVT::f32 &&
428	TLI.isTypeLegal(MVT::VT: i32)) {
429	SDValue Con = DAG.getConstant(CFP->getValueAPF().
430	bitcastToAPInt().zextOrTrunc(width: 32),
431	SDLoc(CFP), MVT::i32);
432	return DAG.getStore(Chain, dl, Val: Con, Ptr, PtrInfo: ST->getPointerInfo(),
433	Alignment: ST->getOriginalAlign(), MMOFlags, AAInfo);
434	}
435
436	if (CFP->getValueType(ResNo: 0) == MVT::f64 &&
437	!TLI.isFPImmLegal(CFP->getValueAPF(), MVT::f64)) {
438	// If this target supports 64-bit registers, do a single 64-bit store.
439	if (TLI.isTypeLegal(MVT::VT: i64)) {
440	SDValue Con = DAG.getConstant(CFP->getValueAPF().bitcastToAPInt().
441	zextOrTrunc(width: 64), SDLoc(CFP), MVT::i64);
442	return DAG.getStore(Chain, dl, Val: Con, Ptr, PtrInfo: ST->getPointerInfo(),
443	Alignment: ST->getOriginalAlign(), MMOFlags, AAInfo);
444	}
445
446	if (TLI.isTypeLegal(MVT::VT: i32) && !ST->isVolatile()) {
447	// Otherwise, if the target supports 32-bit registers, use 2 32-bit
448	// stores. If the target supports neither 32- nor 64-bits, this
449	// xform is certainly not worth it.
450	const APInt &IntVal = CFP->getValueAPF().bitcastToAPInt();
451	SDValue Lo = DAG.getConstant(IntVal.trunc(width: 32), dl, MVT::i32);
452	SDValue Hi = DAG.getConstant(IntVal.lshr(shiftAmt: 32).trunc(width: 32), dl, MVT::i32);
453	if (DAG.getDataLayout().isBigEndian())
454	std::swap(a&: Lo, b&: Hi);
455
456	Lo = DAG.getStore(Chain, dl, Val: Lo, Ptr, PtrInfo: ST->getPointerInfo(),
457	Alignment: ST->getOriginalAlign(), MMOFlags, AAInfo);
458	Ptr = DAG.getMemBasePlusOffset(Base: Ptr, Offset: TypeSize::getFixed(ExactSize: 4), DL: dl);
459	Hi = DAG.getStore(Chain, dl, Val: Hi, Ptr,
460	PtrInfo: ST->getPointerInfo().getWithOffset(O: 4),
461	Alignment: ST->getOriginalAlign(), MMOFlags, AAInfo);
462
463	return DAG.getNode(ISD::TokenFactor, dl, MVT::Other, Lo, Hi);
464	}
465	}
466	}
467	return SDValue();
468	}
469
470	void SelectionDAGLegalize::LegalizeStoreOps(SDNode *Node) {
471	StoreSDNode *ST = cast<StoreSDNode>(Val: Node);
472	SDValue Chain = ST->getChain();
473	SDValue Ptr = ST->getBasePtr();
474	SDLoc dl(Node);
475
476	MachineMemOperand::Flags MMOFlags = ST->getMemOperand()->getFlags();
477	AAMDNodes AAInfo = ST->getAAInfo();
478
479	if (!ST->isTruncatingStore()) {
480	LLVM_DEBUG(dbgs() << "Legalizing store operation\n");
481	if (SDNode *OptStore = OptimizeFloatStore(ST).getNode()) {
482	ReplaceNode(Old: ST, New: OptStore);
483	return;
484	}
485
486	SDValue Value = ST->getValue();
487	MVT VT = Value.getSimpleValueType();
488	switch (TLI.getOperationAction(Op: ISD::STORE, VT)) {
489	default: llvm_unreachable("This action is not supported yet!");
490	case TargetLowering::Legal: {
491	// If this is an unaligned store and the target doesn't support it,
492	// expand it.
493	EVT MemVT = ST->getMemoryVT();
494	const DataLayout &DL = DAG.getDataLayout();
495	if (!TLI.allowsMemoryAccessForAlignment(Context&: *DAG.getContext(), DL, VT: MemVT,
496	MMO: *ST->getMemOperand())) {
497	LLVM_DEBUG(dbgs() << "Expanding unsupported unaligned store\n");
498	SDValue Result = TLI.expandUnalignedStore(ST, DAG);
499	ReplaceNode(Old: SDValue(ST, 0), New: Result);
500	} else
501	LLVM_DEBUG(dbgs() << "Legal store\n");
502	break;
503	}
504	case TargetLowering::Custom: {
505	LLVM_DEBUG(dbgs() << "Trying custom lowering\n");
506	SDValue Res = TLI.LowerOperation(Op: SDValue(Node, 0), DAG);
507	if (Res && Res != SDValue(Node, 0))
508	ReplaceNode(Old: SDValue(Node, 0), New: Res);
509	return;
510	}
511	case TargetLowering::Promote: {
512	MVT NVT = TLI.getTypeToPromoteTo(Op: ISD::STORE, VT);
513	assert(NVT.getSizeInBits() == VT.getSizeInBits() &&
514	"Can only promote stores to same size type");
515	Value = DAG.getNode(Opcode: ISD::BITCAST, DL: dl, VT: NVT, Operand: Value);
516	SDValue Result = DAG.getStore(Chain, dl, Val: Value, Ptr, PtrInfo: ST->getPointerInfo(),
517	Alignment: ST->getOriginalAlign(), MMOFlags, AAInfo);
518	ReplaceNode(Old: SDValue(Node, 0), New: Result);
519	break;
520	}
521	}
522	return;
523	}
524
525	LLVM_DEBUG(dbgs() << "Legalizing truncating store operations\n");
526	SDValue Value = ST->getValue();
527	EVT StVT = ST->getMemoryVT();
528	TypeSize StWidth = StVT.getSizeInBits();
529	TypeSize StSize = StVT.getStoreSizeInBits();
530	auto &DL = DAG.getDataLayout();
531
532	if (StWidth != StSize) {
533	// Promote to a byte-sized store with upper bits zero if not
534	// storing an integral number of bytes. For example, promote
535	// TRUNCSTORE:i1 X -> TRUNCSTORE:i8 (and X, 1)
536	EVT NVT = EVT::getIntegerVT(Context&: *DAG.getContext(), BitWidth: StSize.getFixedValue());
537	Value = DAG.getZeroExtendInReg(Op: Value, DL: dl, VT: StVT);
538	SDValue Result =
539	DAG.getTruncStore(Chain, dl, Val: Value, Ptr, PtrInfo: ST->getPointerInfo(), SVT: NVT,
540	Alignment: ST->getOriginalAlign(), MMOFlags, AAInfo);
541	ReplaceNode(Old: SDValue(Node, 0), New: Result);
542	} else if (!StVT.isVector() && !isPowerOf2_64(Value: StWidth.getFixedValue())) {
543	// If not storing a power-of-2 number of bits, expand as two stores.
544	assert(!StVT.isVector() && "Unsupported truncstore!");
545	unsigned StWidthBits = StWidth.getFixedValue();
546	unsigned LogStWidth = Log2_32(Value: StWidthBits);
547	assert(LogStWidth < 32);
548	unsigned RoundWidth = 1 << LogStWidth;
549	assert(RoundWidth < StWidthBits);
550	unsigned ExtraWidth = StWidthBits - RoundWidth;
551	assert(ExtraWidth < RoundWidth);
552	assert(!(RoundWidth % 8) && !(ExtraWidth % 8) &&
553	"Store size not an integral number of bytes!");
554	EVT RoundVT = EVT::getIntegerVT(Context&: *DAG.getContext(), BitWidth: RoundWidth);
555	EVT ExtraVT = EVT::getIntegerVT(Context&: *DAG.getContext(), BitWidth: ExtraWidth);
556	SDValue Lo, Hi;
557	unsigned IncrementSize;
558
559	if (DL.isLittleEndian()) {
560	// TRUNCSTORE:i24 X -> TRUNCSTORE:i16 X, TRUNCSTORE@+2:i8 (srl X, 16)
561	// Store the bottom RoundWidth bits.
562	Lo = DAG.getTruncStore(Chain, dl, Val: Value, Ptr, PtrInfo: ST->getPointerInfo(),
563	SVT: RoundVT, Alignment: ST->getOriginalAlign(), MMOFlags, AAInfo);
564
565	// Store the remaining ExtraWidth bits.
566	IncrementSize = RoundWidth / 8;
567	Ptr =
568	DAG.getMemBasePlusOffset(Base: Ptr, Offset: TypeSize::getFixed(ExactSize: IncrementSize), DL: dl);
569	Hi = DAG.getNode(
570	Opcode: ISD::SRL, DL: dl, VT: Value.getValueType(), N1: Value,
571	N2: DAG.getConstant(Val: RoundWidth, DL: dl,
572	VT: TLI.getShiftAmountTy(LHSTy: Value.getValueType(), DL)));
573	Hi = DAG.getTruncStore(Chain, dl, Val: Hi, Ptr,
574	PtrInfo: ST->getPointerInfo().getWithOffset(O: IncrementSize),
575	SVT: ExtraVT, Alignment: ST->getOriginalAlign(), MMOFlags, AAInfo);
576	} else {
577	// Big endian - avoid unaligned stores.
578	// TRUNCSTORE:i24 X -> TRUNCSTORE:i16 (srl X, 8), TRUNCSTORE@+2:i8 X
579	// Store the top RoundWidth bits.
580	Hi = DAG.getNode(
581	Opcode: ISD::SRL, DL: dl, VT: Value.getValueType(), N1: Value,
582	N2: DAG.getConstant(Val: ExtraWidth, DL: dl,
583	VT: TLI.getShiftAmountTy(LHSTy: Value.getValueType(), DL)));
584	Hi = DAG.getTruncStore(Chain, dl, Val: Hi, Ptr, PtrInfo: ST->getPointerInfo(), SVT: RoundVT,
585	Alignment: ST->getOriginalAlign(), MMOFlags, AAInfo);
586
587	// Store the remaining ExtraWidth bits.
588	IncrementSize = RoundWidth / 8;
589	Ptr = DAG.getNode(Opcode: ISD::ADD, DL: dl, VT: Ptr.getValueType(), N1: Ptr,
590	N2: DAG.getConstant(Val: IncrementSize, DL: dl,
591	VT: Ptr.getValueType()));
592	Lo = DAG.getTruncStore(Chain, dl, Val: Value, Ptr,
593	PtrInfo: ST->getPointerInfo().getWithOffset(O: IncrementSize),
594	SVT: ExtraVT, Alignment: ST->getOriginalAlign(), MMOFlags, AAInfo);
595	}
596
597	// The order of the stores doesn't matter.
598	SDValue Result = DAG.getNode(ISD::TokenFactor, dl, MVT::Other, Lo, Hi);
599	ReplaceNode(Old: SDValue(Node, 0), New: Result);
600	} else {
601	switch (TLI.getTruncStoreAction(ValVT: ST->getValue().getValueType(), MemVT: StVT)) {
602	default: llvm_unreachable("This action is not supported yet!");
603	case TargetLowering::Legal: {
604	EVT MemVT = ST->getMemoryVT();
605	// If this is an unaligned store and the target doesn't support it,
606	// expand it.
607	if (!TLI.allowsMemoryAccessForAlignment(Context&: *DAG.getContext(), DL, VT: MemVT,
608	MMO: *ST->getMemOperand())) {
609	SDValue Result = TLI.expandUnalignedStore(ST, DAG);
610	ReplaceNode(Old: SDValue(ST, 0), New: Result);
611	}
612	break;
613	}
614	case TargetLowering::Custom: {
615	SDValue Res = TLI.LowerOperation(Op: SDValue(Node, 0), DAG);
616	if (Res && Res != SDValue(Node, 0))
617	ReplaceNode(Old: SDValue(Node, 0), New: Res);
618	return;
619	}
620	case TargetLowering::Expand:
621	assert(!StVT.isVector() &&
622	"Vector Stores are handled in LegalizeVectorOps");
623
624	SDValue Result;
625
626	// TRUNCSTORE:i16 i32 -> STORE i16
627	if (TLI.isTypeLegal(VT: StVT)) {
628	Value = DAG.getNode(Opcode: ISD::TRUNCATE, DL: dl, VT: StVT, Operand: Value);
629	Result = DAG.getStore(Chain, dl, Val: Value, Ptr, PtrInfo: ST->getPointerInfo(),
630	Alignment: ST->getOriginalAlign(), MMOFlags, AAInfo);
631	} else {
632	// The in-memory type isn't legal. Truncate to the type it would promote
633	// to, and then do a truncstore.
634	Value = DAG.getNode(Opcode: ISD::TRUNCATE, DL: dl,
635	VT: TLI.getTypeToTransformTo(Context&: *DAG.getContext(), VT: StVT),
636	Operand: Value);
637	Result =
638	DAG.getTruncStore(Chain, dl, Val: Value, Ptr, PtrInfo: ST->getPointerInfo(), SVT: StVT,
639	Alignment: ST->getOriginalAlign(), MMOFlags, AAInfo);
640	}
641
642	ReplaceNode(Old: SDValue(Node, 0), New: Result);
643	break;
644	}
645	}
646	}
647
648	void SelectionDAGLegalize::LegalizeLoadOps(SDNode *Node) {
649	LoadSDNode *LD = cast<LoadSDNode>(Val: Node);
650	SDValue Chain = LD->getChain(); // The chain.
651	SDValue Ptr = LD->getBasePtr(); // The base pointer.
652	SDValue Value; // The value returned by the load op.
653	SDLoc dl(Node);
654
655	ISD::LoadExtType ExtType = LD->getExtensionType();
656	if (ExtType == ISD::NON_EXTLOAD) {
657	LLVM_DEBUG(dbgs() << "Legalizing non-extending load operation\n");
658	MVT VT = Node->getSimpleValueType(ResNo: 0);
659	SDValue RVal = SDValue(Node, 0);
660	SDValue RChain = SDValue(Node, 1);
661
662	switch (TLI.getOperationAction(Op: Node->getOpcode(), VT)) {
663	default: llvm_unreachable("This action is not supported yet!");
664	case TargetLowering::Legal: {
665	EVT MemVT = LD->getMemoryVT();
666	const DataLayout &DL = DAG.getDataLayout();
667	// If this is an unaligned load and the target doesn't support it,
668	// expand it.
669	if (!TLI.allowsMemoryAccessForAlignment(Context&: *DAG.getContext(), DL, VT: MemVT,
670	MMO: *LD->getMemOperand())) {
671	std::tie(args&: RVal, args&: RChain) = TLI.expandUnalignedLoad(LD, DAG);
672	}
673	break;
674	}
675	case TargetLowering::Custom:
676	if (SDValue Res = TLI.LowerOperation(Op: RVal, DAG)) {
677	RVal = Res;
678	RChain = Res.getValue(R: 1);
679	}
680	break;
681
682	case TargetLowering::Promote: {
683	MVT NVT = TLI.getTypeToPromoteTo(Op: Node->getOpcode(), VT);
684	assert(NVT.getSizeInBits() == VT.getSizeInBits() &&
685	"Can only promote loads to same size type");
686
687	SDValue Res = DAG.getLoad(VT: NVT, dl, Chain, Ptr, MMO: LD->getMemOperand());
688	RVal = DAG.getNode(Opcode: ISD::BITCAST, DL: dl, VT, Operand: Res);
689	RChain = Res.getValue(R: 1);
690	break;
691	}
692	}
693	if (RChain.getNode() != Node) {
694	assert(RVal.getNode() != Node && "Load must be completely replaced");
695	DAG.ReplaceAllUsesOfValueWith(From: SDValue(Node, 0), To: RVal);
696	DAG.ReplaceAllUsesOfValueWith(From: SDValue(Node, 1), To: RChain);
697	if (UpdatedNodes) {
698	UpdatedNodes->insert(X: RVal.getNode());
699	UpdatedNodes->insert(X: RChain.getNode());
700	}
701	ReplacedNode(N: Node);
702	}
703	return;
704	}
705
706	LLVM_DEBUG(dbgs() << "Legalizing extending load operation\n");
707	EVT SrcVT = LD->getMemoryVT();
708	TypeSize SrcWidth = SrcVT.getSizeInBits();
709	MachineMemOperand::Flags MMOFlags = LD->getMemOperand()->getFlags();
710	AAMDNodes AAInfo = LD->getAAInfo();
711
712	if (SrcWidth != SrcVT.getStoreSizeInBits() &&
713	// Some targets pretend to have an i1 loading operation, and actually
714	// load an i8. This trick is correct for ZEXTLOAD because the top 7
715	// bits are guaranteed to be zero; it helps the optimizers understand
716	// that these bits are zero. It is also useful for EXTLOAD, since it
717	// tells the optimizers that those bits are undefined. It would be
718	// nice to have an effective generic way of getting these benefits...
719	// Until such a way is found, don't insist on promoting i1 here.
720	(SrcVT != MVT::i1 ||
721	TLI.getLoadExtAction(ExtType, ValVT: Node->getValueType(ResNo: 0), MVT::MemVT: i1) ==
722	TargetLowering::Promote)) {
723	// Promote to a byte-sized load if not loading an integral number of
724	// bytes. For example, promote EXTLOAD:i20 -> EXTLOAD:i24.
725	unsigned NewWidth = SrcVT.getStoreSizeInBits();
726	EVT NVT = EVT::getIntegerVT(Context&: *DAG.getContext(), BitWidth: NewWidth);
727	SDValue Ch;
728
729	// The extra bits are guaranteed to be zero, since we stored them that
730	// way. A zext load from NVT thus automatically gives zext from SrcVT.
731
732	ISD::LoadExtType NewExtType =
733	ExtType == ISD::ZEXTLOAD ? ISD::ZEXTLOAD : ISD::EXTLOAD;
734
735	SDValue Result = DAG.getExtLoad(ExtType: NewExtType, dl, VT: Node->getValueType(ResNo: 0),
736	Chain, Ptr, PtrInfo: LD->getPointerInfo(), MemVT: NVT,
737	Alignment: LD->getOriginalAlign(), MMOFlags, AAInfo);
738
739	Ch = Result.getValue(R: 1); // The chain.
740
741	if (ExtType == ISD::SEXTLOAD)
742	// Having the top bits zero doesn't help when sign extending.
743	Result = DAG.getNode(Opcode: ISD::SIGN_EXTEND_INREG, DL: dl,
744	VT: Result.getValueType(),
745	N1: Result, N2: DAG.getValueType(SrcVT));
746	else if (ExtType == ISD::ZEXTLOAD || NVT == Result.getValueType())
747	// All the top bits are guaranteed to be zero - inform the optimizers.
748	Result = DAG.getNode(Opcode: ISD::AssertZext, DL: dl,
749	VT: Result.getValueType(), N1: Result,
750	N2: DAG.getValueType(SrcVT));
751
752	Value = Result;
753	Chain = Ch;
754	} else if (!isPowerOf2_64(Value: SrcWidth.getKnownMinValue())) {
755	// If not loading a power-of-2 number of bits, expand as two loads.
756	assert(!SrcVT.isVector() && "Unsupported extload!");
757	unsigned SrcWidthBits = SrcWidth.getFixedValue();
758	unsigned LogSrcWidth = Log2_32(Value: SrcWidthBits);
759	assert(LogSrcWidth < 32);
760	unsigned RoundWidth = 1 << LogSrcWidth;
761	assert(RoundWidth < SrcWidthBits);
762	unsigned ExtraWidth = SrcWidthBits - RoundWidth;
763	assert(ExtraWidth < RoundWidth);
764	assert(!(RoundWidth % 8) && !(ExtraWidth % 8) &&
765	"Load size not an integral number of bytes!");
766	EVT RoundVT = EVT::getIntegerVT(Context&: *DAG.getContext(), BitWidth: RoundWidth);
767	EVT ExtraVT = EVT::getIntegerVT(Context&: *DAG.getContext(), BitWidth: ExtraWidth);
768	SDValue Lo, Hi, Ch;
769	unsigned IncrementSize;
770	auto &DL = DAG.getDataLayout();
771
772	if (DL.isLittleEndian()) {
773	// EXTLOAD:i24 -> ZEXTLOAD:i16 | (shl EXTLOAD@+2:i8, 16)
774	// Load the bottom RoundWidth bits.
775	Lo = DAG.getExtLoad(ExtType: ISD::ZEXTLOAD, dl, VT: Node->getValueType(ResNo: 0), Chain, Ptr,
776	PtrInfo: LD->getPointerInfo(), MemVT: RoundVT, Alignment: LD->getOriginalAlign(),
777	MMOFlags, AAInfo);
778
779	// Load the remaining ExtraWidth bits.
780	IncrementSize = RoundWidth / 8;
781	Ptr =
782	DAG.getMemBasePlusOffset(Base: Ptr, Offset: TypeSize::getFixed(ExactSize: IncrementSize), DL: dl);
783	Hi = DAG.getExtLoad(ExtType, dl, VT: Node->getValueType(ResNo: 0), Chain, Ptr,
784	PtrInfo: LD->getPointerInfo().getWithOffset(O: IncrementSize),
785	MemVT: ExtraVT, Alignment: LD->getOriginalAlign(), MMOFlags, AAInfo);
786
787	// Build a factor node to remember that this load is independent of
788	// the other one.
789	Ch = DAG.getNode(ISD::TokenFactor, dl, MVT::Other, Lo.getValue(R: 1),
790	Hi.getValue(R: 1));
791
792	// Move the top bits to the right place.
793	Hi = DAG.getNode(
794	Opcode: ISD::SHL, DL: dl, VT: Hi.getValueType(), N1: Hi,
795	N2: DAG.getConstant(Val: RoundWidth, DL: dl,
796	VT: TLI.getShiftAmountTy(LHSTy: Hi.getValueType(), DL)));
797
798	// Join the hi and lo parts.
799	Value = DAG.getNode(Opcode: ISD::OR, DL: dl, VT: Node->getValueType(ResNo: 0), N1: Lo, N2: Hi);
800	} else {
801	// Big endian - avoid unaligned loads.
802	// EXTLOAD:i24 -> (shl EXTLOAD:i16, 8) | ZEXTLOAD@+2:i8
803	// Load the top RoundWidth bits.
804	Hi = DAG.getExtLoad(ExtType, dl, VT: Node->getValueType(ResNo: 0), Chain, Ptr,
805	PtrInfo: LD->getPointerInfo(), MemVT: RoundVT, Alignment: LD->getOriginalAlign(),
806	MMOFlags, AAInfo);
807
808	// Load the remaining ExtraWidth bits.
809	IncrementSize = RoundWidth / 8;
810	Ptr =
811	DAG.getMemBasePlusOffset(Base: Ptr, Offset: TypeSize::getFixed(ExactSize: IncrementSize), DL: dl);
812	Lo = DAG.getExtLoad(ExtType: ISD::ZEXTLOAD, dl, VT: Node->getValueType(ResNo: 0), Chain, Ptr,
813	PtrInfo: LD->getPointerInfo().getWithOffset(O: IncrementSize),
814	MemVT: ExtraVT, Alignment: LD->getOriginalAlign(), MMOFlags, AAInfo);
815
816	// Build a factor node to remember that this load is independent of
817	// the other one.
818	Ch = DAG.getNode(ISD::TokenFactor, dl, MVT::Other, Lo.getValue(R: 1),
819	Hi.getValue(R: 1));
820
821	// Move the top bits to the right place.
822	Hi = DAG.getNode(
823	Opcode: ISD::SHL, DL: dl, VT: Hi.getValueType(), N1: Hi,
824	N2: DAG.getConstant(Val: ExtraWidth, DL: dl,
825	VT: TLI.getShiftAmountTy(LHSTy: Hi.getValueType(), DL)));
826
827	// Join the hi and lo parts.
828	Value = DAG.getNode(Opcode: ISD::OR, DL: dl, VT: Node->getValueType(ResNo: 0), N1: Lo, N2: Hi);
829	}
830
831	Chain = Ch;
832	} else {
833	bool isCustom = false;
834	switch (TLI.getLoadExtAction(ExtType, ValVT: Node->getValueType(ResNo: 0),
835	MemVT: SrcVT.getSimpleVT())) {
836	default: llvm_unreachable("This action is not supported yet!");
837	case TargetLowering::Custom:
838	isCustom = true;
839	[[fallthrough]];
840	case TargetLowering::Legal:
841	Value = SDValue(Node, 0);
842	Chain = SDValue(Node, 1);
843
844	if (isCustom) {
845	if (SDValue Res = TLI.LowerOperation(Op: SDValue(Node, 0), DAG)) {
846	Value = Res;
847	Chain = Res.getValue(R: 1);
848	}
849	} else {
850	// If this is an unaligned load and the target doesn't support it,
851	// expand it.
852	EVT MemVT = LD->getMemoryVT();
853	const DataLayout &DL = DAG.getDataLayout();
854	if (!TLI.allowsMemoryAccess(Context&: *DAG.getContext(), DL, VT: MemVT,
855	MMO: *LD->getMemOperand())) {
856	std::tie(args&: Value, args&: Chain) = TLI.expandUnalignedLoad(LD, DAG);
857	}
858	}
859	break;
860
861	case TargetLowering::Expand: {
862	EVT DestVT = Node->getValueType(ResNo: 0);
863	if (!TLI.isLoadExtLegal(ExtType: ISD::EXTLOAD, ValVT: DestVT, MemVT: SrcVT)) {
864	// If the source type is not legal, see if there is a legal extload to
865	// an intermediate type that we can then extend further.
866	EVT LoadVT = TLI.getRegisterType(VT: SrcVT.getSimpleVT());
867	if ((LoadVT.isFloatingPoint() == SrcVT.isFloatingPoint()) &&
868	(TLI.isTypeLegal(VT: SrcVT) || // Same as SrcVT == LoadVT?
869	TLI.isLoadExtLegal(ExtType, ValVT: LoadVT, MemVT: SrcVT))) {
870	// If we are loading a legal type, this is a non-extload followed by a
871	// full extend.
872	ISD::LoadExtType MidExtType =
873	(LoadVT == SrcVT) ? ISD::NON_EXTLOAD : ExtType;
874
875	SDValue Load = DAG.getExtLoad(ExtType: MidExtType, dl, VT: LoadVT, Chain, Ptr,
876	MemVT: SrcVT, MMO: LD->getMemOperand());
877	unsigned ExtendOp =
878	ISD::getExtForLoadExtType(IsFP: SrcVT.isFloatingPoint(), ExtType);
879	Value = DAG.getNode(Opcode: ExtendOp, DL: dl, VT: Node->getValueType(ResNo: 0), Operand: Load);
880	Chain = Load.getValue(R: 1);
881	break;
882	}
883
884	// Handle the special case of fp16 extloads. EXTLOAD doesn't have the
885	// normal undefined upper bits behavior to allow using an in-reg extend
886	// with the illegal FP type, so load as an integer and do the
887	// from-integer conversion.
888	EVT SVT = SrcVT.getScalarType();
889	if (SVT == MVT::f16 || SVT == MVT::bf16) {
890	EVT ISrcVT = SrcVT.changeTypeToInteger();
891	EVT IDestVT = DestVT.changeTypeToInteger();
892	EVT ILoadVT = TLI.getRegisterType(VT: IDestVT.getSimpleVT());
893
894	SDValue Result = DAG.getExtLoad(ExtType: ISD::ZEXTLOAD, dl, VT: ILoadVT, Chain,
895	Ptr, MemVT: ISrcVT, MMO: LD->getMemOperand());
896	Value =
897	DAG.getNode(SVT == MVT::f16 ? ISD::FP16_TO_FP : ISD::BF16_TO_FP,
898	dl, DestVT, Result);
899	Chain = Result.getValue(R: 1);
900	break;
901	}
902	}
903
904	assert(!SrcVT.isVector() &&
905	"Vector Loads are handled in LegalizeVectorOps");
906
907	// FIXME: This does not work for vectors on most targets. Sign-
908	// and zero-extend operations are currently folded into extending
909	// loads, whether they are legal or not, and then we end up here
910	// without any support for legalizing them.
911	assert(ExtType != ISD::EXTLOAD &&
912	"EXTLOAD should always be supported!");
913	// Turn the unsupported load into an EXTLOAD followed by an
914	// explicit zero/sign extend inreg.
915	SDValue Result = DAG.getExtLoad(ExtType: ISD::EXTLOAD, dl,
916	VT: Node->getValueType(ResNo: 0),
917	Chain, Ptr, MemVT: SrcVT,
918	MMO: LD->getMemOperand());
919	SDValue ValRes;
920	if (ExtType == ISD::SEXTLOAD)
921	ValRes = DAG.getNode(Opcode: ISD::SIGN_EXTEND_INREG, DL: dl,
922	VT: Result.getValueType(),
923	N1: Result, N2: DAG.getValueType(SrcVT));
924	else
925	ValRes = DAG.getZeroExtendInReg(Op: Result, DL: dl, VT: SrcVT);
926	Value = ValRes;
927	Chain = Result.getValue(R: 1);
928	break;
929	}
930	}
931	}
932
933	// Since loads produce two values, make sure to remember that we legalized
934	// both of them.
935	if (Chain.getNode() != Node) {
936	assert(Value.getNode() != Node && "Load must be completely replaced");
937	DAG.ReplaceAllUsesOfValueWith(From: SDValue(Node, 0), To: Value);
938	DAG.ReplaceAllUsesOfValueWith(From: SDValue(Node, 1), To: Chain);
939	if (UpdatedNodes) {
940	UpdatedNodes->insert(X: Value.getNode());
941	UpdatedNodes->insert(X: Chain.getNode());
942	}
943	ReplacedNode(N: Node);
944	}
945	}
946
947	/// Return a legal replacement for the given operation, with all legal operands.
948	void SelectionDAGLegalize::LegalizeOp(SDNode *Node) {
949	LLVM_DEBUG(dbgs() << "\nLegalizing: "; Node->dump(&DAG));
950
951	// Allow illegal target nodes and illegal registers.
952	if (Node->getOpcode() == ISD::TargetConstant ||
953	Node->getOpcode() == ISD::Register)
954	return;
955
956	#ifndef NDEBUG
957	for (unsigned i = 0, e = Node->getNumValues(); i != e; ++i)
958	assert(TLI.getTypeAction(*DAG.getContext(), Node->getValueType(i)) ==
959	TargetLowering::TypeLegal &&
960	"Unexpected illegal type!");
961
962	for (const SDValue &Op : Node->op_values())
963	assert((TLI.getTypeAction(*DAG.getContext(), Op.getValueType()) ==
964	TargetLowering::TypeLegal ||
965	Op.getOpcode() == ISD::TargetConstant ||
966	Op.getOpcode() == ISD::Register) &&
967	"Unexpected illegal type!");
968	#endif
969
970	// Figure out the correct action; the way to query this varies by opcode
971	TargetLowering::LegalizeAction Action = TargetLowering::Legal;
972	bool SimpleFinishLegalizing = true;
973	switch (Node->getOpcode()) {
974	case ISD::INTRINSIC_W_CHAIN:
975	case ISD::INTRINSIC_WO_CHAIN:
976	case ISD::INTRINSIC_VOID:
977	case ISD::STACKSAVE:
978	Action = TLI.getOperationAction(Op: Node->getOpcode(), MVT::VT: Other);
979	break;
980	case ISD::GET_DYNAMIC_AREA_OFFSET:
981	Action = TLI.getOperationAction(Op: Node->getOpcode(),
982	VT: Node->getValueType(ResNo: 0));
983	break;
984	case ISD::VAARG:
985	Action = TLI.getOperationAction(Op: Node->getOpcode(),
986	VT: Node->getValueType(ResNo: 0));
987	if (Action != TargetLowering::Promote)
988	Action = TLI.getOperationAction(Op: Node->getOpcode(), MVT::VT: Other);
989	break;
990	case ISD::SET_FPENV:
991	case ISD::SET_FPMODE:
992	Action = TLI.getOperationAction(Op: Node->getOpcode(),
993	VT: Node->getOperand(Num: 1).getValueType());
994	break;
995	case ISD::FP_TO_FP16:
996	case ISD::FP_TO_BF16:
997	case ISD::SINT_TO_FP:
998	case ISD::UINT_TO_FP:
999	case ISD::EXTRACT_VECTOR_ELT:
1000	case ISD::LROUND:
1001	case ISD::LLROUND:
1002	case ISD::LRINT:
1003	case ISD::LLRINT:
1004	Action = TLI.getOperationAction(Op: Node->getOpcode(),
1005	VT: Node->getOperand(Num: 0).getValueType());
1006	break;
1007	case ISD::STRICT_FP_TO_FP16:
1008	case ISD::STRICT_FP_TO_BF16:
1009	case ISD::STRICT_SINT_TO_FP:
1010	case ISD::STRICT_UINT_TO_FP:
1011	case ISD::STRICT_LRINT:
1012	case ISD::STRICT_LLRINT:
1013	case ISD::STRICT_LROUND:
1014	case ISD::STRICT_LLROUND:
1015	// These pseudo-ops are the same as the other STRICT_ ops except
1016	// they are registered with setOperationAction() using the input type
1017	// instead of the output type.
1018	Action = TLI.getOperationAction(Op: Node->getOpcode(),
1019	VT: Node->getOperand(Num: 1).getValueType());
1020	break;
1021	case ISD::SIGN_EXTEND_INREG: {
1022	EVT InnerType = cast<VTSDNode>(Val: Node->getOperand(Num: 1))->getVT();
1023	Action = TLI.getOperationAction(Op: Node->getOpcode(), VT: InnerType);
1024	break;
1025	}
1026	case ISD::ATOMIC_STORE:
1027	Action = TLI.getOperationAction(Op: Node->getOpcode(),
1028	VT: Node->getOperand(Num: 1).getValueType());
1029	break;
1030	case ISD::SELECT_CC:
1031	case ISD::STRICT_FSETCC:
1032	case ISD::STRICT_FSETCCS:
1033	case ISD::SETCC:
1034	case ISD::SETCCCARRY:
1035	case ISD::VP_SETCC:
1036	case ISD::BR_CC: {
1037	unsigned Opc = Node->getOpcode();
1038	unsigned CCOperand = Opc == ISD::SELECT_CC ? 4
1039	: Opc == ISD::STRICT_FSETCC ? 3
1040	: Opc == ISD::STRICT_FSETCCS ? 3
1041	: Opc == ISD::SETCCCARRY ? 3
1042	: (Opc == ISD::SETCC || Opc == ISD::VP_SETCC) ? 2
1043	: 1;
1044	unsigned CompareOperand = Opc == ISD::BR_CC ? 2
1045	: Opc == ISD::STRICT_FSETCC ? 1
1046	: Opc == ISD::STRICT_FSETCCS ? 1
1047	: 0;
1048	MVT OpVT = Node->getOperand(Num: CompareOperand).getSimpleValueType();
1049	ISD::CondCode CCCode =
1050	cast<CondCodeSDNode>(Val: Node->getOperand(Num: CCOperand))->get();
1051	Action = TLI.getCondCodeAction(CC: CCCode, VT: OpVT);
1052	if (Action == TargetLowering::Legal) {
1053	if (Node->getOpcode() == ISD::SELECT_CC)
1054	Action = TLI.getOperationAction(Op: Node->getOpcode(),
1055	VT: Node->getValueType(ResNo: 0));
1056	else
1057	Action = TLI.getOperationAction(Op: Node->getOpcode(), VT: OpVT);
1058	}
1059	break;
1060	}
1061	case ISD::LOAD:
1062	case ISD::STORE:
1063	// FIXME: Model these properly. LOAD and STORE are complicated, and
1064	// STORE expects the unlegalized operand in some cases.
1065	SimpleFinishLegalizing = false;
1066	break;
1067	case ISD::CALLSEQ_START:
1068	case ISD::CALLSEQ_END:
1069	// FIXME: This shouldn't be necessary. These nodes have special properties
1070	// dealing with the recursive nature of legalization. Removing this
1071	// special case should be done as part of making LegalizeDAG non-recursive.
1072	SimpleFinishLegalizing = false;
1073	break;
1074	case ISD::EXTRACT_ELEMENT:
1075	case ISD::GET_ROUNDING:
1076	case ISD::MERGE_VALUES:
1077	case ISD::EH_RETURN:
1078	case ISD::FRAME_TO_ARGS_OFFSET:
1079	case ISD::EH_DWARF_CFA:
1080	case ISD::EH_SJLJ_SETJMP:
1081	case ISD::EH_SJLJ_LONGJMP:
1082	case ISD::EH_SJLJ_SETUP_DISPATCH:
1083	// These operations lie about being legal: when they claim to be legal,
1084	// they should actually be expanded.
1085	Action = TLI.getOperationAction(Op: Node->getOpcode(), VT: Node->getValueType(ResNo: 0));
1086	if (Action == TargetLowering::Legal)
1087	Action = TargetLowering::Expand;
1088	break;
1089	case ISD::INIT_TRAMPOLINE:
1090	case ISD::ADJUST_TRAMPOLINE:
1091	case ISD::FRAMEADDR:
1092	case ISD::RETURNADDR:
1093	case ISD::ADDROFRETURNADDR:
1094	case ISD::SPONENTRY:
1095	// These operations lie about being legal: when they claim to be legal,
1096	// they should actually be custom-lowered.
1097	Action = TLI.getOperationAction(Op: Node->getOpcode(), VT: Node->getValueType(ResNo: 0));
1098	if (Action == TargetLowering::Legal)
1099	Action = TargetLowering::Custom;
1100	break;
1101	case ISD::READCYCLECOUNTER:
1102	case ISD::READSTEADYCOUNTER:
1103	// READCYCLECOUNTER and READSTEADYCOUNTER return a i64, even if type
1104	// legalization might have expanded that to several smaller types.
1105	Action = TLI.getOperationAction(Op: Node->getOpcode(), MVT::VT: i64);
1106	break;
1107	case ISD::READ_REGISTER:
1108	case ISD::WRITE_REGISTER:
1109	// Named register is legal in the DAG, but blocked by register name
1110	// selection if not implemented by target (to chose the correct register)
1111	// They'll be converted to Copy(To/From)Reg.
1112	Action = TargetLowering::Legal;
1113	break;
1114	case ISD::UBSANTRAP:
1115	Action = TLI.getOperationAction(Op: Node->getOpcode(), VT: Node->getValueType(ResNo: 0));
1116	if (Action == TargetLowering::Expand) {
1117	// replace ISD::UBSANTRAP with ISD::TRAP
1118	SDValue NewVal;
1119	NewVal = DAG.getNode(Opcode: ISD::TRAP, DL: SDLoc(Node), VTList: Node->getVTList(),
1120	N: Node->getOperand(Num: 0));
1121	ReplaceNode(Old: Node, New: NewVal.getNode());
1122	LegalizeOp(Node: NewVal.getNode());
1123	return;
1124	}
1125	break;
1126	case ISD::DEBUGTRAP:
1127	Action = TLI.getOperationAction(Op: Node->getOpcode(), VT: Node->getValueType(ResNo: 0));
1128	if (Action == TargetLowering::Expand) {
1129	// replace ISD::DEBUGTRAP with ISD::TRAP
1130	SDValue NewVal;
1131	NewVal = DAG.getNode(Opcode: ISD::TRAP, DL: SDLoc(Node), VTList: Node->getVTList(),
1132	N: Node->getOperand(Num: 0));
1133	ReplaceNode(Old: Node, New: NewVal.getNode());
1134	LegalizeOp(Node: NewVal.getNode());
1135	return;
1136	}
1137	break;
1138	case ISD::SADDSAT:
1139	case ISD::UADDSAT:
1140	case ISD::SSUBSAT:
1141	case ISD::USUBSAT:
1142	case ISD::SSHLSAT:
1143	case ISD::USHLSAT:
1144	case ISD::FP_TO_SINT_SAT:
1145	case ISD::FP_TO_UINT_SAT:
1146	Action = TLI.getOperationAction(Op: Node->getOpcode(), VT: Node->getValueType(ResNo: 0));
1147	break;
1148	case ISD::SMULFIX:
1149	case ISD::SMULFIXSAT:
1150	case ISD::UMULFIX:
1151	case ISD::UMULFIXSAT:
1152	case ISD::SDIVFIX:
1153	case ISD::SDIVFIXSAT:
1154	case ISD::UDIVFIX:
1155	case ISD::UDIVFIXSAT: {
1156	unsigned Scale = Node->getConstantOperandVal(Num: 2);
1157	Action = TLI.getFixedPointOperationAction(Op: Node->getOpcode(),
1158	VT: Node->getValueType(ResNo: 0), Scale);
1159	break;
1160	}
1161	case ISD::MSCATTER:
1162	Action = TLI.getOperationAction(Op: Node->getOpcode(),
1163	VT: cast<MaskedScatterSDNode>(Val: Node)->getValue().getValueType());
1164	break;
1165	case ISD::MSTORE:
1166	Action = TLI.getOperationAction(Op: Node->getOpcode(),
1167	VT: cast<MaskedStoreSDNode>(Val: Node)->getValue().getValueType());
1168	break;
1169	case ISD::VP_SCATTER:
1170	Action = TLI.getOperationAction(
1171	Op: Node->getOpcode(),
1172	VT: cast<VPScatterSDNode>(Val: Node)->getValue().getValueType());
1173	break;
1174	case ISD::VP_STORE:
1175	Action = TLI.getOperationAction(
1176	Op: Node->getOpcode(),
1177	VT: cast<VPStoreSDNode>(Val: Node)->getValue().getValueType());
1178	break;
1179	case ISD::EXPERIMENTAL_VP_STRIDED_STORE:
1180	Action = TLI.getOperationAction(
1181	Op: Node->getOpcode(),
1182	VT: cast<VPStridedStoreSDNode>(Val: Node)->getValue().getValueType());
1183	break;
1184	case ISD::VECREDUCE_FADD:
1185	case ISD::VECREDUCE_FMUL:
1186	case ISD::VECREDUCE_ADD:
1187	case ISD::VECREDUCE_MUL:
1188	case ISD::VECREDUCE_AND:
1189	case ISD::VECREDUCE_OR:
1190	case ISD::VECREDUCE_XOR:
1191	case ISD::VECREDUCE_SMAX:
1192	case ISD::VECREDUCE_SMIN:
1193	case ISD::VECREDUCE_UMAX:
1194	case ISD::VECREDUCE_UMIN:
1195	case ISD::VECREDUCE_FMAX:
1196	case ISD::VECREDUCE_FMIN:
1197	case ISD::VECREDUCE_FMAXIMUM:
1198	case ISD::VECREDUCE_FMINIMUM:
1199	case ISD::IS_FPCLASS:
1200	Action = TLI.getOperationAction(
1201	Op: Node->getOpcode(), VT: Node->getOperand(Num: 0).getValueType());
1202	break;
1203	case ISD::VECREDUCE_SEQ_FADD:
1204	case ISD::VECREDUCE_SEQ_FMUL:
1205	case ISD::VP_REDUCE_FADD:
1206	case ISD::VP_REDUCE_FMUL:
1207	case ISD::VP_REDUCE_ADD:
1208	case ISD::VP_REDUCE_MUL:
1209	case ISD::VP_REDUCE_AND:
1210	case ISD::VP_REDUCE_OR:
1211	case ISD::VP_REDUCE_XOR:
1212	case ISD::VP_REDUCE_SMAX:
1213	case ISD::VP_REDUCE_SMIN:
1214	case ISD::VP_REDUCE_UMAX:
1215	case ISD::VP_REDUCE_UMIN:
1216	case ISD::VP_REDUCE_FMAX:
1217	case ISD::VP_REDUCE_FMIN:
1218	case ISD::VP_REDUCE_SEQ_FADD:
1219	case ISD::VP_REDUCE_SEQ_FMUL:
1220	Action = TLI.getOperationAction(
1221	Op: Node->getOpcode(), VT: Node->getOperand(Num: 1).getValueType());
1222	break;
1223	default:
1224	if (Node->getOpcode() >= ISD::BUILTIN_OP_END) {
1225	Action = TLI.getCustomOperationAction(Op&: *Node);
1226	} else {
1227	Action = TLI.getOperationAction(Op: Node->getOpcode(), VT: Node->getValueType(ResNo: 0));
1228	}
1229	break;
1230	}
1231
1232	if (SimpleFinishLegalizing) {
1233	SDNode *NewNode = Node;
1234	switch (Node->getOpcode()) {
1235	default: break;
1236	case ISD::SHL:
1237	case ISD::SRL:
1238	case ISD::SRA:
1239	case ISD::ROTL:
1240	case ISD::ROTR: {
1241	// Legalizing shifts/rotates requires adjusting the shift amount
1242	// to the appropriate width.
1243	SDValue Op0 = Node->getOperand(Num: 0);
1244	SDValue Op1 = Node->getOperand(Num: 1);
1245	if (!Op1.getValueType().isVector()) {
1246	SDValue SAO = DAG.getShiftAmountOperand(LHSTy: Op0.getValueType(), Op: Op1);
1247	// The getShiftAmountOperand() may create a new operand node or
1248	// return the existing one. If new operand is created we need
1249	// to update the parent node.
1250	// Do not try to legalize SAO here! It will be automatically legalized
1251	// in the next round.
1252	if (SAO != Op1)
1253	NewNode = DAG.UpdateNodeOperands(N: Node, Op1: Op0, Op2: SAO);
1254	}
1255	}
1256	break;
1257	case ISD::FSHL:
1258	case ISD::FSHR:
1259	case ISD::SRL_PARTS:
1260	case ISD::SRA_PARTS:
1261	case ISD::SHL_PARTS: {
1262	// Legalizing shifts/rotates requires adjusting the shift amount
1263	// to the appropriate width.
1264	SDValue Op0 = Node->getOperand(Num: 0);
1265	SDValue Op1 = Node->getOperand(Num: 1);
1266	SDValue Op2 = Node->getOperand(Num: 2);
1267	if (!Op2.getValueType().isVector()) {
1268	SDValue SAO = DAG.getShiftAmountOperand(LHSTy: Op0.getValueType(), Op: Op2);
1269	// The getShiftAmountOperand() may create a new operand node or
1270	// return the existing one. If new operand is created we need
1271	// to update the parent node.
1272	if (SAO != Op2)
1273	NewNode = DAG.UpdateNodeOperands(N: Node, Op1: Op0, Op2: Op1, Op3: SAO);
1274	}
1275	break;
1276	}
1277	}
1278
1279	if (NewNode != Node) {
1280	ReplaceNode(Old: Node, New: NewNode);
1281	Node = NewNode;
1282	}
1283	switch (Action) {
1284	case TargetLowering::Legal:
1285	LLVM_DEBUG(dbgs() << "Legal node: nothing to do\n");
1286	return;
1287	case TargetLowering::Custom:
1288	LLVM_DEBUG(dbgs() << "Trying custom legalization\n");
1289	// FIXME: The handling for custom lowering with multiple results is
1290	// a complete mess.
1291	if (SDValue Res = TLI.LowerOperation(Op: SDValue(Node, 0), DAG)) {
1292	if (!(Res.getNode() != Node || Res.getResNo() != 0))
1293	return;
1294
1295	if (Node->getNumValues() == 1) {
1296	// Verify the new types match the original. Glue is waived because
1297	// ISD::ADDC can be legalized by replacing Glue with an integer type.
1298	assert((Res.getValueType() == Node->getValueType(0) ||
1299	Node->getValueType(0) == MVT::Glue) &&
1300	"Type mismatch for custom legalized operation");
1301	LLVM_DEBUG(dbgs() << "Successfully custom legalized node\n");
1302	// We can just directly replace this node with the lowered value.
1303	ReplaceNode(Old: SDValue(Node, 0), New: Res);
1304	return;
1305	}
1306
1307	SmallVector<SDValue, 8> ResultVals;
1308	for (unsigned i = 0, e = Node->getNumValues(); i != e; ++i) {
1309	// Verify the new types match the original. Glue is waived because
1310	// ISD::ADDC can be legalized by replacing Glue with an integer type.
1311	assert((Res->getValueType(i) == Node->getValueType(i) ||
1312	Node->getValueType(i) == MVT::Glue) &&
1313	"Type mismatch for custom legalized operation");
1314	ResultVals.push_back(Elt: Res.getValue(R: i));
1315	}
1316	LLVM_DEBUG(dbgs() << "Successfully custom legalized node\n");
1317	ReplaceNode(Old: Node, New: ResultVals.data());
1318	return;
1319	}
1320	LLVM_DEBUG(dbgs() << "Could not custom legalize node\n");
1321	[[fallthrough]];
1322	case TargetLowering::Expand:
1323	if (ExpandNode(Node))
1324	return;
1325	[[fallthrough]];
1326	case TargetLowering::LibCall:
1327	ConvertNodeToLibcall(Node);
1328	return;
1329	case TargetLowering::Promote:
1330	PromoteNode(Node);
1331	return;
1332	}
1333	}
1334
1335	switch (Node->getOpcode()) {
1336	default:
1337	#ifndef NDEBUG
1338	dbgs() << "NODE: ";
1339	Node->dump( G: &DAG);
1340	dbgs() << "\n";
1341	#endif
1342	llvm_unreachable("Do not know how to legalize this operator!");
1343
1344	case ISD::CALLSEQ_START:
1345	case ISD::CALLSEQ_END:
1346	break;
1347	case ISD::LOAD:
1348	return LegalizeLoadOps(Node);
1349	case ISD::STORE:
1350	return LegalizeStoreOps(Node);
1351	}
1352	}
1353
1354	SDValue SelectionDAGLegalize::ExpandExtractFromVectorThroughStack(SDValue Op) {
1355	SDValue Vec = Op.getOperand(i: 0);
1356	SDValue Idx = Op.getOperand(i: 1);
1357	SDLoc dl(Op);
1358
1359	// Before we generate a new store to a temporary stack slot, see if there is
1360	// already one that we can use. There often is because when we scalarize
1361	// vector operations (using SelectionDAG::UnrollVectorOp for example) a whole
1362	// series of EXTRACT_VECTOR_ELT nodes are generated, one for each element in
1363	// the vector. If all are expanded here, we don't want one store per vector
1364	// element.
1365
1366	// Caches for hasPredecessorHelper
1367	SmallPtrSet<const SDNode *, 32> Visited;
1368	SmallVector<const SDNode *, 16> Worklist;
1369	Visited.insert(Ptr: Op.getNode());
1370	Worklist.push_back(Elt: Idx.getNode());
1371	SDValue StackPtr, Ch;
1372	for (SDNode *User : Vec.getNode()->uses()) {
1373	if (StoreSDNode *ST = dyn_cast<StoreSDNode>(Val: User)) {
1374	if (ST->isIndexed() || ST->isTruncatingStore() ||
1375	ST->getValue() != Vec)
1376	continue;
1377
1378	// Make sure that nothing else could have stored into the destination of
1379	// this store.
1380	if (!ST->getChain().reachesChainWithoutSideEffects(Dest: DAG.getEntryNode()))
1381	continue;
1382
1383	// If the index is dependent on the store we will introduce a cycle when
1384	// creating the load (the load uses the index, and by replacing the chain
1385	// we will make the index dependent on the load). Also, the store might be
1386	// dependent on the extractelement and introduce a cycle when creating
1387	// the load.
1388	if (SDNode::hasPredecessorHelper(N: ST, Visited, Worklist) ||
1389	ST->hasPredecessor(N: Op.getNode()))
1390	continue;
1391
1392	StackPtr = ST->getBasePtr();
1393	Ch = SDValue(ST, 0);
1394	break;
1395	}
1396	}
1397
1398	EVT VecVT = Vec.getValueType();
1399
1400	if (!Ch.getNode()) {
1401	// Store the value to a temporary stack slot, then LOAD the returned part.
1402	StackPtr = DAG.CreateStackTemporary(VT: VecVT);
1403	MachineMemOperand *StoreMMO = getStackAlignedMMO(
1404	StackPtr, MF&: DAG.getMachineFunction(), isObjectScalable: VecVT.isScalableVector());
1405	Ch = DAG.getStore(Chain: DAG.getEntryNode(), dl, Val: Vec, Ptr: StackPtr, MMO: StoreMMO);
1406	}
1407
1408	SDValue NewLoad;
1409	Align ElementAlignment =
1410	std::min(a: cast<StoreSDNode>(Val&: Ch)->getAlign(),
1411	b: DAG.getDataLayout().getPrefTypeAlign(
1412	Ty: Op.getValueType().getTypeForEVT(Context&: *DAG.getContext())));
1413
1414	if (Op.getValueType().isVector()) {
1415	StackPtr = TLI.getVectorSubVecPointer(DAG, VecPtr: StackPtr, VecVT,
1416	SubVecVT: Op.getValueType(), Index: Idx);
1417	NewLoad = DAG.getLoad(VT: Op.getValueType(), dl, Chain: Ch, Ptr: StackPtr,
1418	PtrInfo: MachinePointerInfo(), Alignment: ElementAlignment);
1419	} else {
1420	StackPtr = TLI.getVectorElementPointer(DAG, VecPtr: StackPtr, VecVT, Index: Idx);
1421	NewLoad = DAG.getExtLoad(ExtType: ISD::EXTLOAD, dl, VT: Op.getValueType(), Chain: Ch, Ptr: StackPtr,
1422	PtrInfo: MachinePointerInfo(), MemVT: VecVT.getVectorElementType(),
1423	Alignment: ElementAlignment);
1424	}
1425
1426	// Replace the chain going out of the store, by the one out of the load.
1427	DAG.ReplaceAllUsesOfValueWith(From: Ch, To: SDValue(NewLoad.getNode(), 1));
1428
1429	// We introduced a cycle though, so update the loads operands, making sure
1430	// to use the original store's chain as an incoming chain.
1431	SmallVector<SDValue, 6> NewLoadOperands(NewLoad->op_begin(),
1432	NewLoad->op_end());
1433	NewLoadOperands[0] = Ch;
1434	NewLoad =
1435	SDValue(DAG.UpdateNodeOperands(N: NewLoad.getNode(), Ops: NewLoadOperands), 0);
1436	return NewLoad;
1437	}
1438
1439	SDValue SelectionDAGLegalize::ExpandInsertToVectorThroughStack(SDValue Op) {
1440	assert(Op.getValueType().isVector() && "Non-vector insert subvector!");
1441
1442	SDValue Vec = Op.getOperand(i: 0);
1443	SDValue Part = Op.getOperand(i: 1);
1444	SDValue Idx = Op.getOperand(i: 2);
1445	SDLoc dl(Op);
1446
1447	// Store the value to a temporary stack slot, then LOAD the returned part.
1448	EVT VecVT = Vec.getValueType();
1449	EVT PartVT = Part.getValueType();
1450	SDValue StackPtr = DAG.CreateStackTemporary(VT: VecVT);
1451	int FI = cast<FrameIndexSDNode>(Val: StackPtr.getNode())->getIndex();
1452	MachinePointerInfo PtrInfo =
1453	MachinePointerInfo::getFixedStack(MF&: DAG.getMachineFunction(), FI);
1454
1455	// First store the whole vector.
1456	SDValue Ch = DAG.getStore(Chain: DAG.getEntryNode(), dl, Val: Vec, Ptr: StackPtr, PtrInfo);
1457
1458	// Freeze the index so we don't poison the clamping code we're about to emit.
1459	Idx = DAG.getFreeze(V: Idx);
1460
1461	// Then store the inserted part.
1462	if (PartVT.isVector()) {
1463	SDValue SubStackPtr =
1464	TLI.getVectorSubVecPointer(DAG, VecPtr: StackPtr, VecVT, SubVecVT: PartVT, Index: Idx);
1465
1466	// Store the subvector.
1467	Ch = DAG.getStore(
1468	Chain: Ch, dl, Val: Part, Ptr: SubStackPtr,
1469	PtrInfo: MachinePointerInfo::getUnknownStack(MF&: DAG.getMachineFunction()));
1470	} else {
1471	SDValue SubStackPtr =
1472	TLI.getVectorElementPointer(DAG, VecPtr: StackPtr, VecVT, Index: Idx);
1473
1474	// Store the scalar value.
1475	Ch = DAG.getTruncStore(
1476	Chain: Ch, dl, Val: Part, Ptr: SubStackPtr,
1477	PtrInfo: MachinePointerInfo::getUnknownStack(MF&: DAG.getMachineFunction()),
1478	SVT: VecVT.getVectorElementType());
1479	}
1480
1481	// Finally, load the updated vector.
1482	return DAG.getLoad(VT: Op.getValueType(), dl, Chain: Ch, Ptr: StackPtr, PtrInfo);
1483	}
1484
1485	SDValue SelectionDAGLegalize::ExpandVectorBuildThroughStack(SDNode* Node) {
1486	assert((Node->getOpcode() == ISD::BUILD_VECTOR ||
1487	Node->getOpcode() == ISD::CONCAT_VECTORS) &&
1488	"Unexpected opcode!");
1489
1490	// We can't handle this case efficiently. Allocate a sufficiently
1491	// aligned object on the stack, store each operand into it, then load
1492	// the result as a vector.
1493	// Create the stack frame object.
1494	EVT VT = Node->getValueType(ResNo: 0);
1495	EVT MemVT = isa<BuildVectorSDNode>(Val: Node) ? VT.getVectorElementType()
1496	: Node->getOperand(Num: 0).getValueType();
1497	SDLoc dl(Node);
1498	SDValue FIPtr = DAG.CreateStackTemporary(VT);
1499	int FI = cast<FrameIndexSDNode>(Val: FIPtr.getNode())->getIndex();
1500	MachinePointerInfo PtrInfo =
1501	MachinePointerInfo::getFixedStack(MF&: DAG.getMachineFunction(), FI);
1502
1503	// Emit a store of each element to the stack slot.
1504	SmallVector<SDValue, 8> Stores;
1505	unsigned TypeByteSize = MemVT.getSizeInBits() / 8;
1506	assert(TypeByteSize > 0 && "Vector element type too small for stack store!");
1507
1508	// If the destination vector element type of a BUILD_VECTOR is narrower than
1509	// the source element type, only store the bits necessary.
1510	bool Truncate = isa<BuildVectorSDNode>(Val: Node) &&
1511	MemVT.bitsLT(VT: Node->getOperand(Num: 0).getValueType());
1512
1513	// Store (in the right endianness) the elements to memory.
1514	for (unsigned i = 0, e = Node->getNumOperands(); i != e; ++i) {
1515	// Ignore undef elements.
1516	if (Node->getOperand(Num: i).isUndef()) continue;
1517
1518	unsigned Offset = TypeByteSize*i;
1519
1520	SDValue Idx =
1521	DAG.getMemBasePlusOffset(Base: FIPtr, Offset: TypeSize::getFixed(ExactSize: Offset), DL: dl);
1522
1523	if (Truncate)
1524	Stores.push_back(Elt: DAG.getTruncStore(Chain: DAG.getEntryNode(), dl,
1525	Val: Node->getOperand(Num: i), Ptr: Idx,
1526	PtrInfo: PtrInfo.getWithOffset(O: Offset), SVT: MemVT));
1527	else
1528	Stores.push_back(Elt: DAG.getStore(Chain: DAG.getEntryNode(), dl, Val: Node->getOperand(Num: i),
1529	Ptr: Idx, PtrInfo: PtrInfo.getWithOffset(O: Offset)));
1530	}
1531
1532	SDValue StoreChain;
1533	if (!Stores.empty()) // Not all undef elements?
1534	StoreChain = DAG.getNode(ISD::TokenFactor, dl, MVT::Other, Stores);
1535	else
1536	StoreChain = DAG.getEntryNode();
1537
1538	// Result is a load from the stack slot.
1539	return DAG.getLoad(VT, dl, Chain: StoreChain, Ptr: FIPtr, PtrInfo);
1540	}
1541
1542	/// Bitcast a floating-point value to an integer value. Only bitcast the part
1543	/// containing the sign bit if the target has no integer value capable of
1544	/// holding all bits of the floating-point value.
1545	void SelectionDAGLegalize::getSignAsIntValue(FloatSignAsInt &State,
1546	const SDLoc &DL,
1547	SDValue Value) const {
1548	EVT FloatVT = Value.getValueType();
1549	unsigned NumBits = FloatVT.getScalarSizeInBits();
1550	State.FloatVT = FloatVT;
1551	EVT IVT = EVT::getIntegerVT(Context&: *DAG.getContext(), BitWidth: NumBits);
1552	// Convert to an integer of the same size.
1553	if (TLI.isTypeLegal(VT: IVT)) {
1554	State.IntValue = DAG.getNode(Opcode: ISD::BITCAST, DL, VT: IVT, Operand: Value);
1555	State.SignMask = APInt::getSignMask(BitWidth: NumBits);
1556	State.SignBit = NumBits - 1;
1557	return;
1558	}
1559
1560	auto &DataLayout = DAG.getDataLayout();
1561	// Store the float to memory, then load the sign part out as an integer.
1562	MVT LoadTy = TLI.getRegisterType(MVT::i8);
1563	// First create a temporary that is aligned for both the load and store.
1564	SDValue StackPtr = DAG.CreateStackTemporary(VT1: FloatVT, VT2: LoadTy);
1565	int FI = cast<FrameIndexSDNode>(Val: StackPtr.getNode())->getIndex();
1566	// Then store the float to it.
1567	State.FloatPtr = StackPtr;
1568	MachineFunction &MF = DAG.getMachineFunction();
1569	State.FloatPointerInfo = MachinePointerInfo::getFixedStack(MF, FI);
1570	State.Chain = DAG.getStore(Chain: DAG.getEntryNode(), dl: DL, Val: Value, Ptr: State.FloatPtr,
1571	PtrInfo: State.FloatPointerInfo);
1572
1573	SDValue IntPtr;
1574	if (DataLayout.isBigEndian()) {
1575	assert(FloatVT.isByteSized() && "Unsupported floating point type!");
1576	// Load out a legal integer with the same sign bit as the float.
1577	IntPtr = StackPtr;
1578	State.IntPointerInfo = State.FloatPointerInfo;
1579	} else {
1580	// Advance the pointer so that the loaded byte will contain the sign bit.
1581	unsigned ByteOffset = (NumBits / 8) - 1;
1582	IntPtr =
1583	DAG.getMemBasePlusOffset(Base: StackPtr, Offset: TypeSize::getFixed(ExactSize: ByteOffset), DL);
1584	State.IntPointerInfo = MachinePointerInfo::getFixedStack(MF, FI,
1585	Offset: ByteOffset);
1586	}
1587
1588	State.IntPtr = IntPtr;
1589	State.IntValue = DAG.getExtLoad(ISD::EXTLOAD, DL, LoadTy, State.Chain, IntPtr,
1590	State.IntPointerInfo, MVT::i8);
1591	State.SignMask = APInt::getOneBitSet(numBits: LoadTy.getScalarSizeInBits(), BitNo: 7);
1592	State.SignBit = 7;
1593	}
1594
1595	/// Replace the integer value produced by getSignAsIntValue() with a new value
1596	/// and cast the result back to a floating-point type.
1597	SDValue SelectionDAGLegalize::modifySignAsInt(const FloatSignAsInt &State,
1598	const SDLoc &DL,
1599	SDValue NewIntValue) const {
1600	if (!State.Chain)
1601	return DAG.getNode(Opcode: ISD::BITCAST, DL, VT: State.FloatVT, Operand: NewIntValue);
1602
1603	// Override the part containing the sign bit in the value stored on the stack.
1604	SDValue Chain = DAG.getTruncStore(State.Chain, DL, NewIntValue, State.IntPtr,
1605	State.IntPointerInfo, MVT::i8);
1606	return DAG.getLoad(VT: State.FloatVT, dl: DL, Chain, Ptr: State.FloatPtr,
1607	PtrInfo: State.FloatPointerInfo);
1608	}
1609
1610	SDValue SelectionDAGLegalize::ExpandFCOPYSIGN(SDNode *Node) const {
1611	SDLoc DL(Node);
1612	SDValue Mag = Node->getOperand(Num: 0);
1613	SDValue Sign = Node->getOperand(Num: 1);
1614
1615	// Get sign bit into an integer value.
1616	FloatSignAsInt SignAsInt;
1617	getSignAsIntValue(State&: SignAsInt, DL, Value: Sign);
1618
1619	EVT IntVT = SignAsInt.IntValue.getValueType();
1620	SDValue SignMask = DAG.getConstant(Val: SignAsInt.SignMask, DL, VT: IntVT);
1621	SDValue SignBit = DAG.getNode(Opcode: ISD::AND, DL, VT: IntVT, N1: SignAsInt.IntValue,
1622	N2: SignMask);
1623
1624	// If FABS is legal transform FCOPYSIGN(x, y) => sign(x) ? -FABS(x) : FABS(X)
1625	EVT FloatVT = Mag.getValueType();
1626	if (TLI.isOperationLegalOrCustom(Op: ISD::FABS, VT: FloatVT) &&
1627	TLI.isOperationLegalOrCustom(Op: ISD::FNEG, VT: FloatVT)) {
1628	SDValue AbsValue = DAG.getNode(Opcode: ISD::FABS, DL, VT: FloatVT, Operand: Mag);
1629	SDValue NegValue = DAG.getNode(Opcode: ISD::FNEG, DL, VT: FloatVT, Operand: AbsValue);
1630	SDValue Cond = DAG.getSetCC(DL, VT: getSetCCResultType(VT: IntVT), LHS: SignBit,
1631	RHS: DAG.getConstant(Val: 0, DL, VT: IntVT), Cond: ISD::SETNE);
1632	return DAG.getSelect(DL, VT: FloatVT, Cond, LHS: NegValue, RHS: AbsValue);
1633	}
1634
1635	// Transform Mag value to integer, and clear the sign bit.
1636	FloatSignAsInt MagAsInt;
1637	getSignAsIntValue(State&: MagAsInt, DL, Value: Mag);
1638	EVT MagVT = MagAsInt.IntValue.getValueType();
1639	SDValue ClearSignMask = DAG.getConstant(Val: ~MagAsInt.SignMask, DL, VT: MagVT);
1640	SDValue ClearedSign = DAG.getNode(Opcode: ISD::AND, DL, VT: MagVT, N1: MagAsInt.IntValue,
1641	N2: ClearSignMask);
1642
1643	// Get the signbit at the right position for MagAsInt.
1644	int ShiftAmount = SignAsInt.SignBit - MagAsInt.SignBit;
1645	EVT ShiftVT = IntVT;
1646	if (SignBit.getScalarValueSizeInBits() <
1647	ClearedSign.getScalarValueSizeInBits()) {
1648	SignBit = DAG.getNode(Opcode: ISD::ZERO_EXTEND, DL, VT: MagVT, Operand: SignBit);
1649	ShiftVT = MagVT;
1650	}
1651	if (ShiftAmount > 0) {
1652	SDValue ShiftCnst = DAG.getConstant(Val: ShiftAmount, DL, VT: ShiftVT);
1653	SignBit = DAG.getNode(Opcode: ISD::SRL, DL, VT: ShiftVT, N1: SignBit, N2: ShiftCnst);
1654	} else if (ShiftAmount < 0) {
1655	SDValue ShiftCnst = DAG.getConstant(Val: -ShiftAmount, DL, VT: ShiftVT);
1656	SignBit = DAG.getNode(Opcode: ISD::SHL, DL, VT: ShiftVT, N1: SignBit, N2: ShiftCnst);
1657	}
1658	if (SignBit.getScalarValueSizeInBits() >
1659	ClearedSign.getScalarValueSizeInBits()) {
1660	SignBit = DAG.getNode(Opcode: ISD::TRUNCATE, DL, VT: MagVT, Operand: SignBit);
1661	}
1662
1663	// Store the part with the modified sign and convert back to float.
1664	SDValue CopiedSign = DAG.getNode(Opcode: ISD::OR, DL, VT: MagVT, N1: ClearedSign, N2: SignBit);
1665	return modifySignAsInt(State: MagAsInt, DL, NewIntValue: CopiedSign);
1666	}
1667
1668	SDValue SelectionDAGLegalize::ExpandFNEG(SDNode *Node) const {
1669	// Get the sign bit as an integer.
1670	SDLoc DL(Node);
1671	FloatSignAsInt SignAsInt;
1672	getSignAsIntValue(State&: SignAsInt, DL, Value: Node->getOperand(Num: 0));
1673	EVT IntVT = SignAsInt.IntValue.getValueType();
1674
1675	// Flip the sign.
1676	SDValue SignMask = DAG.getConstant(Val: SignAsInt.SignMask, DL, VT: IntVT);
1677	SDValue SignFlip =
1678	DAG.getNode(Opcode: ISD::XOR, DL, VT: IntVT, N1: SignAsInt.IntValue, N2: SignMask);
1679
1680	// Convert back to float.
1681	return modifySignAsInt(State: SignAsInt, DL, NewIntValue: SignFlip);
1682	}
1683
1684	SDValue SelectionDAGLegalize::ExpandFABS(SDNode *Node) const {
1685	SDLoc DL(Node);
1686	SDValue Value = Node->getOperand(Num: 0);
1687
1688	// Transform FABS(x) => FCOPYSIGN(x, 0.0) if FCOPYSIGN is legal.
1689	EVT FloatVT = Value.getValueType();
1690	if (TLI.isOperationLegalOrCustom(Op: ISD::FCOPYSIGN, VT: FloatVT)) {
1691	SDValue Zero = DAG.getConstantFP(Val: 0.0, DL, VT: FloatVT);
1692	return DAG.getNode(Opcode: ISD::FCOPYSIGN, DL, VT: FloatVT, N1: Value, N2: Zero);
1693	}
1694
1695	// Transform value to integer, clear the sign bit and transform back.
1696	FloatSignAsInt ValueAsInt;
1697	getSignAsIntValue(State&: ValueAsInt, DL, Value);
1698	EVT IntVT = ValueAsInt.IntValue.getValueType();
1699	SDValue ClearSignMask = DAG.getConstant(Val: ~ValueAsInt.SignMask, DL, VT: IntVT);
1700	SDValue ClearedSign = DAG.getNode(Opcode: ISD::AND, DL, VT: IntVT, N1: ValueAsInt.IntValue,
1701	N2: ClearSignMask);
1702	return modifySignAsInt(State: ValueAsInt, DL, NewIntValue: ClearedSign);
1703	}
1704
1705	void SelectionDAGLegalize::ExpandDYNAMIC_STACKALLOC(SDNode* Node,
1706	SmallVectorImpl<SDValue> &Results) {
1707	Register SPReg = TLI.getStackPointerRegisterToSaveRestore();
1708	assert(SPReg && "Target cannot require DYNAMIC_STACKALLOC expansion and"
1709	" not tell us which reg is the stack pointer!");
1710	SDLoc dl(Node);
1711	EVT VT = Node->getValueType(ResNo: 0);
1712	SDValue Tmp1 = SDValue(Node, 0);
1713	SDValue Tmp2 = SDValue(Node, 1);
1714	SDValue Tmp3 = Node->getOperand(Num: 2);
1715	SDValue Chain = Tmp1.getOperand(i: 0);
1716
1717	// Chain the dynamic stack allocation so that it doesn't modify the stack
1718	// pointer when other instructions are using the stack.
1719	Chain = DAG.getCALLSEQ_START(Chain, InSize: 0, OutSize: 0, DL: dl);
1720
1721	SDValue Size = Tmp2.getOperand(i: 1);
1722	SDValue SP = DAG.getCopyFromReg(Chain, dl, Reg: SPReg, VT);
1723	Chain = SP.getValue(R: 1);
1724	Align Alignment = cast<ConstantSDNode>(Val&: Tmp3)->getAlignValue();
1725	const TargetFrameLowering *TFL = DAG.getSubtarget().getFrameLowering();
1726	unsigned Opc =
1727	TFL->getStackGrowthDirection() == TargetFrameLowering::StackGrowsUp ?
1728	ISD::ADD : ISD::SUB;
1729
1730	Align StackAlign = TFL->getStackAlign();
1731	Tmp1 = DAG.getNode(Opcode: Opc, DL: dl, VT, N1: SP, N2: Size); // Value
1732	if (Alignment > StackAlign)
1733	Tmp1 = DAG.getNode(Opcode: ISD::AND, DL: dl, VT, N1: Tmp1,
1734	N2: DAG.getConstant(Val: -Alignment.value(), DL: dl, VT));
1735	Chain = DAG.getCopyToReg(Chain, dl, Reg: SPReg, N: Tmp1); // Output chain
1736
1737	Tmp2 = DAG.getCALLSEQ_END(Chain, Size1: 0, Size2: 0, Glue: SDValue(), DL: dl);
1738
1739	Results.push_back(Elt: Tmp1);
1740	Results.push_back(Elt: Tmp2);
1741	}
1742
1743	/// Emit a store/load combination to the stack. This stores
1744	/// SrcOp to a stack slot of type SlotVT, truncating it if needed. It then does
1745	/// a load from the stack slot to DestVT, extending it if needed.
1746	/// The resultant code need not be legal.
1747	SDValue SelectionDAGLegalize::EmitStackConvert(SDValue SrcOp, EVT SlotVT,
1748	EVT DestVT, const SDLoc &dl) {
1749	return EmitStackConvert(SrcOp, SlotVT, DestVT, dl, ChainIn: DAG.getEntryNode());
1750	}
1751
1752	SDValue SelectionDAGLegalize::EmitStackConvert(SDValue SrcOp, EVT SlotVT,
1753	EVT DestVT, const SDLoc &dl,
1754	SDValue Chain) {
1755	EVT SrcVT = SrcOp.getValueType();
1756	Type *DestType = DestVT.getTypeForEVT(Context&: *DAG.getContext());
1757	Align DestAlign = DAG.getDataLayout().getPrefTypeAlign(Ty: DestType);
1758
1759	// Don't convert with stack if the load/store is expensive.
1760	if ((SrcVT.bitsGT(VT: SlotVT) &&
1761	!TLI.isTruncStoreLegalOrCustom(ValVT: SrcOp.getValueType(), MemVT: SlotVT)) ||
1762	(SlotVT.bitsLT(VT: DestVT) &&
1763	!TLI.isLoadExtLegalOrCustom(ExtType: ISD::EXTLOAD, ValVT: DestVT, MemVT: SlotVT)))
1764	return SDValue();
1765
1766	// Create the stack frame object.
1767	Align SrcAlign = DAG.getDataLayout().getPrefTypeAlign(
1768	Ty: SrcOp.getValueType().getTypeForEVT(Context&: *DAG.getContext()));
1769	SDValue FIPtr = DAG.CreateStackTemporary(Bytes: SlotVT.getStoreSize(), Alignment: SrcAlign);
1770
1771	FrameIndexSDNode *StackPtrFI = cast<FrameIndexSDNode>(Val&: FIPtr);
1772	int SPFI = StackPtrFI->getIndex();
1773	MachinePointerInfo PtrInfo =
1774	MachinePointerInfo::getFixedStack(MF&: DAG.getMachineFunction(), FI: SPFI);
1775
1776	// Emit a store to the stack slot. Use a truncstore if the input value is
1777	// later than DestVT.
1778	SDValue Store;
1779
1780	if (SrcVT.bitsGT(VT: SlotVT))
1781	Store = DAG.getTruncStore(Chain, dl, Val: SrcOp, Ptr: FIPtr, PtrInfo,
1782	SVT: SlotVT, Alignment: SrcAlign);
1783	else {
1784	assert(SrcVT.bitsEq(SlotVT) && "Invalid store");
1785	Store = DAG.getStore(Chain, dl, Val: SrcOp, Ptr: FIPtr, PtrInfo, Alignment: SrcAlign);
1786	}
1787
1788	// Result is a load from the stack slot.
1789	if (SlotVT.bitsEq(VT: DestVT))
1790	return DAG.getLoad(VT: DestVT, dl, Chain: Store, Ptr: FIPtr, PtrInfo, Alignment: DestAlign);
1791
1792	assert(SlotVT.bitsLT(DestVT) && "Unknown extension!");
1793	return DAG.getExtLoad(ExtType: ISD::EXTLOAD, dl, VT: DestVT, Chain: Store, Ptr: FIPtr, PtrInfo, MemVT: SlotVT,
1794	Alignment: DestAlign);
1795	}
1796
1797	SDValue SelectionDAGLegalize::ExpandSCALAR_TO_VECTOR(SDNode *Node) {
1798	SDLoc dl(Node);
1799	// Create a vector sized/aligned stack slot, store the value to element #0,
1800	// then load the whole vector back out.
1801	SDValue StackPtr = DAG.CreateStackTemporary(VT: Node->getValueType(ResNo: 0));
1802
1803	FrameIndexSDNode *StackPtrFI = cast<FrameIndexSDNode>(Val&: StackPtr);
1804	int SPFI = StackPtrFI->getIndex();
1805
1806	SDValue Ch = DAG.getTruncStore(
1807	Chain: DAG.getEntryNode(), dl, Val: Node->getOperand(Num: 0), Ptr: StackPtr,
1808	PtrInfo: MachinePointerInfo::getFixedStack(MF&: DAG.getMachineFunction(), FI: SPFI),
1809	SVT: Node->getValueType(ResNo: 0).getVectorElementType());
1810	return DAG.getLoad(
1811	VT: Node->getValueType(ResNo: 0), dl, Chain: Ch, Ptr: StackPtr,
1812	PtrInfo: MachinePointerInfo::getFixedStack(MF&: DAG.getMachineFunction(), FI: SPFI));
1813	}
1814
1815	static bool
1816	ExpandBVWithShuffles(SDNode *Node, SelectionDAG &DAG,
1817	const TargetLowering &TLI, SDValue &Res) {
1818	unsigned NumElems = Node->getNumOperands();
1819	SDLoc dl(Node);
1820	EVT VT = Node->getValueType(ResNo: 0);
1821
1822	// Try to group the scalars into pairs, shuffle the pairs together, then
1823	// shuffle the pairs of pairs together, etc. until the vector has
1824	// been built. This will work only if all of the necessary shuffle masks
1825	// are legal.
1826
1827	// We do this in two phases; first to check the legality of the shuffles,
1828	// and next, assuming that all shuffles are legal, to create the new nodes.
1829	for (int Phase = 0; Phase < 2; ++Phase) {
1830	SmallVector<std::pair<SDValue, SmallVector<int, 16>>, 16> IntermedVals,
1831	NewIntermedVals;
1832	for (unsigned i = 0; i < NumElems; ++i) {
1833	SDValue V = Node->getOperand(Num: i);
1834	if (V.isUndef())
1835	continue;
1836
1837	SDValue Vec;
1838	if (Phase)
1839	Vec = DAG.getNode(Opcode: ISD::SCALAR_TO_VECTOR, DL: dl, VT, Operand: V);
1840	IntermedVals.push_back(Elt: std::make_pair(x&: Vec, y: SmallVector<int, 16>(1, i)));
1841	}
1842
1843	while (IntermedVals.size() > 2) {
1844	NewIntermedVals.clear();
1845	for (unsigned i = 0, e = (IntermedVals.size() & ~1u); i < e; i += 2) {
1846	// This vector and the next vector are shuffled together (simply to
1847	// append the one to the other).
1848	SmallVector<int, 16> ShuffleVec(NumElems, -1);
1849
1850	SmallVector<int, 16> FinalIndices;
1851	FinalIndices.reserve(N: IntermedVals[i].second.size() +
1852	IntermedVals[i+1].second.size());
1853
1854	int k = 0;
1855	for (unsigned j = 0, f = IntermedVals[i].second.size(); j != f;
1856	++j, ++k) {
1857	ShuffleVec[k] = j;
1858	FinalIndices.push_back(Elt: IntermedVals[i].second[j]);
1859	}
1860	for (unsigned j = 0, f = IntermedVals[i+1].second.size(); j != f;
1861	++j, ++k) {
1862	ShuffleVec[k] = NumElems + j;
1863	FinalIndices.push_back(Elt: IntermedVals[i+1].second[j]);
1864	}
1865
1866	SDValue Shuffle;
1867	if (Phase)
1868	Shuffle = DAG.getVectorShuffle(VT, dl, N1: IntermedVals[i].first,
1869	N2: IntermedVals[i+1].first,
1870	Mask: ShuffleVec);
1871	else if (!TLI.isShuffleMaskLegal(ShuffleVec, VT))
1872	return false;
1873	NewIntermedVals.push_back(
1874	Elt: std::make_pair(x&: Shuffle, y: std::move(FinalIndices)));
1875	}
1876
1877	// If we had an odd number of defined values, then append the last
1878	// element to the array of new vectors.
1879	if ((IntermedVals.size() & 1) != 0)
1880	NewIntermedVals.push_back(Elt: IntermedVals.back());
1881
1882	IntermedVals.swap(RHS&: NewIntermedVals);
1883	}
1884
1885	assert(IntermedVals.size() <= 2 && IntermedVals.size() > 0 &&
1886	"Invalid number of intermediate vectors");
1887	SDValue Vec1 = IntermedVals[0].first;
1888	SDValue Vec2;
1889	if (IntermedVals.size() > 1)
1890	Vec2 = IntermedVals[1].first;
1891	else if (Phase)
1892	Vec2 = DAG.getUNDEF(VT);
1893
1894	SmallVector<int, 16> ShuffleVec(NumElems, -1);
1895	for (unsigned i = 0, e = IntermedVals[0].second.size(); i != e; ++i)
1896	ShuffleVec[IntermedVals[0].second[i]] = i;
1897	for (unsigned i = 0, e = IntermedVals[1].second.size(); i != e; ++i)
1898	ShuffleVec[IntermedVals[1].second[i]] = NumElems + i;
1899
1900	if (Phase)
1901	Res = DAG.getVectorShuffle(VT, dl, N1: Vec1, N2: Vec2, Mask: ShuffleVec);
1902	else if (!TLI.isShuffleMaskLegal(ShuffleVec, VT))
1903	return false;
1904	}
1905
1906	return true;
1907	}
1908
1909	/// Expand a BUILD_VECTOR node on targets that don't
1910	/// support the operation, but do support the resultant vector type.
1911	SDValue SelectionDAGLegalize::ExpandBUILD_VECTOR(SDNode *Node) {
1912	unsigned NumElems = Node->getNumOperands();
1913	SDValue Value1, Value2;
1914	SDLoc dl(Node);
1915	EVT VT = Node->getValueType(ResNo: 0);
1916	EVT OpVT = Node->getOperand(Num: 0).getValueType();
1917	EVT EltVT = VT.getVectorElementType();
1918
1919	// If the only non-undef value is the low element, turn this into a
1920	// SCALAR_TO_VECTOR node. If this is { X, X, X, X }, determine X.
1921	bool isOnlyLowElement = true;
1922	bool MoreThanTwoValues = false;
1923	bool isConstant = true;
1924	for (unsigned i = 0; i < NumElems; ++i) {
1925	SDValue V = Node->getOperand(Num: i);
1926	if (V.isUndef())
1927	continue;
1928	if (i > 0)
1929	isOnlyLowElement = false;
1930	if (!isa<ConstantFPSDNode>(Val: V) && !isa<ConstantSDNode>(Val: V))
1931	isConstant = false;
1932
1933	if (!Value1.getNode()) {
1934	Value1 = V;
1935	} else if (!Value2.getNode()) {
1936	if (V != Value1)
1937	Value2 = V;
1938	} else if (V != Value1 && V != Value2) {
1939	MoreThanTwoValues = true;
1940	}
1941	}
1942
1943	if (!Value1.getNode())
1944	return DAG.getUNDEF(VT);
1945
1946	if (isOnlyLowElement)
1947	return DAG.getNode(Opcode: ISD::SCALAR_TO_VECTOR, DL: dl, VT, Operand: Node->getOperand(Num: 0));
1948
1949	// If all elements are constants, create a load from the constant pool.
1950	if (isConstant) {
1951	SmallVector<Constant*, 16> CV;
1952	for (unsigned i = 0, e = NumElems; i != e; ++i) {
1953	if (ConstantFPSDNode *V =
1954	dyn_cast<ConstantFPSDNode>(Val: Node->getOperand(Num: i))) {
1955	CV.push_back(Elt: const_cast<ConstantFP *>(V->getConstantFPValue()));
1956	} else if (ConstantSDNode *V =
1957	dyn_cast<ConstantSDNode>(Val: Node->getOperand(Num: i))) {
1958	if (OpVT==EltVT)
1959	CV.push_back(Elt: const_cast<ConstantInt *>(V->getConstantIntValue()));
1960	else {
1961	// If OpVT and EltVT don't match, EltVT is not legal and the
1962	// element values have been promoted/truncated earlier. Undo this;
1963	// we don't want a v16i8 to become a v16i32 for example.
1964	const ConstantInt *CI = V->getConstantIntValue();
1965	CV.push_back(Elt: ConstantInt::get(Ty: EltVT.getTypeForEVT(Context&: *DAG.getContext()),
1966	V: CI->getZExtValue()));
1967	}
1968	} else {
1969	assert(Node->getOperand(i).isUndef());
1970	Type *OpNTy = EltVT.getTypeForEVT(Context&: *DAG.getContext());
1971	CV.push_back(Elt: UndefValue::get(T: OpNTy));
1972	}
1973	}
1974	Constant *CP = ConstantVector::get(V: CV);
1975	SDValue CPIdx =
1976	DAG.getConstantPool(C: CP, VT: TLI.getPointerTy(DL: DAG.getDataLayout()));
1977	Align Alignment = cast<ConstantPoolSDNode>(Val&: CPIdx)->getAlign();
1978	return DAG.getLoad(
1979	VT, dl, Chain: DAG.getEntryNode(), Ptr: CPIdx,
1980	PtrInfo: MachinePointerInfo::getConstantPool(MF&: DAG.getMachineFunction()),
1981	Alignment);
1982	}
1983
1984	SmallSet<SDValue, 16> DefinedValues;
1985	for (unsigned i = 0; i < NumElems; ++i) {
1986	if (Node->getOperand(Num: i).isUndef())
1987	continue;
1988	DefinedValues.insert(V: Node->getOperand(Num: i));
1989	}
1990
1991	if (TLI.shouldExpandBuildVectorWithShuffles(VT, DefinedValues: DefinedValues.size())) {
1992	if (!MoreThanTwoValues) {
1993	SmallVector<int, 8> ShuffleVec(NumElems, -1);
1994	for (unsigned i = 0; i < NumElems; ++i) {
1995	SDValue V = Node->getOperand(Num: i);
1996	if (V.isUndef())
1997	continue;
1998	ShuffleVec[i] = V == Value1 ? 0 : NumElems;
1999	}
2000	if (TLI.isShuffleMaskLegal(ShuffleVec, Node->getValueType(ResNo: 0))) {
2001	// Get the splatted value into the low element of a vector register.
2002	SDValue Vec1 = DAG.getNode(Opcode: ISD::SCALAR_TO_VECTOR, DL: dl, VT, Operand: Value1);
2003	SDValue Vec2;
2004	if (Value2.getNode())
2005	Vec2 = DAG.getNode(Opcode: ISD::SCALAR_TO_VECTOR, DL: dl, VT, Operand: Value2);
2006	else
2007	Vec2 = DAG.getUNDEF(VT);
2008
2009	// Return shuffle(LowValVec, undef, <0,0,0,0>)
2010	return DAG.getVectorShuffle(VT, dl, N1: Vec1, N2: Vec2, Mask: ShuffleVec);
2011	}
2012	} else {
2013	SDValue Res;
2014	if (ExpandBVWithShuffles(Node, DAG, TLI, Res))
2015	return Res;
2016	}
2017	}
2018
2019	// Otherwise, we can't handle this case efficiently.
2020	return ExpandVectorBuildThroughStack(Node);
2021	}
2022
2023	SDValue SelectionDAGLegalize::ExpandSPLAT_VECTOR(SDNode *Node) {
2024	SDLoc DL(Node);
2025	EVT VT = Node->getValueType(ResNo: 0);
2026	SDValue SplatVal = Node->getOperand(Num: 0);
2027
2028	return DAG.getSplatBuildVector(VT, DL, Op: SplatVal);
2029	}
2030
2031	// Expand a node into a call to a libcall, returning the value as the first
2032	// result and the chain as the second. If the result value does not fit into a
2033	// register, return the lo part and set the hi part to the by-reg argument in
2034	// the first. If it does fit into a single register, return the result and
2035	// leave the Hi part unset.
2036	std::pair<SDValue, SDValue> SelectionDAGLegalize::ExpandLibCall(RTLIB::Libcall LC, SDNode *Node,
2037	TargetLowering::ArgListTy &&Args,
2038	bool isSigned) {
2039	SDValue Callee = DAG.getExternalSymbol(Sym: TLI.getLibcallName(Call: LC),
2040	VT: TLI.getPointerTy(DL: DAG.getDataLayout()));
2041
2042	EVT RetVT = Node->getValueType(ResNo: 0);
2043	Type *RetTy = RetVT.getTypeForEVT(Context&: *DAG.getContext());
2044
2045	// By default, the input chain to this libcall is the entry node of the
2046	// function. If the libcall is going to be emitted as a tail call then
2047	// TLI.isUsedByReturnOnly will change it to the right chain if the return
2048	// node which is being folded has a non-entry input chain.
2049	SDValue InChain = DAG.getEntryNode();
2050
2051	// isTailCall may be true since the callee does not reference caller stack
2052	// frame. Check if it's in the right position and that the return types match.
2053	SDValue TCChain = InChain;
2054	const Function &F = DAG.getMachineFunction().getFunction();
2055	bool isTailCall =
2056	TLI.isInTailCallPosition(DAG, Node, Chain&: TCChain) &&
2057	(RetTy == F.getReturnType() || F.getReturnType()->isVoidTy());
2058	if (isTailCall)
2059	InChain = TCChain;
2060
2061	TargetLowering::CallLoweringInfo CLI(DAG);
2062	bool signExtend = TLI.shouldSignExtendTypeInLibCall(Type: RetVT, IsSigned: isSigned);
2063	CLI.setDebugLoc(SDLoc(Node))
2064	.setChain(InChain)
2065	.setLibCallee(CC: TLI.getLibcallCallingConv(Call: LC), ResultType: RetTy, Target: Callee,
2066	ArgsList: std::move(Args))
2067	.setTailCall(isTailCall)
2068	.setSExtResult(signExtend)
2069	.setZExtResult(!signExtend)
2070	.setIsPostTypeLegalization(true);
2071
2072	std::pair<SDValue, SDValue> CallInfo = TLI.LowerCallTo(CLI);
2073
2074	if (!CallInfo.second.getNode()) {
2075	LLVM_DEBUG(dbgs() << "Created tailcall: "; DAG.getRoot().dump(&DAG));
2076	// It's a tailcall, return the chain (which is the DAG root).
2077	return {DAG.getRoot(), DAG.getRoot()};
2078	}
2079
2080	LLVM_DEBUG(dbgs() << "Created libcall: "; CallInfo.first.dump(&DAG));
2081	return CallInfo;
2082	}
2083
2084	std::pair<SDValue, SDValue> SelectionDAGLegalize::ExpandLibCall(RTLIB::Libcall LC, SDNode *Node,
2085	bool isSigned) {
2086	TargetLowering::ArgListTy Args;
2087	TargetLowering::ArgListEntry Entry;
2088	for (const SDValue &Op : Node->op_values()) {
2089	EVT ArgVT = Op.getValueType();
2090	Type *ArgTy = ArgVT.getTypeForEVT(Context&: *DAG.getContext());
2091	Entry.Node = Op;
2092	Entry.Ty = ArgTy;
2093	Entry.IsSExt = TLI.shouldSignExtendTypeInLibCall(Type: ArgVT, IsSigned: isSigned);
2094	Entry.IsZExt = !Entry.IsSExt;
2095	Args.push_back(x: Entry);
2096	}
2097
2098	return ExpandLibCall(LC, Node, Args: std::move(Args), isSigned);
2099	}
2100
2101	void SelectionDAGLegalize::ExpandFrexpLibCall(
2102	SDNode *Node, SmallVectorImpl<SDValue> &Results) {
2103	SDLoc dl(Node);
2104	EVT VT = Node->getValueType(ResNo: 0);
2105	EVT ExpVT = Node->getValueType(ResNo: 1);
2106
2107	SDValue FPOp = Node->getOperand(Num: 0);
2108
2109	EVT ArgVT = FPOp.getValueType();
2110	Type *ArgTy = ArgVT.getTypeForEVT(Context&: *DAG.getContext());
2111
2112	TargetLowering::ArgListEntry FPArgEntry;
2113	FPArgEntry.Node = FPOp;
2114	FPArgEntry.Ty = ArgTy;
2115
2116	SDValue StackSlot = DAG.CreateStackTemporary(VT: ExpVT);
2117	TargetLowering::ArgListEntry PtrArgEntry;
2118	PtrArgEntry.Node = StackSlot;
2119	PtrArgEntry.Ty = PointerType::get(C&: *DAG.getContext(),
2120	AddressSpace: DAG.getDataLayout().getAllocaAddrSpace());
2121
2122	TargetLowering::ArgListTy Args = {FPArgEntry, PtrArgEntry};
2123
2124	RTLIB::Libcall LC = RTLIB::getFREXP(RetVT: VT);
2125	auto [Call, Chain] = ExpandLibCall(LC, Node, Args: std::move(Args), isSigned: false);
2126
2127	// FIXME: Get type of int for libcall declaration and cast
2128
2129	int FrameIdx = cast<FrameIndexSDNode>(Val&: StackSlot)->getIndex();
2130	auto PtrInfo =
2131	MachinePointerInfo::getFixedStack(MF&: DAG.getMachineFunction(), FI: FrameIdx);
2132
2133	SDValue LoadExp = DAG.getLoad(VT: ExpVT, dl, Chain, Ptr: StackSlot, PtrInfo);
2134	SDValue OutputChain = DAG.getNode(ISD::TokenFactor, dl, MVT::Other,
2135	LoadExp.getValue(R: 1), DAG.getRoot());
2136	DAG.setRoot(OutputChain);
2137
2138	Results.push_back(Elt: Call);
2139	Results.push_back(Elt: LoadExp);
2140	}
2141
2142	void SelectionDAGLegalize::ExpandFPLibCall(SDNode* Node,
2143	RTLIB::Libcall LC,
2144	SmallVectorImpl<SDValue> &Results) {
2145	if (LC == RTLIB::UNKNOWN_LIBCALL)
2146	llvm_unreachable("Can't create an unknown libcall!");
2147
2148	if (Node->isStrictFPOpcode()) {
2149	EVT RetVT = Node->getValueType(ResNo: 0);
2150	SmallVector<SDValue, 4> Ops(drop_begin(RangeOrContainer: Node->ops()));
2151	TargetLowering::MakeLibCallOptions CallOptions;
2152	// FIXME: This doesn't support tail calls.
2153	std::pair<SDValue, SDValue> Tmp = TLI.makeLibCall(DAG, LC, RetVT,
2154	Ops, CallOptions,
2155	dl: SDLoc(Node),
2156	Chain: Node->getOperand(Num: 0));
2157	Results.push_back(Elt: Tmp.first);
2158	Results.push_back(Elt: Tmp.second);
2159	} else {
2160	SDValue Tmp = ExpandLibCall(LC, Node, isSigned: false).first;
2161	Results.push_back(Elt: Tmp);
2162	}
2163	}
2164
2165	/// Expand the node to a libcall based on the result type.
2166	void SelectionDAGLegalize::ExpandFPLibCall(SDNode* Node,
2167	RTLIB::Libcall Call_F32,
2168	RTLIB::Libcall Call_F64,
2169	RTLIB::Libcall Call_F80,
2170	RTLIB::Libcall Call_F128,
2171	RTLIB::Libcall Call_PPCF128,
2172	SmallVectorImpl<SDValue> &Results) {
2173	RTLIB::Libcall LC = RTLIB::getFPLibCall(VT: Node->getSimpleValueType(ResNo: 0),
2174	Call_F32, Call_F64, Call_F80,
2175	Call_F128, Call_PPCF128);
2176	ExpandFPLibCall(Node, LC, Results);
2177	}
2178
2179	SDValue SelectionDAGLegalize::ExpandIntLibCall(SDNode* Node, bool isSigned,
2180	RTLIB::Libcall Call_I8,
2181	RTLIB::Libcall Call_I16,
2182	RTLIB::Libcall Call_I32,
2183	RTLIB::Libcall Call_I64,
2184	RTLIB::Libcall Call_I128) {
2185	RTLIB::Libcall LC;
2186	switch (Node->getSimpleValueType(ResNo: 0).SimpleTy) {
2187	default: llvm_unreachable("Unexpected request for libcall!");
2188	case MVT::i8: LC = Call_I8; break;
2189	case MVT::i16: LC = Call_I16; break;
2190	case MVT::i32: LC = Call_I32; break;
2191	case MVT::i64: LC = Call_I64; break;
2192	case MVT::i128: LC = Call_I128; break;
2193	}
2194	return ExpandLibCall(LC, Node, isSigned).first;
2195	}
2196
2197	/// Expand the node to a libcall based on first argument type (for instance
2198	/// lround and its variant).
2199	void SelectionDAGLegalize::ExpandArgFPLibCall(SDNode* Node,
2200	RTLIB::Libcall Call_F32,
2201	RTLIB::Libcall Call_F64,
2202	RTLIB::Libcall Call_F80,
2203	RTLIB::Libcall Call_F128,
2204	RTLIB::Libcall Call_PPCF128,
2205	SmallVectorImpl<SDValue> &Results) {
2206	EVT InVT = Node->getOperand(Num: Node->isStrictFPOpcode() ? 1 : 0).getValueType();
2207	RTLIB::Libcall LC = RTLIB::getFPLibCall(VT: InVT.getSimpleVT(),
2208	Call_F32, Call_F64, Call_F80,
2209	Call_F128, Call_PPCF128);
2210	ExpandFPLibCall(Node, LC, Results);
2211	}
2212
2213	/// Issue libcalls to __{u}divmod to compute div / rem pairs.
2214	void
2215	SelectionDAGLegalize::ExpandDivRemLibCall(SDNode *Node,
2216	SmallVectorImpl<SDValue> &Results) {
2217	unsigned Opcode = Node->getOpcode();
2218	bool isSigned = Opcode == ISD::SDIVREM;
2219
2220	RTLIB::Libcall LC;
2221	switch (Node->getSimpleValueType(ResNo: 0).SimpleTy) {
2222	default: llvm_unreachable("Unexpected request for libcall!");
2223	case MVT::i8: LC= isSigned ? RTLIB::SDIVREM_I8 : RTLIB::UDIVREM_I8; break;
2224	case MVT::i16: LC= isSigned ? RTLIB::SDIVREM_I16 : RTLIB::UDIVREM_I16; break;
2225	case MVT::i32: LC= isSigned ? RTLIB::SDIVREM_I32 : RTLIB::UDIVREM_I32; break;
2226	case MVT::i64: LC= isSigned ? RTLIB::SDIVREM_I64 : RTLIB::UDIVREM_I64; break;
2227	case MVT::i128: LC= isSigned ? RTLIB::SDIVREM_I128:RTLIB::UDIVREM_I128; break;
2228	}
2229
2230	// The input chain to this libcall is the entry node of the function.
2231	// Legalizing the call will automatically add the previous call to the
2232	// dependence.
2233	SDValue InChain = DAG.getEntryNode();
2234
2235	EVT RetVT = Node->getValueType(ResNo: 0);
2236	Type *RetTy = RetVT.getTypeForEVT(Context&: *DAG.getContext());
2237
2238	TargetLowering::ArgListTy Args;
2239	TargetLowering::ArgListEntry Entry;
2240	for (const SDValue &Op : Node->op_values()) {
2241	EVT ArgVT = Op.getValueType();
2242	Type *ArgTy = ArgVT.getTypeForEVT(Context&: *DAG.getContext());
2243	Entry.Node = Op;
2244	Entry.Ty = ArgTy;
2245	Entry.IsSExt = isSigned;
2246	Entry.IsZExt = !isSigned;
2247	Args.push_back(x: Entry);
2248	}
2249
2250	// Also pass the return address of the remainder.
2251	SDValue FIPtr = DAG.CreateStackTemporary(VT: RetVT);
2252	Entry.Node = FIPtr;
2253	Entry.Ty = PointerType::getUnqual(C&: RetTy->getContext());
2254	Entry.IsSExt = isSigned;
2255	Entry.IsZExt = !isSigned;
2256	Args.push_back(x: Entry);
2257
2258	SDValue Callee = DAG.getExternalSymbol(Sym: TLI.getLibcallName(Call: LC),
2259	VT: TLI.getPointerTy(DL: DAG.getDataLayout()));
2260
2261	SDLoc dl(Node);
2262	TargetLowering::CallLoweringInfo CLI(DAG);
2263	CLI.setDebugLoc(dl)
2264	.setChain(InChain)
2265	.setLibCallee(CC: TLI.getLibcallCallingConv(Call: LC), ResultType: RetTy, Target: Callee,
2266	ArgsList: std::move(Args))
2267	.setSExtResult(isSigned)
2268	.setZExtResult(!isSigned);
2269
2270	std::pair<SDValue, SDValue> CallInfo = TLI.LowerCallTo(CLI);
2271
2272	// Remainder is loaded back from the stack frame.
2273	SDValue Rem =
2274	DAG.getLoad(VT: RetVT, dl, Chain: CallInfo.second, Ptr: FIPtr, PtrInfo: MachinePointerInfo());
2275	Results.push_back(Elt: CallInfo.first);
2276	Results.push_back(Elt: Rem);
2277	}
2278
2279	/// Return true if sincos libcall is available.
2280	static bool isSinCosLibcallAvailable(SDNode *Node, const TargetLowering &TLI) {
2281	RTLIB::Libcall LC;
2282	switch (Node->getSimpleValueType(ResNo: 0).SimpleTy) {
2283	default: llvm_unreachable("Unexpected request for libcall!");
2284	case MVT::f32: LC = RTLIB::SINCOS_F32; break;
2285	case MVT::f64: LC = RTLIB::SINCOS_F64; break;
2286	case MVT::f80: LC = RTLIB::SINCOS_F80; break;
2287	case MVT::f128: LC = RTLIB::SINCOS_F128; break;
2288	case MVT::ppcf128: LC = RTLIB::SINCOS_PPCF128; break;
2289	}
2290	return TLI.getLibcallName(Call: LC) != nullptr;
2291	}
2292
2293	/// Only issue sincos libcall if both sin and cos are needed.
2294	static bool useSinCos(SDNode *Node) {
2295	unsigned OtherOpcode = Node->getOpcode() == ISD::FSIN
2296	? ISD::FCOS : ISD::FSIN;
2297
2298	SDValue Op0 = Node->getOperand(Num: 0);
2299	for (const SDNode *User : Op0.getNode()->uses()) {
2300	if (User == Node)
2301	continue;
2302	// The other user might have been turned into sincos already.
2303	if (User->getOpcode() == OtherOpcode || User->getOpcode() == ISD::FSINCOS)
2304	return true;
2305	}
2306	return false;
2307	}
2308
2309	/// Issue libcalls to sincos to compute sin / cos pairs.
2310	void
2311	SelectionDAGLegalize::ExpandSinCosLibCall(SDNode *Node,
2312	SmallVectorImpl<SDValue> &Results) {
2313	RTLIB::Libcall LC;
2314	switch (Node->getSimpleValueType(ResNo: 0).SimpleTy) {
2315	default: llvm_unreachable("Unexpected request for libcall!");
2316	case MVT::f32: LC = RTLIB::SINCOS_F32; break;
2317	case MVT::f64: LC = RTLIB::SINCOS_F64; break;
2318	case MVT::f80: LC = RTLIB::SINCOS_F80; break;
2319	case MVT::f128: LC = RTLIB::SINCOS_F128; break;
2320	case MVT::ppcf128: LC = RTLIB::SINCOS_PPCF128; break;
2321	}
2322
2323	// The input chain to this libcall is the entry node of the function.
2324	// Legalizing the call will automatically add the previous call to the
2325	// dependence.
2326	SDValue InChain = DAG.getEntryNode();
2327
2328	EVT RetVT = Node->getValueType(ResNo: 0);
2329	Type *RetTy = RetVT.getTypeForEVT(Context&: *DAG.getContext());
2330
2331	TargetLowering::ArgListTy Args;
2332	TargetLowering::ArgListEntry Entry;
2333
2334	// Pass the argument.
2335	Entry.Node = Node->getOperand(Num: 0);
2336	Entry.Ty = RetTy;
2337	Entry.IsSExt = false;
2338	Entry.IsZExt = false;
2339	Args.push_back(x: Entry);
2340
2341	// Pass the return address of sin.
2342	SDValue SinPtr = DAG.CreateStackTemporary(VT: RetVT);
2343	Entry.Node = SinPtr;
2344	Entry.Ty = PointerType::getUnqual(C&: RetTy->getContext());
2345	Entry.IsSExt = false;
2346	Entry.IsZExt = false;
2347	Args.push_back(x: Entry);
2348
2349	// Also pass the return address of the cos.
2350	SDValue CosPtr = DAG.CreateStackTemporary(VT: RetVT);
2351	Entry.Node = CosPtr;
2352	Entry.Ty = PointerType::getUnqual(C&: RetTy->getContext());
2353	Entry.IsSExt = false;
2354	Entry.IsZExt = false;
2355	Args.push_back(x: Entry);
2356
2357	SDValue Callee = DAG.getExternalSymbol(Sym: TLI.getLibcallName(Call: LC),
2358	VT: TLI.getPointerTy(DL: DAG.getDataLayout()));
2359
2360	SDLoc dl(Node);
2361	TargetLowering::CallLoweringInfo CLI(DAG);
2362	CLI.setDebugLoc(dl).setChain(InChain).setLibCallee(
2363	CC: TLI.getLibcallCallingConv(Call: LC), ResultType: Type::getVoidTy(C&: *DAG.getContext()), Target: Callee,
2364	ArgsList: std::move(Args));
2365
2366	std::pair<SDValue, SDValue> CallInfo = TLI.LowerCallTo(CLI);
2367
2368	Results.push_back(
2369	Elt: DAG.getLoad(VT: RetVT, dl, Chain: CallInfo.second, Ptr: SinPtr, PtrInfo: MachinePointerInfo()));
2370	Results.push_back(
2371	Elt: DAG.getLoad(VT: RetVT, dl, Chain: CallInfo.second, Ptr: CosPtr, PtrInfo: MachinePointerInfo()));
2372	}
2373
2374	SDValue SelectionDAGLegalize::expandLdexp(SDNode *Node) const {
2375	SDLoc dl(Node);
2376	EVT VT = Node->getValueType(ResNo: 0);
2377	SDValue X = Node->getOperand(Num: 0);
2378	SDValue N = Node->getOperand(Num: 1);
2379	EVT ExpVT = N.getValueType();
2380	EVT AsIntVT = VT.changeTypeToInteger();
2381	if (AsIntVT == EVT()) // TODO: How to handle f80?
2382	return SDValue();
2383
2384	if (Node->getOpcode() == ISD::STRICT_FLDEXP) // TODO
2385	return SDValue();
2386
2387	SDNodeFlags NSW;
2388	NSW.setNoSignedWrap(true);
2389	SDNodeFlags NUW_NSW;
2390	NUW_NSW.setNoUnsignedWrap(true);
2391	NUW_NSW.setNoSignedWrap(true);
2392
2393	EVT SetCCVT =
2394	TLI.getSetCCResultType(DL: DAG.getDataLayout(), Context&: *DAG.getContext(), VT: ExpVT);
2395	const fltSemantics &FltSem = SelectionDAG::EVTToAPFloatSemantics(VT);
2396
2397	const APFloat::ExponentType MaxExpVal = APFloat::semanticsMaxExponent(FltSem);
2398	const APFloat::ExponentType MinExpVal = APFloat::semanticsMinExponent(FltSem);
2399	const int Precision = APFloat::semanticsPrecision(FltSem);
2400
2401	const SDValue MaxExp = DAG.getConstant(Val: MaxExpVal, DL: dl, VT: ExpVT);
2402	const SDValue MinExp = DAG.getConstant(Val: MinExpVal, DL: dl, VT: ExpVT);
2403
2404	const SDValue DoubleMaxExp = DAG.getConstant(Val: 2 * MaxExpVal, DL: dl, VT: ExpVT);
2405
2406	const APFloat One(FltSem, "1.0");
2407	APFloat ScaleUpK = scalbn(X: One, Exp: MaxExpVal, RM: APFloat::rmNearestTiesToEven);
2408
2409	// Offset by precision to avoid denormal range.
2410	APFloat ScaleDownK =
2411	scalbn(X: One, Exp: MinExpVal + Precision, RM: APFloat::rmNearestTiesToEven);
2412
2413	// TODO: Should really introduce control flow and use a block for the >
2414	// MaxExp, < MinExp cases
2415
2416	// First, handle exponents Exp > MaxExp and scale down.
2417	SDValue NGtMaxExp = DAG.getSetCC(DL: dl, VT: SetCCVT, LHS: N, RHS: MaxExp, Cond: ISD::SETGT);
2418
2419	SDValue DecN0 = DAG.getNode(Opcode: ISD::SUB, DL: dl, VT: ExpVT, N1: N, N2: MaxExp, Flags: NSW);
2420	SDValue ClampMaxVal = DAG.getConstant(Val: 3 * MaxExpVal, DL: dl, VT: ExpVT);
2421	SDValue ClampN_Big = DAG.getNode(Opcode: ISD::SMIN, DL: dl, VT: ExpVT, N1: N, N2: ClampMaxVal);
2422	SDValue DecN1 =
2423	DAG.getNode(Opcode: ISD::SUB, DL: dl, VT: ExpVT, N1: ClampN_Big, N2: DoubleMaxExp, Flags: NSW);
2424
2425	SDValue ScaleUpTwice =
2426	DAG.getSetCC(DL: dl, VT: SetCCVT, LHS: N, RHS: DoubleMaxExp, Cond: ISD::SETUGT);
2427
2428	const SDValue ScaleUpVal = DAG.getConstantFP(Val: ScaleUpK, DL: dl, VT);
2429	SDValue ScaleUp0 = DAG.getNode(Opcode: ISD::FMUL, DL: dl, VT, N1: X, N2: ScaleUpVal);
2430	SDValue ScaleUp1 = DAG.getNode(Opcode: ISD::FMUL, DL: dl, VT, N1: ScaleUp0, N2: ScaleUpVal);
2431
2432	SDValue SelectN_Big =
2433	DAG.getNode(Opcode: ISD::SELECT, DL: dl, VT: ExpVT, N1: ScaleUpTwice, N2: DecN1, N3: DecN0);
2434	SDValue SelectX_Big =
2435	DAG.getNode(Opcode: ISD::SELECT, DL: dl, VT, N1: ScaleUpTwice, N2: ScaleUp1, N3: ScaleUp0);
2436
2437	// Now handle exponents Exp < MinExp
2438	SDValue NLtMinExp = DAG.getSetCC(DL: dl, VT: SetCCVT, LHS: N, RHS: MinExp, Cond: ISD::SETLT);
2439
2440	SDValue Increment0 = DAG.getConstant(Val: -(MinExpVal + Precision), DL: dl, VT: ExpVT);
2441	SDValue Increment1 = DAG.getConstant(Val: -2 * (MinExpVal + Precision), DL: dl, VT: ExpVT);
2442
2443	SDValue IncN0 = DAG.getNode(Opcode: ISD::ADD, DL: dl, VT: ExpVT, N1: N, N2: Increment0, Flags: NUW_NSW);
2444
2445	SDValue ClampMinVal =
2446	DAG.getConstant(Val: 3 * MinExpVal + 2 * Precision, DL: dl, VT: ExpVT);
2447	SDValue ClampN_Small = DAG.getNode(Opcode: ISD::SMAX, DL: dl, VT: ExpVT, N1: N, N2: ClampMinVal);
2448	SDValue IncN1 =
2449	DAG.getNode(Opcode: ISD::ADD, DL: dl, VT: ExpVT, N1: ClampN_Small, N2: Increment1, Flags: NSW);
2450
2451	const SDValue ScaleDownVal = DAG.getConstantFP(Val: ScaleDownK, DL: dl, VT);
2452	SDValue ScaleDown0 = DAG.getNode(Opcode: ISD::FMUL, DL: dl, VT, N1: X, N2: ScaleDownVal);
2453	SDValue ScaleDown1 = DAG.getNode(Opcode: ISD::FMUL, DL: dl, VT, N1: ScaleDown0, N2: ScaleDownVal);
2454
2455	SDValue ScaleDownTwice = DAG.getSetCC(
2456	DL: dl, VT: SetCCVT, LHS: N, RHS: DAG.getConstant(Val: 2 * MinExpVal + Precision, DL: dl, VT: ExpVT),
2457	Cond: ISD::SETULT);
2458
2459	SDValue SelectN_Small =
2460	DAG.getNode(Opcode: ISD::SELECT, DL: dl, VT: ExpVT, N1: ScaleDownTwice, N2: IncN1, N3: IncN0);
2461	SDValue SelectX_Small =
2462	DAG.getNode(Opcode: ISD::SELECT, DL: dl, VT, N1: ScaleDownTwice, N2: ScaleDown1, N3: ScaleDown0);
2463
2464	// Now combine the two out of range exponent handling cases with the base
2465	// case.
2466	SDValue NewX = DAG.getNode(
2467	Opcode: ISD::SELECT, DL: dl, VT, N1: NGtMaxExp, N2: SelectX_Big,
2468	N3: DAG.getNode(Opcode: ISD::SELECT, DL: dl, VT, N1: NLtMinExp, N2: SelectX_Small, N3: X));
2469
2470	SDValue NewN = DAG.getNode(
2471	Opcode: ISD::SELECT, DL: dl, VT: ExpVT, N1: NGtMaxExp, N2: SelectN_Big,
2472	N3: DAG.getNode(Opcode: ISD::SELECT, DL: dl, VT: ExpVT, N1: NLtMinExp, N2: SelectN_Small, N3: N));
2473
2474	SDValue BiasedN = DAG.getNode(Opcode: ISD::ADD, DL: dl, VT: ExpVT, N1: NewN, N2: MaxExp, Flags: NSW);
2475
2476	SDValue ExponentShiftAmt =
2477	DAG.getShiftAmountConstant(Val: Precision - 1, VT: ExpVT, DL: dl);
2478	SDValue CastExpToValTy = DAG.getZExtOrTrunc(Op: BiasedN, DL: dl, VT: AsIntVT);
2479
2480	SDValue AsInt = DAG.getNode(Opcode: ISD::SHL, DL: dl, VT: AsIntVT, N1: CastExpToValTy,
2481	N2: ExponentShiftAmt, Flags: NUW_NSW);
2482	SDValue AsFP = DAG.getNode(Opcode: ISD::BITCAST, DL: dl, VT, Operand: AsInt);
2483	return DAG.getNode(Opcode: ISD::FMUL, DL: dl, VT, N1: NewX, N2: AsFP);
2484	}
2485
2486	SDValue SelectionDAGLegalize::expandFrexp(SDNode *Node) const {
2487	SDLoc dl(Node);
2488	SDValue Val = Node->getOperand(Num: 0);
2489	EVT VT = Val.getValueType();
2490	EVT ExpVT = Node->getValueType(ResNo: 1);
2491	EVT AsIntVT = VT.changeTypeToInteger();
2492	if (AsIntVT == EVT()) // TODO: How to handle f80?
2493	return SDValue();
2494
2495	const fltSemantics &FltSem = SelectionDAG::EVTToAPFloatSemantics(VT);
2496	const APFloat::ExponentType MinExpVal = APFloat::semanticsMinExponent(FltSem);
2497	const unsigned Precision = APFloat::semanticsPrecision(FltSem);
2498	const unsigned BitSize = VT.getScalarSizeInBits();
2499
2500	// TODO: Could introduce control flow and skip over the denormal handling.
2501
2502	// scale_up = fmul value, scalbn(1.0, precision + 1)
2503	// extracted_exp = (bitcast value to uint) >> precision - 1
2504	// biased_exp = extracted_exp + min_exp
2505	// extracted_fract = (bitcast value to uint) & (fract_mask | sign_mask)
2506	//
2507	// is_denormal = val < smallest_normalized
2508	// computed_fract = is_denormal ? scale_up : extracted_fract
2509	// computed_exp = is_denormal ? biased_exp + (-precision - 1) : biased_exp
2510	//
2511	// result_0 = (!isfinite(val) || iszero(val)) ? val : computed_fract
2512	// result_1 = (!isfinite(val) || iszero(val)) ? 0 : computed_exp
2513
2514	SDValue NegSmallestNormalizedInt = DAG.getConstant(
2515	Val: APFloat::getSmallestNormalized(Sem: FltSem, Negative: true).bitcastToAPInt(), DL: dl,
2516	VT: AsIntVT);
2517
2518	SDValue SmallestNormalizedInt = DAG.getConstant(
2519	Val: APFloat::getSmallestNormalized(Sem: FltSem, Negative: false).bitcastToAPInt(), DL: dl,
2520	VT: AsIntVT);
2521
2522	// Masks out the exponent bits.
2523	SDValue ExpMask =
2524	DAG.getConstant(Val: APFloat::getInf(Sem: FltSem).bitcastToAPInt(), DL: dl, VT: AsIntVT);
2525
2526	// Mask out the exponent part of the value.
2527	//
2528	// e.g, for f32 FractSignMaskVal = 0x807fffff
2529	APInt FractSignMaskVal = APInt::getBitsSet(numBits: BitSize, loBit: 0, hiBit: Precision - 1);
2530	FractSignMaskVal.setBit(BitSize - 1); // Set the sign bit
2531
2532	APInt SignMaskVal = APInt::getSignedMaxValue(numBits: BitSize);
2533	SDValue SignMask = DAG.getConstant(Val: SignMaskVal, DL: dl, VT: AsIntVT);
2534
2535	SDValue FractSignMask = DAG.getConstant(Val: FractSignMaskVal, DL: dl, VT: AsIntVT);
2536
2537	const APFloat One(FltSem, "1.0");
2538	// Scale a possible denormal input.
2539	// e.g., for f64, 0x1p+54
2540	APFloat ScaleUpKVal =
2541	scalbn(X: One, Exp: Precision + 1, RM: APFloat::rmNearestTiesToEven);
2542
2543	SDValue ScaleUpK = DAG.getConstantFP(Val: ScaleUpKVal, DL: dl, VT);
2544	SDValue ScaleUp = DAG.getNode(Opcode: ISD::FMUL, DL: dl, VT, N1: Val, N2: ScaleUpK);
2545
2546	EVT SetCCVT =
2547	TLI.getSetCCResultType(DL: DAG.getDataLayout(), Context&: *DAG.getContext(), VT);
2548
2549	SDValue AsInt = DAG.getNode(Opcode: ISD::BITCAST, DL: dl, VT: AsIntVT, Operand: Val);
2550
2551	SDValue Abs = DAG.getNode(Opcode: ISD::AND, DL: dl, VT: AsIntVT, N1: AsInt, N2: SignMask);
2552
2553	SDValue AddNegSmallestNormal =
2554	DAG.getNode(Opcode: ISD::ADD, DL: dl, VT: AsIntVT, N1: Abs, N2: NegSmallestNormalizedInt);
2555	SDValue DenormOrZero = DAG.getSetCC(DL: dl, VT: SetCCVT, LHS: AddNegSmallestNormal,
2556	RHS: NegSmallestNormalizedInt, Cond: ISD::SETULE);
2557
2558	SDValue IsDenormal =
2559	DAG.getSetCC(DL: dl, VT: SetCCVT, LHS: Abs, RHS: SmallestNormalizedInt, Cond: ISD::SETULT);
2560
2561	SDValue MinExp = DAG.getConstant(Val: MinExpVal, DL: dl, VT: ExpVT);
2562	SDValue Zero = DAG.getConstant(Val: 0, DL: dl, VT: ExpVT);
2563
2564	SDValue ScaledAsInt = DAG.getNode(Opcode: ISD::BITCAST, DL: dl, VT: AsIntVT, Operand: ScaleUp);
2565	SDValue ScaledSelect =
2566	DAG.getNode(Opcode: ISD::SELECT, DL: dl, VT: AsIntVT, N1: IsDenormal, N2: ScaledAsInt, N3: AsInt);
2567
2568	SDValue ExpMaskScaled =
2569	DAG.getNode(Opcode: ISD::AND, DL: dl, VT: AsIntVT, N1: ScaledAsInt, N2: ExpMask);
2570
2571	SDValue ScaledValue =
2572	DAG.getNode(Opcode: ISD::SELECT, DL: dl, VT: AsIntVT, N1: IsDenormal, N2: ExpMaskScaled, N3: Abs);
2573
2574	// Extract the exponent bits.
2575	SDValue ExponentShiftAmt =
2576	DAG.getShiftAmountConstant(Val: Precision - 1, VT: AsIntVT, DL: dl);
2577	SDValue ShiftedExp =
2578	DAG.getNode(Opcode: ISD::SRL, DL: dl, VT: AsIntVT, N1: ScaledValue, N2: ExponentShiftAmt);
2579	SDValue Exp = DAG.getSExtOrTrunc(Op: ShiftedExp, DL: dl, VT: ExpVT);
2580
2581	SDValue NormalBiasedExp = DAG.getNode(Opcode: ISD::ADD, DL: dl, VT: ExpVT, N1: Exp, N2: MinExp);
2582	SDValue DenormalOffset = DAG.getConstant(Val: -Precision - 1, DL: dl, VT: ExpVT);
2583	SDValue DenormalExpBias =
2584	DAG.getNode(Opcode: ISD::SELECT, DL: dl, VT: ExpVT, N1: IsDenormal, N2: DenormalOffset, N3: Zero);
2585
2586	SDValue MaskedFractAsInt =
2587	DAG.getNode(Opcode: ISD::AND, DL: dl, VT: AsIntVT, N1: ScaledSelect, N2: FractSignMask);
2588	const APFloat Half(FltSem, "0.5");
2589	SDValue FPHalf = DAG.getConstant(Val: Half.bitcastToAPInt(), DL: dl, VT: AsIntVT);
2590	SDValue Or = DAG.getNode(Opcode: ISD::OR, DL: dl, VT: AsIntVT, N1: MaskedFractAsInt, N2: FPHalf);
2591	SDValue MaskedFract = DAG.getNode(Opcode: ISD::BITCAST, DL: dl, VT, Operand: Or);
2592
2593	SDValue ComputedExp =
2594	DAG.getNode(Opcode: ISD::ADD, DL: dl, VT: ExpVT, N1: NormalBiasedExp, N2: DenormalExpBias);
2595
2596	SDValue Result0 =
2597	DAG.getNode(Opcode: ISD::SELECT, DL: dl, VT, N1: DenormOrZero, N2: Val, N3: MaskedFract);
2598
2599	SDValue Result1 =
2600	DAG.getNode(Opcode: ISD::SELECT, DL: dl, VT: ExpVT, N1: DenormOrZero, N2: Zero, N3: ComputedExp);
2601
2602	return DAG.getMergeValues(Ops: {Result0, Result1}, dl);
2603	}
2604
2605	/// This function is responsible for legalizing a
2606	/// INT_TO_FP operation of the specified operand when the target requests that
2607	/// we expand it. At this point, we know that the result and operand types are
2608	/// legal for the target.
2609	SDValue SelectionDAGLegalize::ExpandLegalINT_TO_FP(SDNode *Node,
2610	SDValue &Chain) {
2611	bool isSigned = (Node->getOpcode() == ISD::STRICT_SINT_TO_FP ||
2612	Node->getOpcode() == ISD::SINT_TO_FP);
2613	EVT DestVT = Node->getValueType(ResNo: 0);
2614	SDLoc dl(Node);
2615	unsigned OpNo = Node->isStrictFPOpcode() ? 1 : 0;
2616	SDValue Op0 = Node->getOperand(Num: OpNo);
2617	EVT SrcVT = Op0.getValueType();
2618
2619	// TODO: Should any fast-math-flags be set for the created nodes?
2620	LLVM_DEBUG(dbgs() << "Legalizing INT_TO_FP\n");
2621	if (SrcVT == MVT::i32 && TLI.isTypeLegal(MVT::f64) &&
2622	(DestVT.bitsLE(MVT::f64) ||
2623	TLI.isOperationLegal(Node->isStrictFPOpcode() ? ISD::STRICT_FP_EXTEND
2624	: ISD::FP_EXTEND,
2625	DestVT))) {
2626	LLVM_DEBUG(dbgs() << "32-bit [signed|unsigned] integer to float/double "
2627	"expansion\n");
2628
2629	// Get the stack frame index of a 8 byte buffer.
2630	SDValue StackSlot = DAG.CreateStackTemporary(MVT::f64);
2631
2632	SDValue Lo = Op0;
2633	// if signed map to unsigned space
2634	if (isSigned) {
2635	// Invert sign bit (signed to unsigned mapping).
2636	Lo = DAG.getNode(ISD::XOR, dl, MVT::i32, Lo,
2637	DAG.getConstant(0x80000000u, dl, MVT::i32));
2638	}
2639	// Initial hi portion of constructed double.
2640	SDValue Hi = DAG.getConstant(0x43300000u, dl, MVT::i32);
2641
2642	// If this a big endian target, swap the lo and high data.
2643	if (DAG.getDataLayout().isBigEndian())
2644	std::swap(a&: Lo, b&: Hi);
2645
2646	SDValue MemChain = DAG.getEntryNode();
2647
2648	// Store the lo of the constructed double.
2649	SDValue Store1 = DAG.getStore(Chain: MemChain, dl, Val: Lo, Ptr: StackSlot,
2650	PtrInfo: MachinePointerInfo());
2651	// Store the hi of the constructed double.
2652	SDValue HiPtr =
2653	DAG.getMemBasePlusOffset(Base: StackSlot, Offset: TypeSize::getFixed(ExactSize: 4), DL: dl);
2654	SDValue Store2 =
2655	DAG.getStore(Chain: MemChain, dl, Val: Hi, Ptr: HiPtr, PtrInfo: MachinePointerInfo());
2656	MemChain = DAG.getNode(ISD::TokenFactor, dl, MVT::Other, Store1, Store2);
2657
2658	// load the constructed double
2659	SDValue Load =
2660	DAG.getLoad(MVT::f64, dl, MemChain, StackSlot, MachinePointerInfo());
2661	// FP constant to bias correct the final result
2662	SDValue Bias = DAG.getConstantFP(
2663	isSigned ? llvm::bit_cast<double>(0x4330000080000000ULL)
2664	: llvm::bit_cast<double>(0x4330000000000000ULL),
2665	dl, MVT::f64);
2666	// Subtract the bias and get the final result.
2667	SDValue Sub;
2668	SDValue Result;
2669	if (Node->isStrictFPOpcode()) {
2670	Sub = DAG.getNode(ISD::STRICT_FSUB, dl, {MVT::f64, MVT::Other},
2671	{Node->getOperand(0), Load, Bias});
2672	Chain = Sub.getValue(R: 1);
2673	if (DestVT != Sub.getValueType()) {
2674	std::pair<SDValue, SDValue> ResultPair;
2675	ResultPair =
2676	DAG.getStrictFPExtendOrRound(Op: Sub, Chain, DL: dl, VT: DestVT);
2677	Result = ResultPair.first;
2678	Chain = ResultPair.second;
2679	}
2680	else
2681	Result = Sub;
2682	} else {
2683	Sub = DAG.getNode(ISD::FSUB, dl, MVT::f64, Load, Bias);
2684	Result = DAG.getFPExtendOrRound(Op: Sub, DL: dl, VT: DestVT);
2685	}
2686	return Result;
2687	}
2688
2689	if (isSigned)
2690	return SDValue();
2691
2692	// TODO: Generalize this for use with other types.
2693	if (((SrcVT == MVT::i32 || SrcVT == MVT::i64) && DestVT == MVT::f32) ||
2694	(SrcVT == MVT::i64 && DestVT == MVT::f64)) {
2695	LLVM_DEBUG(dbgs() << "Converting unsigned i32/i64 to f32/f64\n");
2696	// For unsigned conversions, convert them to signed conversions using the
2697	// algorithm from the x86_64 __floatundisf in compiler_rt. That method
2698	// should be valid for i32->f32 as well.
2699
2700	// More generally this transform should be valid if there are 3 more bits
2701	// in the integer type than the significand. Rounding uses the first bit
2702	// after the width of the significand and the OR of all bits after that. So
2703	// we need to be able to OR the shifted out bit into one of the bits that
2704	// participate in the OR.
2705
2706	// TODO: This really should be implemented using a branch rather than a
2707	// select. We happen to get lucky and machinesink does the right
2708	// thing most of the time. This would be a good candidate for a
2709	// pseudo-op, or, even better, for whole-function isel.
2710	EVT SetCCVT = getSetCCResultType(VT: SrcVT);
2711
2712	SDValue SignBitTest = DAG.getSetCC(
2713	DL: dl, VT: SetCCVT, LHS: Op0, RHS: DAG.getConstant(Val: 0, DL: dl, VT: SrcVT), Cond: ISD::SETLT);
2714
2715	EVT ShiftVT = TLI.getShiftAmountTy(LHSTy: SrcVT, DL: DAG.getDataLayout());
2716	SDValue ShiftConst = DAG.getConstant(Val: 1, DL: dl, VT: ShiftVT);
2717	SDValue Shr = DAG.getNode(Opcode: ISD::SRL, DL: dl, VT: SrcVT, N1: Op0, N2: ShiftConst);
2718	SDValue AndConst = DAG.getConstant(Val: 1, DL: dl, VT: SrcVT);
2719	SDValue And = DAG.getNode(Opcode: ISD::AND, DL: dl, VT: SrcVT, N1: Op0, N2: AndConst);
2720	SDValue Or = DAG.getNode(Opcode: ISD::OR, DL: dl, VT: SrcVT, N1: And, N2: Shr);
2721
2722	SDValue Slow, Fast;
2723	if (Node->isStrictFPOpcode()) {
2724	// In strict mode, we must avoid spurious exceptions, and therefore
2725	// must make sure to only emit a single STRICT_SINT_TO_FP.
2726	SDValue InCvt = DAG.getSelect(DL: dl, VT: SrcVT, Cond: SignBitTest, LHS: Or, RHS: Op0);
2727	Fast = DAG.getNode(ISD::STRICT_SINT_TO_FP, dl, { DestVT, MVT::Other },
2728	{ Node->getOperand(0), InCvt });
2729	Slow = DAG.getNode(ISD::STRICT_FADD, dl, { DestVT, MVT::Other },
2730	{ Fast.getValue(1), Fast, Fast });
2731	Chain = Slow.getValue(R: 1);
2732	// The STRICT_SINT_TO_FP inherits the exception mode from the
2733	// incoming STRICT_UINT_TO_FP node; the STRICT_FADD node can
2734	// never raise any exception.
2735	SDNodeFlags Flags;
2736	Flags.setNoFPExcept(Node->getFlags().hasNoFPExcept());
2737	Fast->setFlags(Flags);
2738	Flags.setNoFPExcept(true);
2739	Slow->setFlags(Flags);
2740	} else {
2741	SDValue SignCvt = DAG.getNode(Opcode: ISD::SINT_TO_FP, DL: dl, VT: DestVT, Operand: Or);
2742	Slow = DAG.getNode(Opcode: ISD::FADD, DL: dl, VT: DestVT, N1: SignCvt, N2: SignCvt);
2743	Fast = DAG.getNode(Opcode: ISD::SINT_TO_FP, DL: dl, VT: DestVT, Operand: Op0);
2744	}
2745
2746	return DAG.getSelect(DL: dl, VT: DestVT, Cond: SignBitTest, LHS: Slow, RHS: Fast);
2747	}
2748
2749	// Don't expand it if there isn't cheap fadd.
2750	if (!TLI.isOperationLegalOrCustom(
2751	Op: Node->isStrictFPOpcode() ? ISD::STRICT_FADD : ISD::FADD, VT: DestVT))
2752	return SDValue();
2753
2754	// The following optimization is valid only if every value in SrcVT (when
2755	// treated as signed) is representable in DestVT. Check that the mantissa
2756	// size of DestVT is >= than the number of bits in SrcVT -1.
2757	assert(APFloat::semanticsPrecision(DAG.EVTToAPFloatSemantics(DestVT)) >=
2758	SrcVT.getSizeInBits() - 1 &&
2759	"Cannot perform lossless SINT_TO_FP!");
2760
2761	SDValue Tmp1;
2762	if (Node->isStrictFPOpcode()) {
2763	Tmp1 = DAG.getNode(ISD::STRICT_SINT_TO_FP, dl, { DestVT, MVT::Other },
2764	{ Node->getOperand(0), Op0 });
2765	} else
2766	Tmp1 = DAG.getNode(Opcode: ISD::SINT_TO_FP, DL: dl, VT: DestVT, Operand: Op0);
2767
2768	SDValue SignSet = DAG.getSetCC(DL: dl, VT: getSetCCResultType(VT: SrcVT), LHS: Op0,
2769	RHS: DAG.getConstant(Val: 0, DL: dl, VT: SrcVT), Cond: ISD::SETLT);
2770	SDValue Zero = DAG.getIntPtrConstant(Val: 0, DL: dl),
2771	Four = DAG.getIntPtrConstant(Val: 4, DL: dl);
2772	SDValue CstOffset = DAG.getSelect(DL: dl, VT: Zero.getValueType(),
2773	Cond: SignSet, LHS: Four, RHS: Zero);
2774
2775	// If the sign bit of the integer is set, the large number will be treated
2776	// as a negative number. To counteract this, the dynamic code adds an
2777	// offset depending on the data type.
2778	uint64_t FF;
2779	switch (SrcVT.getSimpleVT().SimpleTy) {
2780	default:
2781	return SDValue();
2782	case MVT::i8 : FF = 0x43800000ULL; break; // 2^8 (as a float)
2783	case MVT::i16: FF = 0x47800000ULL; break; // 2^16 (as a float)
2784	case MVT::i32: FF = 0x4F800000ULL; break; // 2^32 (as a float)
2785	case MVT::i64: FF = 0x5F800000ULL; break; // 2^64 (as a float)
2786	}
2787	if (DAG.getDataLayout().isLittleEndian())
2788	FF <<= 32;
2789	Constant *FudgeFactor = ConstantInt::get(
2790	Ty: Type::getInt64Ty(C&: *DAG.getContext()), V: FF);
2791
2792	SDValue CPIdx =
2793	DAG.getConstantPool(C: FudgeFactor, VT: TLI.getPointerTy(DL: DAG.getDataLayout()));
2794	Align Alignment = cast<ConstantPoolSDNode>(Val&: CPIdx)->getAlign();
2795	CPIdx = DAG.getNode(Opcode: ISD::ADD, DL: dl, VT: CPIdx.getValueType(), N1: CPIdx, N2: CstOffset);
2796	Alignment = commonAlignment(A: Alignment, Offset: 4);
2797	SDValue FudgeInReg;
2798	if (DestVT == MVT::f32)
2799	FudgeInReg = DAG.getLoad(
2800	MVT::f32, dl, DAG.getEntryNode(), CPIdx,
2801	MachinePointerInfo::getConstantPool(DAG.getMachineFunction()),
2802	Alignment);
2803	else {
2804	SDValue Load = DAG.getExtLoad(
2805	ISD::EXTLOAD, dl, DestVT, DAG.getEntryNode(), CPIdx,
2806	MachinePointerInfo::getConstantPool(DAG.getMachineFunction()), MVT::f32,
2807	Alignment);
2808	HandleSDNode Handle(Load);
2809	LegalizeOp(Node: Load.getNode());
2810	FudgeInReg = Handle.getValue();
2811	}
2812
2813	if (Node->isStrictFPOpcode()) {
2814	SDValue Result = DAG.getNode(ISD::STRICT_FADD, dl, { DestVT, MVT::Other },
2815	{ Tmp1.getValue(1), Tmp1, FudgeInReg });
2816	Chain = Result.getValue(R: 1);
2817	return Result;
2818	}
2819
2820	return DAG.getNode(Opcode: ISD::FADD, DL: dl, VT: DestVT, N1: Tmp1, N2: FudgeInReg);
2821	}
2822
2823	/// This function is responsible for legalizing a
2824	/// *INT_TO_FP operation of the specified operand when the target requests that
2825	/// we promote it. At this point, we know that the result and operand types are
2826	/// legal for the target, and that there is a legal UINT_TO_FP or SINT_TO_FP
2827	/// operation that takes a larger input.
2828	void SelectionDAGLegalize::PromoteLegalINT_TO_FP(
2829	SDNode *N, const SDLoc &dl, SmallVectorImpl<SDValue> &Results) {
2830	bool IsStrict = N->isStrictFPOpcode();
2831	bool IsSigned = N->getOpcode() == ISD::SINT_TO_FP ||
2832	N->getOpcode() == ISD::STRICT_SINT_TO_FP;
2833	EVT DestVT = N->getValueType(ResNo: 0);
2834	SDValue LegalOp = N->getOperand(Num: IsStrict ? 1 : 0);
2835	unsigned UIntOp = IsStrict ? ISD::STRICT_UINT_TO_FP : ISD::UINT_TO_FP;
2836	unsigned SIntOp = IsStrict ? ISD::STRICT_SINT_TO_FP : ISD::SINT_TO_FP;
2837
2838	// First step, figure out the appropriate *INT_TO_FP operation to use.
2839	EVT NewInTy = LegalOp.getValueType();
2840
2841	unsigned OpToUse = 0;
2842
2843	// Scan for the appropriate larger type to use.
2844	while (true) {
2845	NewInTy = (MVT::SimpleValueType)(NewInTy.getSimpleVT().SimpleTy+1);
2846	assert(NewInTy.isInteger() && "Ran out of possibilities!");
2847
2848	// If the target supports SINT_TO_FP of this type, use it.
2849	if (TLI.isOperationLegalOrCustom(Op: SIntOp, VT: NewInTy)) {
2850	OpToUse = SIntOp;
2851	break;
2852	}
2853	if (IsSigned)
2854	continue;
2855
2856	// If the target supports UINT_TO_FP of this type, use it.
2857	if (TLI.isOperationLegalOrCustom(Op: UIntOp, VT: NewInTy)) {
2858	OpToUse = UIntOp;
2859	break;
2860	}
2861
2862	// Otherwise, try a larger type.
2863	}
2864
2865	// Okay, we found the operation and type to use. Zero extend our input to the
2866	// desired type then run the operation on it.
2867	if (IsStrict) {
2868	SDValue Res =
2869	DAG.getNode(OpToUse, dl, {DestVT, MVT::Other},
2870	{N->getOperand(0),
2871	DAG.getNode(IsSigned ? ISD::SIGN_EXTEND : ISD::ZERO_EXTEND,
2872	dl, NewInTy, LegalOp)});
2873	Results.push_back(Elt: Res);
2874	Results.push_back(Elt: Res.getValue(R: 1));
2875	return;
2876	}
2877
2878	Results.push_back(
2879	Elt: DAG.getNode(Opcode: OpToUse, DL: dl, VT: DestVT,
2880	Operand: DAG.getNode(Opcode: IsSigned ? ISD::SIGN_EXTEND : ISD::ZERO_EXTEND,
2881	DL: dl, VT: NewInTy, Operand: LegalOp)));
2882	}
2883
2884	/// This function is responsible for legalizing a
2885	/// FP_TO_*INT operation of the specified operand when the target requests that
2886	/// we promote it. At this point, we know that the result and operand types are
2887	/// legal for the target, and that there is a legal FP_TO_UINT or FP_TO_SINT
2888	/// operation that returns a larger result.
2889	void SelectionDAGLegalize::PromoteLegalFP_TO_INT(SDNode *N, const SDLoc &dl,
2890	SmallVectorImpl<SDValue> &Results) {
2891	bool IsStrict = N->isStrictFPOpcode();
2892	bool IsSigned = N->getOpcode() == ISD::FP_TO_SINT ||
2893	N->getOpcode() == ISD::STRICT_FP_TO_SINT;
2894	EVT DestVT = N->getValueType(ResNo: 0);
2895	SDValue LegalOp = N->getOperand(Num: IsStrict ? 1 : 0);
2896	// First step, figure out the appropriate FP_TO*INT operation to use.
2897	EVT NewOutTy = DestVT;
2898
2899	unsigned OpToUse = 0;
2900
2901	// Scan for the appropriate larger type to use.
2902	while (true) {
2903	NewOutTy = (MVT::SimpleValueType)(NewOutTy.getSimpleVT().SimpleTy+1);
2904	assert(NewOutTy.isInteger() && "Ran out of possibilities!");
2905
2906	// A larger signed type can hold all unsigned values of the requested type,
2907	// so using FP_TO_SINT is valid
2908	OpToUse = IsStrict ? ISD::STRICT_FP_TO_SINT : ISD::FP_TO_SINT;
2909	if (TLI.isOperationLegalOrCustom(Op: OpToUse, VT: NewOutTy))
2910	break;
2911
2912	// However, if the value may be < 0.0, we *must* use some FP_TO_SINT.
2913	OpToUse = IsStrict ? ISD::STRICT_FP_TO_UINT : ISD::FP_TO_UINT;
2914	if (!IsSigned && TLI.isOperationLegalOrCustom(Op: OpToUse, VT: NewOutTy))
2915	break;
2916
2917	// Otherwise, try a larger type.
2918	}
2919
2920	// Okay, we found the operation and type to use.
2921	SDValue Operation;
2922	if (IsStrict) {
2923	SDVTList VTs = DAG.getVTList(NewOutTy, MVT::Other);
2924	Operation = DAG.getNode(Opcode: OpToUse, DL: dl, VTList: VTs, N1: N->getOperand(Num: 0), N2: LegalOp);
2925	} else
2926	Operation = DAG.getNode(Opcode: OpToUse, DL: dl, VT: NewOutTy, Operand: LegalOp);
2927
2928	// Truncate the result of the extended FP_TO_*INT operation to the desired
2929	// size.
2930	SDValue Trunc = DAG.getNode(Opcode: ISD::TRUNCATE, DL: dl, VT: DestVT, Operand: Operation);
2931	Results.push_back(Elt: Trunc);
2932	if (IsStrict)
2933	Results.push_back(Elt: Operation.getValue(R: 1));
2934	}
2935
2936	/// Promote FP_TO_*INT_SAT operation to a larger result type. At this point
2937	/// the result and operand types are legal and there must be a legal
2938	/// FP_TO_*INT_SAT operation for a larger result type.
2939	SDValue SelectionDAGLegalize::PromoteLegalFP_TO_INT_SAT(SDNode *Node,
2940	const SDLoc &dl) {
2941	unsigned Opcode = Node->getOpcode();
2942
2943	// Scan for the appropriate larger type to use.
2944	EVT NewOutTy = Node->getValueType(ResNo: 0);
2945	while (true) {
2946	NewOutTy = (MVT::SimpleValueType)(NewOutTy.getSimpleVT().SimpleTy + 1);
2947	assert(NewOutTy.isInteger() && "Ran out of possibilities!");
2948
2949	if (TLI.isOperationLegalOrCustom(Op: Opcode, VT: NewOutTy))
2950	break;
2951	}
2952
2953	// Saturation width is determined by second operand, so we don't have to
2954	// perform any fixup and can directly truncate the result.
2955	SDValue Result = DAG.getNode(Opcode, DL: dl, VT: NewOutTy, N1: Node->getOperand(Num: 0),
2956	N2: Node->getOperand(Num: 1));
2957	return DAG.getNode(Opcode: ISD::TRUNCATE, DL: dl, VT: Node->getValueType(ResNo: 0), Operand: Result);
2958	}
2959
2960	/// Open code the operations for PARITY of the specified operation.
2961	SDValue SelectionDAGLegalize::ExpandPARITY(SDValue Op, const SDLoc &dl) {
2962	EVT VT = Op.getValueType();
2963	EVT ShVT = TLI.getShiftAmountTy(LHSTy: VT, DL: DAG.getDataLayout());
2964	unsigned Sz = VT.getScalarSizeInBits();
2965
2966	// If CTPOP is legal, use it. Otherwise use shifts and xor.
2967	SDValue Result;
2968	if (TLI.isOperationLegalOrPromote(Op: ISD::CTPOP, VT)) {
2969	Result = DAG.getNode(Opcode: ISD::CTPOP, DL: dl, VT, Operand: Op);
2970	} else {
2971	Result = Op;
2972	for (unsigned i = Log2_32_Ceil(Value: Sz); i != 0;) {
2973	SDValue Shift = DAG.getNode(Opcode: ISD::SRL, DL: dl, VT, N1: Result,
2974	N2: DAG.getConstant(Val: 1ULL << (--i), DL: dl, VT: ShVT));
2975	Result = DAG.getNode(Opcode: ISD::XOR, DL: dl, VT, N1: Result, N2: Shift);
2976	}
2977	}
2978
2979	return DAG.getNode(Opcode: ISD::AND, DL: dl, VT, N1: Result, N2: DAG.getConstant(Val: 1, DL: dl, VT));
2980	}
2981
2982	bool SelectionDAGLegalize::ExpandNode(SDNode *Node) {
2983	LLVM_DEBUG(dbgs() << "Trying to expand node\n");
2984	SmallVector<SDValue, 8> Results;
2985	SDLoc dl(Node);
2986	SDValue Tmp1, Tmp2, Tmp3, Tmp4;
2987	bool NeedInvert;
2988	switch (Node->getOpcode()) {
2989	case ISD::ABS:
2990	if ((Tmp1 = TLI.expandABS(N: Node, DAG)))
2991	Results.push_back(Elt: Tmp1);
2992	break;
2993	case ISD::ABDS:
2994	case ISD::ABDU:
2995	if ((Tmp1 = TLI.expandABD(N: Node, DAG)))
2996	Results.push_back(Elt: Tmp1);
2997	break;
2998	case ISD::CTPOP:
2999	if ((Tmp1 = TLI.expandCTPOP(N: Node, DAG)))
3000	Results.push_back(Elt: Tmp1);
3001	break;
3002	case ISD::CTLZ:
3003	case ISD::CTLZ_ZERO_UNDEF:
3004	if ((Tmp1 = TLI.expandCTLZ(N: Node, DAG)))
3005	Results.push_back(Elt: Tmp1);
3006	break;
3007	case ISD::CTTZ:
3008	case ISD::CTTZ_ZERO_UNDEF:
3009	if ((Tmp1 = TLI.expandCTTZ(N: Node, DAG)))
3010	Results.push_back(Elt: Tmp1);
3011	break;
3012	case ISD::BITREVERSE:
3013	if ((Tmp1 = TLI.expandBITREVERSE(N: Node, DAG)))
3014	Results.push_back(Elt: Tmp1);
3015	break;
3016	case ISD::BSWAP:
3017	if ((Tmp1 = TLI.expandBSWAP(N: Node, DAG)))
3018	Results.push_back(Elt: Tmp1);
3019	break;
3020	case ISD::PARITY:
3021	Results.push_back(Elt: ExpandPARITY(Op: Node->getOperand(Num: 0), dl));
3022	break;
3023	case ISD::FRAMEADDR:
3024	case ISD::RETURNADDR:
3025	case ISD::FRAME_TO_ARGS_OFFSET:
3026	Results.push_back(Elt: DAG.getConstant(Val: 0, DL: dl, VT: Node->getValueType(ResNo: 0)));
3027	break;
3028	case ISD::EH_DWARF_CFA: {
3029	SDValue CfaArg = DAG.getSExtOrTrunc(Op: Node->getOperand(Num: 0), DL: dl,
3030	VT: TLI.getPointerTy(DL: DAG.getDataLayout()));
3031	SDValue Offset = DAG.getNode(Opcode: ISD::ADD, DL: dl,
3032	VT: CfaArg.getValueType(),
3033	N1: DAG.getNode(Opcode: ISD::FRAME_TO_ARGS_OFFSET, DL: dl,
3034	VT: CfaArg.getValueType()),
3035	N2: CfaArg);
3036	SDValue FA = DAG.getNode(
3037	Opcode: ISD::FRAMEADDR, DL: dl, VT: TLI.getPointerTy(DL: DAG.getDataLayout()),
3038	Operand: DAG.getConstant(Val: 0, DL: dl, VT: TLI.getPointerTy(DL: DAG.getDataLayout())));
3039	Results.push_back(Elt: DAG.getNode(Opcode: ISD::ADD, DL: dl, VT: FA.getValueType(),
3040	N1: FA, N2: Offset));
3041	break;
3042	}
3043	case ISD::GET_ROUNDING:
3044	Results.push_back(Elt: DAG.getConstant(Val: 1, DL: dl, VT: Node->getValueType(ResNo: 0)));
3045	Results.push_back(Elt: Node->getOperand(Num: 0));
3046	break;
3047	case ISD::EH_RETURN:
3048	case ISD::EH_LABEL:
3049	case ISD::PREFETCH:
3050	case ISD::VAEND:
3051	case ISD::EH_SJLJ_LONGJMP:
3052	// If the target didn't expand these, there's nothing to do, so just
3053	// preserve the chain and be done.
3054	Results.push_back(Elt: Node->getOperand(Num: 0));
3055	break;
3056	case ISD::READCYCLECOUNTER:
3057	case ISD::READSTEADYCOUNTER:
3058	// If the target didn't expand this, just return 'zero' and preserve the
3059	// chain.
3060	Results.append(NumInputs: Node->getNumValues() - 1,
3061	Elt: DAG.getConstant(Val: 0, DL: dl, VT: Node->getValueType(ResNo: 0)));
3062	Results.push_back(Elt: Node->getOperand(Num: 0));
3063	break;
3064	case ISD::EH_SJLJ_SETJMP:
3065	// If the target didn't expand this, just return 'zero' and preserve the
3066	// chain.
3067	Results.push_back(DAG.getConstant(0, dl, MVT::i32));
3068	Results.push_back(Elt: Node->getOperand(Num: 0));
3069	break;
3070	case ISD::ATOMIC_LOAD: {
3071	// There is no libcall for atomic load; fake it with ATOMIC_CMP_SWAP.
3072	SDValue Zero = DAG.getConstant(Val: 0, DL: dl, VT: Node->getValueType(ResNo: 0));
3073	SDVTList VTs = DAG.getVTList(Node->getValueType(0), MVT::Other);
3074	SDValue Swap = DAG.getAtomicCmpSwap(
3075	Opcode: ISD::ATOMIC_CMP_SWAP, dl, MemVT: cast<AtomicSDNode>(Val: Node)->getMemoryVT(), VTs,
3076	Chain: Node->getOperand(Num: 0), Ptr: Node->getOperand(Num: 1), Cmp: Zero, Swp: Zero,
3077	MMO: cast<AtomicSDNode>(Val: Node)->getMemOperand());
3078	Results.push_back(Elt: Swap.getValue(R: 0));
3079	Results.push_back(Elt: Swap.getValue(R: 1));
3080	break;
3081	}
3082	case ISD::ATOMIC_STORE: {
3083	// There is no libcall for atomic store; fake it with ATOMIC_SWAP.
3084	SDValue Swap = DAG.getAtomic(
3085	Opcode: ISD::ATOMIC_SWAP, dl, MemVT: cast<AtomicSDNode>(Val: Node)->getMemoryVT(),
3086	Chain: Node->getOperand(Num: 0), Ptr: Node->getOperand(Num: 2), Val: Node->getOperand(Num: 1),
3087	MMO: cast<AtomicSDNode>(Val: Node)->getMemOperand());
3088	Results.push_back(Elt: Swap.getValue(R: 1));
3089	break;
3090	}
3091	case ISD::ATOMIC_CMP_SWAP_WITH_SUCCESS: {
3092	// Expanding an ATOMIC_CMP_SWAP_WITH_SUCCESS produces an ATOMIC_CMP_SWAP and
3093	// splits out the success value as a comparison. Expanding the resulting
3094	// ATOMIC_CMP_SWAP will produce a libcall.
3095	SDVTList VTs = DAG.getVTList(Node->getValueType(0), MVT::Other);
3096	SDValue Res = DAG.getAtomicCmpSwap(
3097	Opcode: ISD::ATOMIC_CMP_SWAP, dl, MemVT: cast<AtomicSDNode>(Val: Node)->getMemoryVT(), VTs,
3098	Chain: Node->getOperand(Num: 0), Ptr: Node->getOperand(Num: 1), Cmp: Node->getOperand(Num: 2),
3099	Swp: Node->getOperand(Num: 3), MMO: cast<MemSDNode>(Val: Node)->getMemOperand());
3100
3101	SDValue ExtRes = Res;
3102	SDValue LHS = Res;
3103	SDValue RHS = Node->getOperand(Num: 1);
3104
3105	EVT AtomicType = cast<AtomicSDNode>(Val: Node)->getMemoryVT();
3106	EVT OuterType = Node->getValueType(ResNo: 0);
3107	switch (TLI.getExtendForAtomicOps()) {
3108	case ISD::SIGN_EXTEND:
3109	LHS = DAG.getNode(Opcode: ISD::AssertSext, DL: dl, VT: OuterType, N1: Res,
3110	N2: DAG.getValueType(AtomicType));
3111	RHS = DAG.getNode(Opcode: ISD::SIGN_EXTEND_INREG, DL: dl, VT: OuterType,
3112	N1: Node->getOperand(Num: 2), N2: DAG.getValueType(AtomicType));
3113	ExtRes = LHS;
3114	break;
3115	case ISD::ZERO_EXTEND:
3116	LHS = DAG.getNode(Opcode: ISD::AssertZext, DL: dl, VT: OuterType, N1: Res,
3117	N2: DAG.getValueType(AtomicType));
3118	RHS = DAG.getZeroExtendInReg(Op: Node->getOperand(Num: 2), DL: dl, VT: AtomicType);
3119	ExtRes = LHS;
3120	break;
3121	case ISD::ANY_EXTEND:
3122	LHS = DAG.getZeroExtendInReg(Op: Res, DL: dl, VT: AtomicType);
3123	RHS = DAG.getZeroExtendInReg(Op: Node->getOperand(Num: 2), DL: dl, VT: AtomicType);
3124	break;
3125	default:
3126	llvm_unreachable("Invalid atomic op extension");
3127	}
3128
3129	SDValue Success =
3130	DAG.getSetCC(DL: dl, VT: Node->getValueType(ResNo: 1), LHS, RHS, Cond: ISD::SETEQ);
3131
3132	Results.push_back(Elt: ExtRes.getValue(R: 0));
3133	Results.push_back(Elt: Success);
3134	Results.push_back(Elt: Res.getValue(R: 1));
3135	break;
3136	}
3137	case ISD::ATOMIC_LOAD_SUB: {
3138	SDLoc DL(Node);
3139	EVT VT = Node->getValueType(ResNo: 0);
3140	SDValue RHS = Node->getOperand(Num: 2);
3141	AtomicSDNode *AN = cast<AtomicSDNode>(Val: Node);
3142	if (RHS->getOpcode() == ISD::SIGN_EXTEND_INREG &&
3143	cast<VTSDNode>(Val: RHS->getOperand(Num: 1))->getVT() == AN->getMemoryVT())
3144	RHS = RHS->getOperand(Num: 0);
3145	SDValue NewRHS =
3146	DAG.getNode(Opcode: ISD::SUB, DL, VT, N1: DAG.getConstant(Val: 0, DL, VT), N2: RHS);
3147	SDValue Res = DAG.getAtomic(Opcode: ISD::ATOMIC_LOAD_ADD, dl: DL, MemVT: AN->getMemoryVT(),
3148	Chain: Node->getOperand(Num: 0), Ptr: Node->getOperand(Num: 1),
3149	Val: NewRHS, MMO: AN->getMemOperand());
3150	Results.push_back(Elt: Res);
3151	Results.push_back(Elt: Res.getValue(R: 1));
3152	break;
3153	}
3154	case ISD::DYNAMIC_STACKALLOC:
3155	ExpandDYNAMIC_STACKALLOC(Node, Results);
3156	break;
3157	case ISD::MERGE_VALUES:
3158	for (unsigned i = 0; i < Node->getNumValues(); i++)
3159	Results.push_back(Elt: Node->getOperand(Num: i));
3160	break;
3161	case ISD::UNDEF: {
3162	EVT VT = Node->getValueType(ResNo: 0);
3163	if (VT.isInteger())
3164	Results.push_back(Elt: DAG.getConstant(Val: 0, DL: dl, VT));
3165	else {
3166	assert(VT.isFloatingPoint() && "Unknown value type!");
3167	Results.push_back(Elt: DAG.getConstantFP(Val: 0, DL: dl, VT));
3168	}
3169	break;
3170	}
3171	case ISD::STRICT_FP_ROUND:
3172	// When strict mode is enforced we can't do expansion because it
3173	// does not honor the "strict" properties. Only libcall is allowed.
3174	if (TLI.isStrictFPEnabled())
3175	break;
3176	// We might as well mutate to FP_ROUND when FP_ROUND operation is legal
3177	// since this operation is more efficient than stack operation.
3178	if (TLI.getStrictFPOperationAction(Op: Node->getOpcode(),
3179	VT: Node->getValueType(ResNo: 0))
3180	== TargetLowering::Legal)
3181	break;
3182	// We fall back to use stack operation when the FP_ROUND operation
3183	// isn't available.
3184	if ((Tmp1 = EmitStackConvert(SrcOp: Node->getOperand(Num: 1), SlotVT: Node->getValueType(ResNo: 0),
3185	DestVT: Node->getValueType(ResNo: 0), dl,
3186	Chain: Node->getOperand(Num: 0)))) {
3187	ReplaceNode(Old: Node, New: Tmp1.getNode());
3188	LLVM_DEBUG(dbgs() << "Successfully expanded STRICT_FP_ROUND node\n");
3189	return true;
3190	}
3191	break;
3192	case ISD::FP_ROUND: {
3193	if ((Tmp1 = TLI.expandFP_ROUND(Node, DAG))) {
3194	Results.push_back(Elt: Tmp1);
3195	break;
3196	}
3197
3198	LLVM_FALLTHROUGH;
3199	}
3200	case ISD::BITCAST:
3201	if ((Tmp1 = EmitStackConvert(SrcOp: Node->getOperand(Num: 0), SlotVT: Node->getValueType(ResNo: 0),
3202	DestVT: Node->getValueType(ResNo: 0), dl)))
3203	Results.push_back(Elt: Tmp1);
3204	break;
3205	case ISD::STRICT_FP_EXTEND:
3206	// When strict mode is enforced we can't do expansion because it
3207	// does not honor the "strict" properties. Only libcall is allowed.
3208	if (TLI.isStrictFPEnabled())
3209	break;
3210	// We might as well mutate to FP_EXTEND when FP_EXTEND operation is legal
3211	// since this operation is more efficient than stack operation.
3212	if (TLI.getStrictFPOperationAction(Op: Node->getOpcode(),
3213	VT: Node->getValueType(ResNo: 0))
3214	== TargetLowering::Legal)
3215	break;
3216	// We fall back to use stack operation when the FP_EXTEND operation
3217	// isn't available.
3218	if ((Tmp1 = EmitStackConvert(
3219	SrcOp: Node->getOperand(Num: 1), SlotVT: Node->getOperand(Num: 1).getValueType(),
3220	DestVT: Node->getValueType(ResNo: 0), dl, Chain: Node->getOperand(Num: 0)))) {
3221	ReplaceNode(Old: Node, New: Tmp1.getNode());
3222	LLVM_DEBUG(dbgs() << "Successfully expanded STRICT_FP_EXTEND node\n");
3223	return true;
3224	}
3225	break;
3226	case ISD::FP_EXTEND: {
3227	SDValue Op = Node->getOperand(Num: 0);
3228	EVT SrcVT = Op.getValueType();
3229	EVT DstVT = Node->getValueType(ResNo: 0);
3230	if (SrcVT.getScalarType() == MVT::bf16) {
3231	Results.push_back(Elt: DAG.getNode(Opcode: ISD::BF16_TO_FP, DL: SDLoc(Node), VT: DstVT, Operand: Op));
3232	break;
3233	}
3234
3235	if ((Tmp1 = EmitStackConvert(SrcOp: Op, SlotVT: SrcVT, DestVT: DstVT, dl)))
3236	Results.push_back(Elt: Tmp1);
3237	break;
3238	}
3239	case ISD::BF16_TO_FP: {
3240	// Always expand bf16 to f32 casts, they lower to ext + shift.
3241	//
3242	// Note that the operand of this code can be bf16 or an integer type in case
3243	// bf16 is not supported on the target and was softened.
3244	SDValue Op = Node->getOperand(Num: 0);
3245	if (Op.getValueType() == MVT::bf16) {
3246	Op = DAG.getNode(ISD::ANY_EXTEND, dl, MVT::i32,
3247	DAG.getNode(ISD::BITCAST, dl, MVT::i16, Op));
3248	} else {
3249	Op = DAG.getAnyExtOrTrunc(Op, dl, MVT::i32);
3250	}
3251	Op = DAG.getNode(
3252	ISD::SHL, dl, MVT::i32, Op,
3253	DAG.getConstant(16, dl,
3254	TLI.getShiftAmountTy(MVT::i32, DAG.getDataLayout())));
3255	Op = DAG.getNode(ISD::BITCAST, dl, MVT::f32, Op);
3256	// Add fp_extend in case the output is bigger than f32.
3257	if (Node->getValueType(0) != MVT::f32)
3258	Op = DAG.getNode(Opcode: ISD::FP_EXTEND, DL: dl, VT: Node->getValueType(ResNo: 0), Operand: Op);
3259	Results.push_back(Elt: Op);
3260	break;
3261	}
3262	case ISD::FP_TO_BF16: {
3263	SDValue Op = Node->getOperand(Num: 0);
3264	if (Op.getValueType() != MVT::f32)
3265	Op = DAG.getNode(ISD::FP_ROUND, dl, MVT::f32, Op,
3266	DAG.getIntPtrConstant(0, dl, /*isTarget=*/true));
3267	// Certain SNaNs will turn into infinities if we do a simple shift right.
3268	if (!DAG.isKnownNeverSNaN(Op)) {
3269	Op = DAG.getNode(ISD::FCANONICALIZE, dl, MVT::f32, Op, Node->getFlags());
3270	}
3271	Op = DAG.getNode(
3272	ISD::SRL, dl, MVT::i32, DAG.getNode(ISD::BITCAST, dl, MVT::i32, Op),
3273	DAG.getConstant(16, dl,
3274	TLI.getShiftAmountTy(MVT::i32, DAG.getDataLayout())));
3275	// The result of this node can be bf16 or an integer type in case bf16 is
3276	// not supported on the target and was softened to i16 for storage.
3277	if (Node->getValueType(0) == MVT::bf16) {
3278	Op = DAG.getNode(ISD::BITCAST, dl, MVT::bf16,
3279	DAG.getNode(ISD::TRUNCATE, dl, MVT::i16, Op));
3280	} else {
3281	Op = DAG.getAnyExtOrTrunc(Op, DL: dl, VT: Node->getValueType(ResNo: 0));
3282	}
3283	Results.push_back(Elt: Op);
3284	break;
3285	}
3286	case ISD::SIGN_EXTEND_INREG: {
3287	EVT ExtraVT = cast<VTSDNode>(Val: Node->getOperand(Num: 1))->getVT();
3288	EVT VT = Node->getValueType(ResNo: 0);
3289
3290	// An in-register sign-extend of a boolean is a negation:
3291	// 'true' (1) sign-extended is -1.
3292	// 'false' (0) sign-extended is 0.
3293	// However, we must mask the high bits of the source operand because the
3294	// SIGN_EXTEND_INREG does not guarantee that the high bits are already zero.
3295
3296	// TODO: Do this for vectors too?
3297	if (ExtraVT.isScalarInteger() && ExtraVT.getSizeInBits() == 1) {
3298	SDValue One = DAG.getConstant(Val: 1, DL: dl, VT);
3299	SDValue And = DAG.getNode(Opcode: ISD::AND, DL: dl, VT, N1: Node->getOperand(Num: 0), N2: One);
3300	SDValue Zero = DAG.getConstant(Val: 0, DL: dl, VT);
3301	SDValue Neg = DAG.getNode(Opcode: ISD::SUB, DL: dl, VT, N1: Zero, N2: And);
3302	Results.push_back(Elt: Neg);
3303	break;
3304	}
3305
3306	// NOTE: we could fall back on load/store here too for targets without
3307	// SRA. However, it is doubtful that any exist.
3308	EVT ShiftAmountTy = TLI.getShiftAmountTy(LHSTy: VT, DL: DAG.getDataLayout());
3309	unsigned BitsDiff = VT.getScalarSizeInBits() -
3310	ExtraVT.getScalarSizeInBits();
3311	SDValue ShiftCst = DAG.getConstant(Val: BitsDiff, DL: dl, VT: ShiftAmountTy);
3312	Tmp1 = DAG.getNode(Opcode: ISD::SHL, DL: dl, VT: Node->getValueType(ResNo: 0),
3313	N1: Node->getOperand(Num: 0), N2: ShiftCst);
3314	Tmp1 = DAG.getNode(Opcode: ISD::SRA, DL: dl, VT: Node->getValueType(ResNo: 0), N1: Tmp1, N2: ShiftCst);
3315	Results.push_back(Elt: Tmp1);
3316	break;
3317	}
3318	case ISD::UINT_TO_FP:
3319	case ISD::STRICT_UINT_TO_FP:
3320	if (TLI.expandUINT_TO_FP(N: Node, Result&: Tmp1, Chain&: Tmp2, DAG)) {
3321	Results.push_back(Elt: Tmp1);
3322	if (Node->isStrictFPOpcode())
3323	Results.push_back(Elt: Tmp2);
3324	break;
3325	}
3326	[[fallthrough]];
3327	case ISD::SINT_TO_FP:
3328	case ISD::STRICT_SINT_TO_FP:
3329	if ((Tmp1 = ExpandLegalINT_TO_FP(Node, Chain&: Tmp2))) {
3330	Results.push_back(Elt: Tmp1);
3331	if (Node->isStrictFPOpcode())
3332	Results.push_back(Elt: Tmp2);
3333	}
3334	break;
3335	case ISD::FP_TO_SINT:
3336	if (TLI.expandFP_TO_SINT(N: Node, Result&: Tmp1, DAG))
3337	Results.push_back(Elt: Tmp1);
3338	break;
3339	case ISD::STRICT_FP_TO_SINT:
3340	if (TLI.expandFP_TO_SINT(N: Node, Result&: Tmp1, DAG)) {
3341	ReplaceNode(Old: Node, New: Tmp1.getNode());
3342	LLVM_DEBUG(dbgs() << "Successfully expanded STRICT_FP_TO_SINT node\n");
3343	return true;
3344	}
3345	break;
3346	case ISD::FP_TO_UINT:
3347	if (TLI.expandFP_TO_UINT(N: Node, Result&: Tmp1, Chain&: Tmp2, DAG))
3348	Results.push_back(Elt: Tmp1);
3349	break;
3350	case ISD::STRICT_FP_TO_UINT:
3351	if (TLI.expandFP_TO_UINT(N: Node, Result&: Tmp1, Chain&: Tmp2, DAG)) {
3352	// Relink the chain.
3353	DAG.ReplaceAllUsesOfValueWith(From: SDValue(Node,1), To: Tmp2);
3354	// Replace the new UINT result.
3355	ReplaceNodeWithValue(Old: SDValue(Node, 0), New: Tmp1);
3356	LLVM_DEBUG(dbgs() << "Successfully expanded STRICT_FP_TO_UINT node\n");
3357	return true;
3358	}
3359	break;
3360	case ISD::FP_TO_SINT_SAT:
3361	case ISD::FP_TO_UINT_SAT:
3362	Results.push_back(Elt: TLI.expandFP_TO_INT_SAT(N: Node, DAG));
3363	break;
3364	case ISD::VAARG:
3365	Results.push_back(Elt: DAG.expandVAArg(Node));
3366	Results.push_back(Elt: Results[0].getValue(R: 1));
3367	break;
3368	case ISD::VACOPY:
3369	Results.push_back(Elt: DAG.expandVACopy(Node));
3370	break;
3371	case ISD::EXTRACT_VECTOR_ELT:
3372	if (Node->getOperand(Num: 0).getValueType().getVectorElementCount().isScalar())
3373	// This must be an access of the only element. Return it.
3374	Tmp1 = DAG.getNode(Opcode: ISD::BITCAST, DL: dl, VT: Node->getValueType(ResNo: 0),
3375	Operand: Node->getOperand(Num: 0));
3376	else
3377	Tmp1 = ExpandExtractFromVectorThroughStack(Op: SDValue(Node, 0));
3378	Results.push_back(Elt: Tmp1);
3379	break;
3380	case ISD::EXTRACT_SUBVECTOR:
3381	Results.push_back(Elt: ExpandExtractFromVectorThroughStack(Op: SDValue(Node, 0)));
3382	break;
3383	case ISD::INSERT_SUBVECTOR:
3384	Results.push_back(Elt: ExpandInsertToVectorThroughStack(Op: SDValue(Node, 0)));
3385	break;
3386	case ISD::CONCAT_VECTORS:
3387	Results.push_back(Elt: ExpandVectorBuildThroughStack(Node));
3388	break;
3389	case ISD::SCALAR_TO_VECTOR:
3390	Results.push_back(Elt: ExpandSCALAR_TO_VECTOR(Node));
3391	break;
3392	case ISD::INSERT_VECTOR_ELT:
3393	Results.push_back(Elt: ExpandINSERT_VECTOR_ELT(Op: SDValue(Node, 0)));
3394	break;
3395	case ISD::VECTOR_SHUFFLE: {
3396	SmallVector<int, 32> NewMask;
3397	ArrayRef<int> Mask = cast<ShuffleVectorSDNode>(Val: Node)->getMask();
3398
3399	EVT VT = Node->getValueType(ResNo: 0);
3400	EVT EltVT = VT.getVectorElementType();
3401	SDValue Op0 = Node->getOperand(Num: 0);
3402	SDValue Op1 = Node->getOperand(Num: 1);
3403	if (!TLI.isTypeLegal(VT: EltVT)) {
3404	EVT NewEltVT = TLI.getTypeToTransformTo(Context&: *DAG.getContext(), VT: EltVT);
3405
3406	// BUILD_VECTOR operands are allowed to be wider than the element type.
3407	// But if NewEltVT is smaller that EltVT the BUILD_VECTOR does not accept
3408	// it.
3409	if (NewEltVT.bitsLT(VT: EltVT)) {
3410	// Convert shuffle node.
3411	// If original node was v4i64 and the new EltVT is i32,
3412	// cast operands to v8i32 and re-build the mask.
3413
3414	// Calculate new VT, the size of the new VT should be equal to original.
3415	EVT NewVT =
3416	EVT::getVectorVT(Context&: *DAG.getContext(), VT: NewEltVT,
3417	NumElements: VT.getSizeInBits() / NewEltVT.getSizeInBits());
3418	assert(NewVT.bitsEq(VT));
3419
3420	// cast operands to new VT
3421	Op0 = DAG.getNode(Opcode: ISD::BITCAST, DL: dl, VT: NewVT, Operand: Op0);
3422	Op1 = DAG.getNode(Opcode: ISD::BITCAST, DL: dl, VT: NewVT, Operand: Op1);
3423
3424	// Convert the shuffle mask
3425	unsigned int factor =
3426	NewVT.getVectorNumElements()/VT.getVectorNumElements();
3427
3428	// EltVT gets smaller
3429	assert(factor > 0);
3430
3431	for (unsigned i = 0; i < VT.getVectorNumElements(); ++i) {
3432	if (Mask[i] < 0) {
3433	for (unsigned fi = 0; fi < factor; ++fi)
3434	NewMask.push_back(Elt: Mask[i]);
3435	}
3436	else {
3437	for (unsigned fi = 0; fi < factor; ++fi)
3438	NewMask.push_back(Elt: Mask[i]*factor+fi);
3439	}
3440	}
3441	Mask = NewMask;
3442	VT = NewVT;
3443	}
3444	EltVT = NewEltVT;
3445	}
3446	unsigned NumElems = VT.getVectorNumElements();
3447	SmallVector<SDValue, 16> Ops;
3448	for (unsigned i = 0; i != NumElems; ++i) {
3449	if (Mask[i] < 0) {
3450	Ops.push_back(Elt: DAG.getUNDEF(VT: EltVT));
3451	continue;
3452	}
3453	unsigned Idx = Mask[i];
3454	if (Idx < NumElems)
3455	Ops.push_back(Elt: DAG.getNode(Opcode: ISD::EXTRACT_VECTOR_ELT, DL: dl, VT: EltVT, N1: Op0,
3456	N2: DAG.getVectorIdxConstant(Val: Idx, DL: dl)));
3457	else
3458	Ops.push_back(
3459	Elt: DAG.getNode(Opcode: ISD::EXTRACT_VECTOR_ELT, DL: dl, VT: EltVT, N1: Op1,
3460	N2: DAG.getVectorIdxConstant(Val: Idx - NumElems, DL: dl)));
3461	}
3462
3463	Tmp1 = DAG.getBuildVector(VT, DL: dl, Ops);
3464	// We may have changed the BUILD_VECTOR type. Cast it back to the Node type.
3465	Tmp1 = DAG.getNode(Opcode: ISD::BITCAST, DL: dl, VT: Node->getValueType(ResNo: 0), Operand: Tmp1);
3466	Results.push_back(Elt: Tmp1);
3467	break;
3468	}
3469	case ISD::VECTOR_SPLICE: {
3470	Results.push_back(Elt: TLI.expandVectorSplice(Node, DAG));
3471	break;
3472	}
3473	case ISD::EXTRACT_ELEMENT: {
3474	EVT OpTy = Node->getOperand(Num: 0).getValueType();
3475	if (Node->getConstantOperandVal(Num: 1)) {
3476	// 1 -> Hi
3477	Tmp1 = DAG.getNode(Opcode: ISD::SRL, DL: dl, VT: OpTy, N1: Node->getOperand(Num: 0),
3478	N2: DAG.getConstant(Val: OpTy.getSizeInBits() / 2, DL: dl,
3479	VT: TLI.getShiftAmountTy(
3480	LHSTy: Node->getOperand(Num: 0).getValueType(),
3481	DL: DAG.getDataLayout())));
3482	Tmp1 = DAG.getNode(Opcode: ISD::TRUNCATE, DL: dl, VT: Node->getValueType(ResNo: 0), Operand: Tmp1);
3483	} else {
3484	// 0 -> Lo
3485	Tmp1 = DAG.getNode(Opcode: ISD::TRUNCATE, DL: dl, VT: Node->getValueType(ResNo: 0),
3486	Operand: Node->getOperand(Num: 0));
3487	}
3488	Results.push_back(Elt: Tmp1);
3489	break;
3490	}
3491	case ISD::STACKSAVE:
3492	// Expand to CopyFromReg if the target set
3493	// StackPointerRegisterToSaveRestore.
3494	if (Register SP = TLI.getStackPointerRegisterToSaveRestore()) {
3495	Results.push_back(Elt: DAG.getCopyFromReg(Chain: Node->getOperand(Num: 0), dl, Reg: SP,
3496	VT: Node->getValueType(ResNo: 0)));
3497	Results.push_back(Elt: Results[0].getValue(R: 1));
3498	} else {
3499	Results.push_back(Elt: DAG.getUNDEF(VT: Node->getValueType(ResNo: 0)));
3500	Results.push_back(Elt: Node->getOperand(Num: 0));
3501	}
3502	break;
3503	case ISD::STACKRESTORE:
3504	// Expand to CopyToReg if the target set
3505	// StackPointerRegisterToSaveRestore.
3506	if (Register SP = TLI.getStackPointerRegisterToSaveRestore()) {
3507	Results.push_back(Elt: DAG.getCopyToReg(Chain: Node->getOperand(Num: 0), dl, Reg: SP,
3508	N: Node->getOperand(Num: 1)));
3509	} else {
3510	Results.push_back(Elt: Node->getOperand(Num: 0));
3511	}
3512	break;
3513	case ISD::GET_DYNAMIC_AREA_OFFSET:
3514	Results.push_back(Elt: DAG.getConstant(Val: 0, DL: dl, VT: Node->getValueType(ResNo: 0)));
3515	Results.push_back(Elt: Results[0].getValue(R: 0));
3516	break;
3517	case ISD::FCOPYSIGN:
3518	Results.push_back(Elt: ExpandFCOPYSIGN(Node));
3519	break;
3520	case ISD::FNEG:
3521	Results.push_back(Elt: ExpandFNEG(Node));
3522	break;
3523	case ISD::FABS:
3524	Results.push_back(Elt: ExpandFABS(Node));
3525	break;
3526	case ISD::IS_FPCLASS: {
3527	auto Test = static_cast<FPClassTest>(Node->getConstantOperandVal(Num: 1));
3528	if (SDValue Expanded =
3529	TLI.expandIS_FPCLASS(ResultVT: Node->getValueType(ResNo: 0), Op: Node->getOperand(Num: 0),
3530	Test, Flags: Node->getFlags(), DL: SDLoc(Node), DAG))
3531	Results.push_back(Elt: Expanded);
3532	break;
3533	}
3534	case ISD::SMIN:
3535	case ISD::SMAX:
3536	case ISD::UMIN:
3537	case ISD::UMAX: {
3538	// Expand Y = MAX(A, B) -> Y = (A > B) ? A : B
3539	ISD::CondCode Pred;
3540	switch (Node->getOpcode()) {
3541	default: llvm_unreachable("How did we get here?");
3542	case ISD::SMAX: Pred = ISD::SETGT; break;
3543	case ISD::SMIN: Pred = ISD::SETLT; break;
3544	case ISD::UMAX: Pred = ISD::SETUGT; break;
3545	case ISD::UMIN: Pred = ISD::SETULT; break;
3546	}
3547	Tmp1 = Node->getOperand(Num: 0);
3548	Tmp2 = Node->getOperand(Num: 1);
3549	Tmp1 = DAG.getSelectCC(DL: dl, LHS: Tmp1, RHS: Tmp2, True: Tmp1, False: Tmp2, Cond: Pred);
3550	Results.push_back(Elt: Tmp1);
3551	break;
3552	}
3553	case ISD::FMINNUM:
3554	case ISD::FMAXNUM: {
3555	if (SDValue Expanded = TLI.expandFMINNUM_FMAXNUM(N: Node, DAG))
3556	Results.push_back(Elt: Expanded);
3557	break;
3558	}
3559	case ISD::FSIN:
3560	case ISD::FCOS: {
3561	EVT VT = Node->getValueType(ResNo: 0);
3562	// Turn fsin / fcos into ISD::FSINCOS node if there are a pair of fsin /
3563	// fcos which share the same operand and both are used.
3564	if ((TLI.isOperationLegalOrCustom(Op: ISD::FSINCOS, VT) ||
3565	isSinCosLibcallAvailable(Node, TLI))
3566	&& useSinCos(Node)) {
3567	SDVTList VTs = DAG.getVTList(VT1: VT, VT2: VT);
3568	Tmp1 = DAG.getNode(Opcode: ISD::FSINCOS, DL: dl, VTList: VTs, N: Node->getOperand(Num: 0));
3569	if (Node->getOpcode() == ISD::FCOS)
3570	Tmp1 = Tmp1.getValue(R: 1);
3571	Results.push_back(Elt: Tmp1);
3572	}
3573	break;
3574	}
3575	case ISD::FLDEXP:
3576	case ISD::STRICT_FLDEXP: {
3577	EVT VT = Node->getValueType(ResNo: 0);
3578	RTLIB::Libcall LC = RTLIB::getLDEXP(RetVT: VT);
3579	// Use the LibCall instead, it is very likely faster
3580	// FIXME: Use separate LibCall action.
3581	if (TLI.getLibcallName(Call: LC))
3582	break;
3583
3584	if (SDValue Expanded = expandLdexp(Node)) {
3585	Results.push_back(Elt: Expanded);
3586	if (Node->getOpcode() == ISD::STRICT_FLDEXP)
3587	Results.push_back(Elt: Expanded.getValue(R: 1));
3588	}
3589
3590	break;
3591	}
3592	case ISD::FFREXP: {
3593	RTLIB::Libcall LC = RTLIB::getFREXP(RetVT: Node->getValueType(ResNo: 0));
3594	// Use the LibCall instead, it is very likely faster
3595	// FIXME: Use separate LibCall action.
3596	if (TLI.getLibcallName(Call: LC))
3597	break;
3598
3599	if (SDValue Expanded = expandFrexp(Node)) {
3600	Results.push_back(Elt: Expanded);
3601	Results.push_back(Elt: Expanded.getValue(R: 1));
3602	}
3603	break;
3604	}
3605	case ISD::FMAD:
3606	llvm_unreachable("Illegal fmad should never be formed");
3607
3608	case ISD::FP16_TO_FP:
3609	if (Node->getValueType(0) != MVT::f32) {
3610	// We can extend to types bigger than f32 in two steps without changing
3611	// the result. Since "f16 -> f32" is much more commonly available, give
3612	// CodeGen the option of emitting that before resorting to a libcall.
3613	SDValue Res =
3614	DAG.getNode(ISD::FP16_TO_FP, dl, MVT::f32, Node->getOperand(0));
3615	Results.push_back(
3616	Elt: DAG.getNode(Opcode: ISD::FP_EXTEND, DL: dl, VT: Node->getValueType(ResNo: 0), Operand: Res));
3617	}
3618	break;
3619	case ISD::STRICT_BF16_TO_FP:
3620	case ISD::STRICT_FP16_TO_FP:
3621	if (Node->getValueType(0) != MVT::f32) {
3622	// We can extend to types bigger than f32 in two steps without changing
3623	// the result. Since "f16 -> f32" is much more commonly available, give
3624	// CodeGen the option of emitting that before resorting to a libcall.
3625	SDValue Res = DAG.getNode(Node->getOpcode(), dl, {MVT::f32, MVT::Other},
3626	{Node->getOperand(0), Node->getOperand(1)});
3627	Res = DAG.getNode(ISD::STRICT_FP_EXTEND, dl,
3628	{Node->getValueType(0), MVT::Other},
3629	{Res.getValue(1), Res});
3630	Results.push_back(Elt: Res);
3631	Results.push_back(Elt: Res.getValue(R: 1));
3632	}
3633	break;
3634	case ISD::FP_TO_FP16:
3635	LLVM_DEBUG(dbgs() << "Legalizing FP_TO_FP16\n");
3636	if (!TLI.useSoftFloat() && TM.Options.UnsafeFPMath) {
3637	SDValue Op = Node->getOperand(Num: 0);
3638	MVT SVT = Op.getSimpleValueType();
3639	if ((SVT == MVT::f64 || SVT == MVT::f80) &&
3640	TLI.isOperationLegalOrCustom(ISD::FP_TO_FP16, MVT::f32)) {
3641	// Under fastmath, we can expand this node into a fround followed by
3642	// a float-half conversion.
3643	SDValue FloatVal =
3644	DAG.getNode(ISD::FP_ROUND, dl, MVT::f32, Op,
3645	DAG.getIntPtrConstant(0, dl, /*isTarget=*/true));
3646	Results.push_back(
3647	Elt: DAG.getNode(Opcode: ISD::FP_TO_FP16, DL: dl, VT: Node->getValueType(ResNo: 0), Operand: FloatVal));
3648	}
3649	}
3650	break;
3651	case ISD::ConstantFP: {
3652	ConstantFPSDNode *CFP = cast<ConstantFPSDNode>(Val: Node);
3653	// Check to see if this FP immediate is already legal.
3654	// If this is a legal constant, turn it into a TargetConstantFP node.
3655	if (!TLI.isFPImmLegal(CFP->getValueAPF(), Node->getValueType(ResNo: 0),
3656	ForCodeSize: DAG.shouldOptForSize()))
3657	Results.push_back(Elt: ExpandConstantFP(CFP, UseCP: true));
3658	break;
3659	}
3660	case ISD::Constant: {
3661	ConstantSDNode *CP = cast<ConstantSDNode>(Val: Node);
3662	Results.push_back(Elt: ExpandConstant(CP));
3663	break;
3664	}
3665	case ISD::FSUB: {
3666	EVT VT = Node->getValueType(ResNo: 0);
3667	if (TLI.isOperationLegalOrCustom(Op: ISD::FADD, VT) &&
3668	TLI.isOperationLegalOrCustom(Op: ISD::FNEG, VT)) {
3669	const SDNodeFlags Flags = Node->getFlags();
3670	Tmp1 = DAG.getNode(Opcode: ISD::FNEG, DL: dl, VT, Operand: Node->getOperand(Num: 1));
3671	Tmp1 = DAG.getNode(Opcode: ISD::FADD, DL: dl, VT, N1: Node->getOperand(Num: 0), N2: Tmp1, Flags);
3672	Results.push_back(Elt: Tmp1);
3673	}
3674	break;
3675	}
3676	case ISD::SUB: {
3677	EVT VT = Node->getValueType(ResNo: 0);
3678	assert(TLI.isOperationLegalOrCustom(ISD::ADD, VT) &&
3679	TLI.isOperationLegalOrCustom(ISD::XOR, VT) &&
3680	"Don't know how to expand this subtraction!");
3681	Tmp1 = DAG.getNOT(DL: dl, Val: Node->getOperand(Num: 1), VT);
3682	Tmp1 = DAG.getNode(Opcode: ISD::ADD, DL: dl, VT, N1: Tmp1, N2: DAG.getConstant(Val: 1, DL: dl, VT));
3683	Results.push_back(Elt: DAG.getNode(Opcode: ISD::ADD, DL: dl, VT, N1: Node->getOperand(Num: 0), N2: Tmp1));
3684	break;
3685	}
3686	case ISD::UREM:
3687	case ISD::SREM:
3688	if (TLI.expandREM(Node, Result&: Tmp1, DAG))
3689	Results.push_back(Elt: Tmp1);
3690	break;
3691	case ISD::UDIV:
3692	case ISD::SDIV: {
3693	bool isSigned = Node->getOpcode() == ISD::SDIV;
3694	unsigned DivRemOpc = isSigned ? ISD::SDIVREM : ISD::UDIVREM;
3695	EVT VT = Node->getValueType(ResNo: 0);
3696	if (TLI.isOperationLegalOrCustom(Op: DivRemOpc, VT)) {
3697	SDVTList VTs = DAG.getVTList(VT1: VT, VT2: VT);
3698	Tmp1 = DAG.getNode(Opcode: DivRemOpc, DL: dl, VTList: VTs, N1: Node->getOperand(Num: 0),
3699	N2: Node->getOperand(Num: 1));
3700	Results.push_back(Elt: Tmp1);
3701	}
3702	break;
3703	}
3704	case ISD::MULHU:
3705	case ISD::MULHS: {
3706	unsigned ExpandOpcode =
3707	Node->getOpcode() == ISD::MULHU ? ISD::UMUL_LOHI : ISD::SMUL_LOHI;
3708	EVT VT = Node->getValueType(ResNo: 0);
3709	SDVTList VTs = DAG.getVTList(VT1: VT, VT2: VT);
3710
3711	Tmp1 = DAG.getNode(Opcode: ExpandOpcode, DL: dl, VTList: VTs, N1: Node->getOperand(Num: 0),
3712	N2: Node->getOperand(Num: 1));
3713	Results.push_back(Elt: Tmp1.getValue(R: 1));
3714	break;
3715	}
3716	case ISD::UMUL_LOHI:
3717	case ISD::SMUL_LOHI: {
3718	SDValue LHS = Node->getOperand(Num: 0);
3719	SDValue RHS = Node->getOperand(Num: 1);
3720	MVT VT = LHS.getSimpleValueType();
3721	unsigned MULHOpcode =
3722	Node->getOpcode() == ISD::UMUL_LOHI ? ISD::MULHU : ISD::MULHS;
3723
3724	if (TLI.isOperationLegalOrCustom(Op: MULHOpcode, VT)) {
3725	Results.push_back(Elt: DAG.getNode(Opcode: ISD::MUL, DL: dl, VT, N1: LHS, N2: RHS));
3726	Results.push_back(Elt: DAG.getNode(Opcode: MULHOpcode, DL: dl, VT, N1: LHS, N2: RHS));
3727	break;
3728	}
3729
3730	SmallVector<SDValue, 4> Halves;
3731	EVT HalfType = EVT(VT).getHalfSizedIntegerVT(Context&: *DAG.getContext());
3732	assert(TLI.isTypeLegal(HalfType));
3733	if (TLI.expandMUL_LOHI(Opcode: Node->getOpcode(), VT, dl, LHS, RHS, Result&: Halves,
3734	HiLoVT: HalfType, DAG,
3735	Kind: TargetLowering::MulExpansionKind::Always)) {
3736	for (unsigned i = 0; i < 2; ++i) {
3737	SDValue Lo = DAG.getNode(Opcode: ISD::ZERO_EXTEND, DL: dl, VT, Operand: Halves[2 * i]);
3738	SDValue Hi = DAG.getNode(Opcode: ISD::ANY_EXTEND, DL: dl, VT, Operand: Halves[2 * i + 1]);
3739	SDValue Shift = DAG.getConstant(
3740	Val: HalfType.getScalarSizeInBits(), DL: dl,
3741	VT: TLI.getShiftAmountTy(LHSTy: HalfType, DL: DAG.getDataLayout()));
3742	Hi = DAG.getNode(Opcode: ISD::SHL, DL: dl, VT, N1: Hi, N2: Shift);
3743	Results.push_back(Elt: DAG.getNode(Opcode: ISD::OR, DL: dl, VT, N1: Lo, N2: Hi));
3744	}
3745	break;
3746	}
3747	break;
3748	}
3749	case ISD::MUL: {
3750	EVT VT = Node->getValueType(ResNo: 0);
3751	SDVTList VTs = DAG.getVTList(VT1: VT, VT2: VT);
3752	// See if multiply or divide can be lowered using two-result operations.
3753	// We just need the low half of the multiply; try both the signed
3754	// and unsigned forms. If the target supports both SMUL_LOHI and
3755	// UMUL_LOHI, form a preference by checking which forms of plain
3756	// MULH it supports.
3757	bool HasSMUL_LOHI = TLI.isOperationLegalOrCustom(Op: ISD::SMUL_LOHI, VT);
3758	bool HasUMUL_LOHI = TLI.isOperationLegalOrCustom(Op: ISD::UMUL_LOHI, VT);
3759	bool HasMULHS = TLI.isOperationLegalOrCustom(Op: ISD::MULHS, VT);
3760	bool HasMULHU = TLI.isOperationLegalOrCustom(Op: ISD::MULHU, VT);
3761	unsigned OpToUse = 0;
3762	if (HasSMUL_LOHI && !HasMULHS) {
3763	OpToUse = ISD::SMUL_LOHI;
3764	} else if (HasUMUL_LOHI && !HasMULHU) {
3765	OpToUse = ISD::UMUL_LOHI;
3766	} else if (HasSMUL_LOHI) {
3767	OpToUse = ISD::SMUL_LOHI;
3768	} else if (HasUMUL_LOHI) {
3769	OpToUse = ISD::UMUL_LOHI;
3770	}
3771	if (OpToUse) {
3772	Results.push_back(Elt: DAG.getNode(Opcode: OpToUse, DL: dl, VTList: VTs, N1: Node->getOperand(Num: 0),
3773	N2: Node->getOperand(Num: 1)));
3774	break;
3775	}
3776
3777	SDValue Lo, Hi;
3778	EVT HalfType = VT.getHalfSizedIntegerVT(Context&: *DAG.getContext());
3779	if (TLI.isOperationLegalOrCustom(Op: ISD::ZERO_EXTEND, VT) &&
3780	TLI.isOperationLegalOrCustom(Op: ISD::ANY_EXTEND, VT) &&
3781	TLI.isOperationLegalOrCustom(Op: ISD::SHL, VT) &&
3782	TLI.isOperationLegalOrCustom(Op: ISD::OR, VT) &&
3783	TLI.expandMUL(N: Node, Lo, Hi, HiLoVT: HalfType, DAG,
3784	Kind: TargetLowering::MulExpansionKind::OnlyLegalOrCustom)) {
3785	Lo = DAG.getNode(Opcode: ISD::ZERO_EXTEND, DL: dl, VT, Operand: Lo);
3786	Hi = DAG.getNode(Opcode: ISD::ANY_EXTEND, DL: dl, VT, Operand: Hi);
3787	SDValue Shift =
3788	DAG.getConstant(Val: HalfType.getSizeInBits(), DL: dl,
3789	VT: TLI.getShiftAmountTy(LHSTy: HalfType, DL: DAG.getDataLayout()));
3790	Hi = DAG.getNode(Opcode: ISD::SHL, DL: dl, VT, N1: Hi, N2: Shift);
3791	Results.push_back(Elt: DAG.getNode(Opcode: ISD::OR, DL: dl, VT, N1: Lo, N2: Hi));
3792	}
3793	break;
3794	}
3795	case ISD::FSHL:
3796	case ISD::FSHR:
3797	if (SDValue Expanded = TLI.expandFunnelShift(N: Node, DAG))
3798	Results.push_back(Elt: Expanded);
3799	break;
3800	case ISD::ROTL:
3801	case ISD::ROTR:
3802	if (SDValue Expanded = TLI.expandROT(N: Node, AllowVectorOps: true /*AllowVectorOps*/, DAG))
3803	Results.push_back(Elt: Expanded);
3804	break;
3805	case ISD::SADDSAT:
3806	case ISD::UADDSAT:
3807	case ISD::SSUBSAT:
3808	case ISD::USUBSAT:
3809	Results.push_back(Elt: TLI.expandAddSubSat(Node, DAG));
3810	break;
3811	case ISD::SSHLSAT:
3812	case ISD::USHLSAT:
3813	Results.push_back(Elt: TLI.expandShlSat(Node, DAG));
3814	break;
3815	case ISD::SMULFIX:
3816	case ISD::SMULFIXSAT:
3817	case ISD::UMULFIX:
3818	case ISD::UMULFIXSAT:
3819	Results.push_back(Elt: TLI.expandFixedPointMul(Node, DAG));
3820	break;
3821	case ISD::SDIVFIX:
3822	case ISD::SDIVFIXSAT:
3823	case ISD::UDIVFIX:
3824	case ISD::UDIVFIXSAT:
3825	if (SDValue V = TLI.expandFixedPointDiv(Opcode: Node->getOpcode(), dl: SDLoc(Node),
3826	LHS: Node->getOperand(Num: 0),
3827	RHS: Node->getOperand(Num: 1),
3828	Scale: Node->getConstantOperandVal(Num: 2),
3829	DAG)) {
3830	Results.push_back(Elt: V);
3831	break;
3832	}
3833	// FIXME: We might want to retry here with a wider type if we fail, if that
3834	// type is legal.
3835	// FIXME: Technically, so long as we only have sdivfixes where BW+Scale is
3836	// <= 128 (which is the case for all of the default Embedded-C types),
3837	// we will only get here with types and scales that we could always expand
3838	// if we were allowed to generate libcalls to division functions of illegal
3839	// type. But we cannot do that.
3840	llvm_unreachable("Cannot expand DIVFIX!");
3841	case ISD::UADDO_CARRY:
3842	case ISD::USUBO_CARRY: {
3843	SDValue LHS = Node->getOperand(Num: 0);
3844	SDValue RHS = Node->getOperand(Num: 1);
3845	SDValue Carry = Node->getOperand(Num: 2);
3846
3847	bool IsAdd = Node->getOpcode() == ISD::UADDO_CARRY;
3848
3849	// Initial add of the 2 operands.
3850	unsigned Op = IsAdd ? ISD::ADD : ISD::SUB;
3851	EVT VT = LHS.getValueType();
3852	SDValue Sum = DAG.getNode(Opcode: Op, DL: dl, VT, N1: LHS, N2: RHS);
3853
3854	// Initial check for overflow.
3855	EVT CarryType = Node->getValueType(ResNo: 1);
3856	EVT SetCCType = getSetCCResultType(VT: Node->getValueType(ResNo: 0));
3857	ISD::CondCode CC = IsAdd ? ISD::SETULT : ISD::SETUGT;
3858	SDValue Overflow = DAG.getSetCC(DL: dl, VT: SetCCType, LHS: Sum, RHS: LHS, Cond: CC);
3859
3860	// Add of the sum and the carry.
3861	SDValue One = DAG.getConstant(Val: 1, DL: dl, VT);
3862	SDValue CarryExt =
3863	DAG.getNode(Opcode: ISD::AND, DL: dl, VT, N1: DAG.getZExtOrTrunc(Op: Carry, DL: dl, VT), N2: One);
3864	SDValue Sum2 = DAG.getNode(Opcode: Op, DL: dl, VT, N1: Sum, N2: CarryExt);
3865
3866	// Second check for overflow. If we are adding, we can only overflow if the
3867	// initial sum is all 1s ang the carry is set, resulting in a new sum of 0.
3868	// If we are subtracting, we can only overflow if the initial sum is 0 and
3869	// the carry is set, resulting in a new sum of all 1s.
3870	SDValue Zero = DAG.getConstant(Val: 0, DL: dl, VT);
3871	SDValue Overflow2 =
3872	IsAdd ? DAG.getSetCC(DL: dl, VT: SetCCType, LHS: Sum2, RHS: Zero, Cond: ISD::SETEQ)
3873	: DAG.getSetCC(DL: dl, VT: SetCCType, LHS: Sum, RHS: Zero, Cond: ISD::SETEQ);
3874	Overflow2 = DAG.getNode(Opcode: ISD::AND, DL: dl, VT: SetCCType, N1: Overflow2,
3875	N2: DAG.getZExtOrTrunc(Op: Carry, DL: dl, VT: SetCCType));
3876
3877	SDValue ResultCarry =
3878	DAG.getNode(Opcode: ISD::OR, DL: dl, VT: SetCCType, N1: Overflow, N2: Overflow2);
3879
3880	Results.push_back(Elt: Sum2);
3881	Results.push_back(Elt: DAG.getBoolExtOrTrunc(Op: ResultCarry, SL: dl, VT: CarryType, OpVT: VT));
3882	break;
3883	}
3884	case ISD::SADDO:
3885	case ISD::SSUBO: {
3886	SDValue Result, Overflow;
3887	TLI.expandSADDSUBO(Node, Result, Overflow, DAG);
3888	Results.push_back(Elt: Result);
3889	Results.push_back(Elt: Overflow);
3890	break;
3891	}
3892	case ISD::UADDO:
3893	case ISD::USUBO: {
3894	SDValue Result, Overflow;
3895	TLI.expandUADDSUBO(Node, Result, Overflow, DAG);
3896	Results.push_back(Elt: Result);
3897	Results.push_back(Elt: Overflow);
3898	break;
3899	}
3900	case ISD::UMULO:
3901	case ISD::SMULO: {
3902	SDValue Result, Overflow;
3903	if (TLI.expandMULO(Node, Result, Overflow, DAG)) {
3904	Results.push_back(Elt: Result);
3905	Results.push_back(Elt: Overflow);
3906	}
3907	break;
3908	}
3909	case ISD::BUILD_PAIR: {
3910	EVT PairTy = Node->getValueType(ResNo: 0);
3911	Tmp1 = DAG.getNode(Opcode: ISD::ZERO_EXTEND, DL: dl, VT: PairTy, Operand: Node->getOperand(Num: 0));
3912	Tmp2 = DAG.getNode(Opcode: ISD::ANY_EXTEND, DL: dl, VT: PairTy, Operand: Node->getOperand(Num: 1));
3913	Tmp2 = DAG.getNode(
3914	Opcode: ISD::SHL, DL: dl, VT: PairTy, N1: Tmp2,
3915	N2: DAG.getConstant(Val: PairTy.getSizeInBits() / 2, DL: dl,
3916	VT: TLI.getShiftAmountTy(LHSTy: PairTy, DL: DAG.getDataLayout())));
3917	Results.push_back(Elt: DAG.getNode(Opcode: ISD::OR, DL: dl, VT: PairTy, N1: Tmp1, N2: Tmp2));
3918	break;
3919	}
3920	case ISD::SELECT:
3921	Tmp1 = Node->getOperand(Num: 0);
3922	Tmp2 = Node->getOperand(Num: 1);
3923	Tmp3 = Node->getOperand(Num: 2);
3924	if (Tmp1.getOpcode() == ISD::SETCC) {
3925	Tmp1 = DAG.getSelectCC(DL: dl, LHS: Tmp1.getOperand(i: 0), RHS: Tmp1.getOperand(i: 1),
3926	True: Tmp2, False: Tmp3,
3927	Cond: cast<CondCodeSDNode>(Val: Tmp1.getOperand(i: 2))->get());
3928	} else {
3929	Tmp1 = DAG.getSelectCC(DL: dl, LHS: Tmp1,
3930	RHS: DAG.getConstant(Val: 0, DL: dl, VT: Tmp1.getValueType()),
3931	True: Tmp2, False: Tmp3, Cond: ISD::SETNE);
3932	}
3933	Tmp1->setFlags(Node->getFlags());
3934	Results.push_back(Elt: Tmp1);
3935	break;
3936	case ISD::BR_JT: {
3937	SDValue Chain = Node->getOperand(Num: 0);
3938	SDValue Table = Node->getOperand(Num: 1);
3939	SDValue Index = Node->getOperand(Num: 2);
3940	int JTI = cast<JumpTableSDNode>(Val: Table.getNode())->getIndex();
3941
3942	const DataLayout &TD = DAG.getDataLayout();
3943	EVT PTy = TLI.getPointerTy(DL: TD);
3944
3945	unsigned EntrySize =
3946	DAG.getMachineFunction().getJumpTableInfo()->getEntrySize(TD);
3947
3948	// For power-of-two jumptable entry sizes convert multiplication to a shift.
3949	// This transformation needs to be done here since otherwise the MIPS
3950	// backend will end up emitting a three instruction multiply sequence
3951	// instead of a single shift and MSP430 will call a runtime function.
3952	if (llvm::isPowerOf2_32(Value: EntrySize))
3953	Index = DAG.getNode(
3954	Opcode: ISD::SHL, DL: dl, VT: Index.getValueType(), N1: Index,
3955	N2: DAG.getConstant(Val: llvm::Log2_32(Value: EntrySize), DL: dl, VT: Index.getValueType()));
3956	else
3957	Index = DAG.getNode(Opcode: ISD::MUL, DL: dl, VT: Index.getValueType(), N1: Index,
3958	N2: DAG.getConstant(Val: EntrySize, DL: dl, VT: Index.getValueType()));
3959	SDValue Addr = DAG.getNode(Opcode: ISD::ADD, DL: dl, VT: Index.getValueType(),
3960	N1: Index, N2: Table);
3961
3962	EVT MemVT = EVT::getIntegerVT(Context&: *DAG.getContext(), BitWidth: EntrySize * 8);
3963	SDValue LD = DAG.getExtLoad(
3964	ExtType: ISD::SEXTLOAD, dl, VT: PTy, Chain, Ptr: Addr,
3965	PtrInfo: MachinePointerInfo::getJumpTable(MF&: DAG.getMachineFunction()), MemVT);
3966	Addr = LD;
3967	if (TLI.isJumpTableRelative()) {
3968	// For PIC, the sequence is:
3969	// BRIND(load(Jumptable + index) + RelocBase)
3970	// RelocBase can be JumpTable, GOT or some sort of global base.
3971	Addr = DAG.getNode(Opcode: ISD::ADD, DL: dl, VT: PTy, N1: Addr,
3972	N2: TLI.getPICJumpTableRelocBase(Table, DAG));
3973	}
3974
3975	Tmp1 = TLI.expandIndirectJTBranch(dl, Value: LD.getValue(R: 1), Addr, JTI, DAG);
3976	Results.push_back(Elt: Tmp1);
3977	break;
3978	}
3979	case ISD::BRCOND:
3980	// Expand brcond's setcc into its constituent parts and create a BR_CC
3981	// Node.
3982	Tmp1 = Node->getOperand(Num: 0);
3983	Tmp2 = Node->getOperand(Num: 1);
3984	if (Tmp2.getOpcode() == ISD::SETCC &&
3985	TLI.isOperationLegalOrCustom(Op: ISD::BR_CC,
3986	VT: Tmp2.getOperand(i: 0).getValueType())) {
3987	Tmp1 = DAG.getNode(ISD::BR_CC, dl, MVT::Other, Tmp1, Tmp2.getOperand(2),
3988	Tmp2.getOperand(0), Tmp2.getOperand(1),
3989	Node->getOperand(2));
3990	} else {
3991	// We test only the i1 bit. Skip the AND if UNDEF or another AND.
3992	if (Tmp2.isUndef() ||
3993	(Tmp2.getOpcode() == ISD::AND && isOneConstant(V: Tmp2.getOperand(i: 1))))
3994	Tmp3 = Tmp2;
3995	else
3996	Tmp3 = DAG.getNode(Opcode: ISD::AND, DL: dl, VT: Tmp2.getValueType(), N1: Tmp2,
3997	N2: DAG.getConstant(Val: 1, DL: dl, VT: Tmp2.getValueType()));
3998	Tmp1 = DAG.getNode(ISD::BR_CC, dl, MVT::Other, Tmp1,
3999	DAG.getCondCode(ISD::SETNE), Tmp3,
4000	DAG.getConstant(0, dl, Tmp3.getValueType()),
4001	Node->getOperand(2));
4002	}
4003	Results.push_back(Elt: Tmp1);
4004	break;
4005	case ISD::SETCC:
4006	case ISD::VP_SETCC:
4007	case ISD::STRICT_FSETCC:
4008	case ISD::STRICT_FSETCCS: {
4009	bool IsVP = Node->getOpcode() == ISD::VP_SETCC;
4010	bool IsStrict = Node->getOpcode() == ISD::STRICT_FSETCC ||
4011	Node->getOpcode() == ISD::STRICT_FSETCCS;
4012	bool IsSignaling = Node->getOpcode() == ISD::STRICT_FSETCCS;
4013	SDValue Chain = IsStrict ? Node->getOperand(Num: 0) : SDValue();
4014	unsigned Offset = IsStrict ? 1 : 0;
4015	Tmp1 = Node->getOperand(Num: 0 + Offset);
4016	Tmp2 = Node->getOperand(Num: 1 + Offset);
4017	Tmp3 = Node->getOperand(Num: 2 + Offset);
4018	SDValue Mask, EVL;
4019	if (IsVP) {
4020	Mask = Node->getOperand(Num: 3 + Offset);
4021	EVL = Node->getOperand(Num: 4 + Offset);
4022	}
4023	bool Legalized = TLI.LegalizeSetCCCondCode(
4024	DAG, VT: Node->getValueType(ResNo: 0), LHS&: Tmp1, RHS&: Tmp2, CC&: Tmp3, Mask, EVL, NeedInvert, dl,
4025	Chain, IsSignaling);
4026
4027	if (Legalized) {
4028	// If we expanded the SETCC by swapping LHS and RHS, or by inverting the
4029	// condition code, create a new SETCC node.
4030	if (Tmp3.getNode()) {
4031	if (IsStrict) {
4032	Tmp1 = DAG.getNode(Opcode: Node->getOpcode(), DL: dl, VTList: Node->getVTList(),
4033	Ops: {Chain, Tmp1, Tmp2, Tmp3}, Flags: Node->getFlags());
4034	Chain = Tmp1.getValue(R: 1);
4035	} else if (IsVP) {
4036	Tmp1 = DAG.getNode(Opcode: Node->getOpcode(), DL: dl, VT: Node->getValueType(ResNo: 0),
4037	Ops: {Tmp1, Tmp2, Tmp3, Mask, EVL}, Flags: Node->getFlags());
4038	} else {
4039	Tmp1 = DAG.getNode(Opcode: Node->getOpcode(), DL: dl, VT: Node->getValueType(ResNo: 0), N1: Tmp1,
4040	N2: Tmp2, N3: Tmp3, Flags: Node->getFlags());
4041	}
4042	}
4043
4044	// If we expanded the SETCC by inverting the condition code, then wrap
4045	// the existing SETCC in a NOT to restore the intended condition.
4046	if (NeedInvert) {
4047	if (!IsVP)
4048	Tmp1 = DAG.getLogicalNOT(DL: dl, Val: Tmp1, VT: Tmp1->getValueType(ResNo: 0));
4049	else
4050	Tmp1 =
4051	DAG.getVPLogicalNOT(DL: dl, Val: Tmp1, Mask, EVL, VT: Tmp1->getValueType(ResNo: 0));
4052	}
4053
4054	Results.push_back(Elt: Tmp1);
4055	if (IsStrict)
4056	Results.push_back(Elt: Chain);
4057
4058	break;
4059	}
4060
4061	// FIXME: It seems Legalized is false iff CCCode is Legal. I don't
4062	// understand if this code is useful for strict nodes.
4063	assert(!IsStrict && "Don't know how to expand for strict nodes.");
4064
4065	// Otherwise, SETCC for the given comparison type must be completely
4066	// illegal; expand it into a SELECT_CC.
4067	// FIXME: This drops the mask/evl for VP_SETCC.
4068	EVT VT = Node->getValueType(ResNo: 0);
4069	EVT Tmp1VT = Tmp1.getValueType();
4070	Tmp1 = DAG.getNode(Opcode: ISD::SELECT_CC, DL: dl, VT, N1: Tmp1, N2: Tmp2,
4071	N3: DAG.getBoolConstant(V: true, DL: dl, VT, OpVT: Tmp1VT),
4072	N4: DAG.getBoolConstant(V: false, DL: dl, VT, OpVT: Tmp1VT), N5: Tmp3);
4073	Tmp1->setFlags(Node->getFlags());
4074	Results.push_back(Elt: Tmp1);
4075	break;
4076	}
4077	case ISD::SELECT_CC: {
4078	// TODO: need to add STRICT_SELECT_CC and STRICT_SELECT_CCS
4079	Tmp1 = Node->getOperand(Num: 0); // LHS
4080	Tmp2 = Node->getOperand(Num: 1); // RHS
4081	Tmp3 = Node->getOperand(Num: 2); // True
4082	Tmp4 = Node->getOperand(Num: 3); // False
4083	EVT VT = Node->getValueType(ResNo: 0);
4084	SDValue Chain;
4085	SDValue CC = Node->getOperand(Num: 4);
4086	ISD::CondCode CCOp = cast<CondCodeSDNode>(Val&: CC)->get();
4087
4088	if (TLI.isCondCodeLegalOrCustom(CC: CCOp, VT: Tmp1.getSimpleValueType())) {
4089	// If the condition code is legal, then we need to expand this
4090	// node using SETCC and SELECT.
4091	EVT CmpVT = Tmp1.getValueType();
4092	assert(!TLI.isOperationExpand(ISD::SELECT, VT) &&
4093	"Cannot expand ISD::SELECT_CC when ISD::SELECT also needs to be "
4094	"expanded.");
4095	EVT CCVT = getSetCCResultType(VT: CmpVT);
4096	SDValue Cond = DAG.getNode(Opcode: ISD::SETCC, DL: dl, VT: CCVT, N1: Tmp1, N2: Tmp2, N3: CC, Flags: Node->getFlags());
4097	Results.push_back(Elt: DAG.getSelect(DL: dl, VT, Cond, LHS: Tmp3, RHS: Tmp4));
4098	break;
4099	}
4100
4101	// SELECT_CC is legal, so the condition code must not be.
4102	bool Legalized = false;
4103	// Try to legalize by inverting the condition. This is for targets that
4104	// might support an ordered version of a condition, but not the unordered
4105	// version (or vice versa).
4106	ISD::CondCode InvCC = ISD::getSetCCInverse(Operation: CCOp, Type: Tmp1.getValueType());
4107	if (TLI.isCondCodeLegalOrCustom(CC: InvCC, VT: Tmp1.getSimpleValueType())) {
4108	// Use the new condition code and swap true and false
4109	Legalized = true;
4110	Tmp1 = DAG.getSelectCC(DL: dl, LHS: Tmp1, RHS: Tmp2, True: Tmp4, False: Tmp3, Cond: InvCC);
4111	Tmp1->setFlags(Node->getFlags());
4112	} else {
4113	// If The inverse is not legal, then try to swap the arguments using
4114	// the inverse condition code.
4115	ISD::CondCode SwapInvCC = ISD::getSetCCSwappedOperands(Operation: InvCC);
4116	if (TLI.isCondCodeLegalOrCustom(CC: SwapInvCC, VT: Tmp1.getSimpleValueType())) {
4117	// The swapped inverse condition is legal, so swap true and false,
4118	// lhs and rhs.
4119	Legalized = true;
4120	Tmp1 = DAG.getSelectCC(DL: dl, LHS: Tmp2, RHS: Tmp1, True: Tmp4, False: Tmp3, Cond: SwapInvCC);
4121	Tmp1->setFlags(Node->getFlags());
4122	}
4123	}
4124
4125	if (!Legalized) {
4126	Legalized = TLI.LegalizeSetCCCondCode(
4127	DAG, VT: getSetCCResultType(VT: Tmp1.getValueType()), LHS&: Tmp1, RHS&: Tmp2, CC,
4128	/*Mask*/ SDValue(), /*EVL*/ SDValue(), NeedInvert, dl, Chain);
4129
4130	assert(Legalized && "Can't legalize SELECT_CC with legal condition!");
4131
4132	// If we expanded the SETCC by inverting the condition code, then swap
4133	// the True/False operands to match.
4134	if (NeedInvert)
4135	std::swap(a&: Tmp3, b&: Tmp4);
4136
4137	// If we expanded the SETCC by swapping LHS and RHS, or by inverting the
4138	// condition code, create a new SELECT_CC node.
4139	if (CC.getNode()) {
4140	Tmp1 = DAG.getNode(Opcode: ISD::SELECT_CC, DL: dl, VT: Node->getValueType(ResNo: 0),
4141	N1: Tmp1, N2: Tmp2, N3: Tmp3, N4: Tmp4, N5: CC);
4142	} else {
4143	Tmp2 = DAG.getConstant(Val: 0, DL: dl, VT: Tmp1.getValueType());
4144	CC = DAG.getCondCode(Cond: ISD::SETNE);
4145	Tmp1 = DAG.getNode(Opcode: ISD::SELECT_CC, DL: dl, VT: Node->getValueType(ResNo: 0), N1: Tmp1,
4146	N2: Tmp2, N3: Tmp3, N4: Tmp4, N5: CC);
4147	}
4148	Tmp1->setFlags(Node->getFlags());
4149	}
4150	Results.push_back(Elt: Tmp1);
4151	break;
4152	}
4153	case ISD::BR_CC: {
4154	// TODO: need to add STRICT_BR_CC and STRICT_BR_CCS
4155	SDValue Chain;
4156	Tmp1 = Node->getOperand(Num: 0); // Chain
4157	Tmp2 = Node->getOperand(Num: 2); // LHS
4158	Tmp3 = Node->getOperand(Num: 3); // RHS
4159	Tmp4 = Node->getOperand(Num: 1); // CC
4160
4161	bool Legalized = TLI.LegalizeSetCCCondCode(
4162	DAG, VT: getSetCCResultType(VT: Tmp2.getValueType()), LHS&: Tmp2, RHS&: Tmp3, CC&: Tmp4,
4163	/*Mask*/ SDValue(), /*EVL*/ SDValue(), NeedInvert, dl, Chain);
4164	(void)Legalized;
4165	assert(Legalized && "Can't legalize BR_CC with legal condition!");
4166
4167	// If we expanded the SETCC by swapping LHS and RHS, create a new BR_CC
4168	// node.
4169	if (Tmp4.getNode()) {
4170	assert(!NeedInvert && "Don't know how to invert BR_CC!");
4171
4172	Tmp1 = DAG.getNode(Opcode: ISD::BR_CC, DL: dl, VT: Node->getValueType(ResNo: 0), N1: Tmp1,
4173	N2: Tmp4, N3: Tmp2, N4: Tmp3, N5: Node->getOperand(Num: 4));
4174	} else {
4175	Tmp3 = DAG.getConstant(Val: 0, DL: dl, VT: Tmp2.getValueType());
4176	Tmp4 = DAG.getCondCode(Cond: NeedInvert ? ISD::SETEQ : ISD::SETNE);
4177	Tmp1 = DAG.getNode(Opcode: ISD::BR_CC, DL: dl, VT: Node->getValueType(ResNo: 0), N1: Tmp1, N2: Tmp4,
4178	N3: Tmp2, N4: Tmp3, N5: Node->getOperand(Num: 4));
4179	}
4180	Results.push_back(Elt: Tmp1);
4181	break;
4182	}
4183	case ISD::BUILD_VECTOR:
4184	Results.push_back(Elt: ExpandBUILD_VECTOR(Node));
4185	break;
4186	case ISD::SPLAT_VECTOR:
4187	Results.push_back(Elt: ExpandSPLAT_VECTOR(Node));
4188	break;
4189	case ISD::SRA:
4190	case ISD::SRL:
4191	case ISD::SHL: {
4192	// Scalarize vector SRA/SRL/SHL.
4193	EVT VT = Node->getValueType(ResNo: 0);
4194	assert(VT.isVector() && "Unable to legalize non-vector shift");
4195	assert(TLI.isTypeLegal(VT.getScalarType())&& "Element type must be legal");
4196	unsigned NumElem = VT.getVectorNumElements();
4197
4198	SmallVector<SDValue, 8> Scalars;
4199	for (unsigned Idx = 0; Idx < NumElem; Idx++) {
4200	SDValue Ex =
4201	DAG.getNode(Opcode: ISD::EXTRACT_VECTOR_ELT, DL: dl, VT: VT.getScalarType(),
4202	N1: Node->getOperand(Num: 0), N2: DAG.getVectorIdxConstant(Val: Idx, DL: dl));
4203	SDValue Sh =
4204	DAG.getNode(Opcode: ISD::EXTRACT_VECTOR_ELT, DL: dl, VT: VT.getScalarType(),
4205	N1: Node->getOperand(Num: 1), N2: DAG.getVectorIdxConstant(Val: Idx, DL: dl));
4206	Scalars.push_back(Elt: DAG.getNode(Opcode: Node->getOpcode(), DL: dl,
4207	VT: VT.getScalarType(), N1: Ex, N2: Sh));
4208	}
4209
4210	SDValue Result = DAG.getBuildVector(VT: Node->getValueType(ResNo: 0), DL: dl, Ops: Scalars);
4211	Results.push_back(Elt: Result);
4212	break;
4213	}
4214	case ISD::VECREDUCE_FADD:
4215	case ISD::VECREDUCE_FMUL:
4216	case ISD::VECREDUCE_ADD:
4217	case ISD::VECREDUCE_MUL:
4218	case ISD::VECREDUCE_AND:
4219	case ISD::VECREDUCE_OR:
4220	case ISD::VECREDUCE_XOR:
4221	case ISD::VECREDUCE_SMAX:
4222	case ISD::VECREDUCE_SMIN:
4223	case ISD::VECREDUCE_UMAX:
4224	case ISD::VECREDUCE_UMIN:
4225	case ISD::VECREDUCE_FMAX:
4226	case ISD::VECREDUCE_FMIN:
4227	case ISD::VECREDUCE_FMAXIMUM:
4228	case ISD::VECREDUCE_FMINIMUM:
4229	Results.push_back(Elt: TLI.expandVecReduce(Node, DAG));
4230	break;
4231	case ISD::GLOBAL_OFFSET_TABLE:
4232	case ISD::GlobalAddress:
4233	case ISD::GlobalTLSAddress:
4234	case ISD::ExternalSymbol:
4235	case ISD::ConstantPool:
4236	case ISD::JumpTable:
4237	case ISD::INTRINSIC_W_CHAIN:
4238	case ISD::INTRINSIC_WO_CHAIN:
4239	case ISD::INTRINSIC_VOID:
4240	// FIXME: Custom lowering for these operations shouldn't return null!
4241	// Return true so that we don't call ConvertNodeToLibcall which also won't
4242	// do anything.
4243	return true;
4244	}
4245
4246	if (!TLI.isStrictFPEnabled() && Results.empty() && Node->isStrictFPOpcode()) {
4247	// FIXME: We were asked to expand a strict floating-point operation,
4248	// but there is currently no expansion implemented that would preserve
4249	// the "strict" properties. For now, we just fall back to the non-strict
4250	// version if that is legal on the target. The actual mutation of the
4251	// operation will happen in SelectionDAGISel::DoInstructionSelection.
4252	switch (Node->getOpcode()) {
4253	default:
4254	if (TLI.getStrictFPOperationAction(Op: Node->getOpcode(),
4255	VT: Node->getValueType(ResNo: 0))
4256	== TargetLowering::Legal)
4257	return true;
4258	break;
4259	case ISD::STRICT_FSUB: {
4260	if (TLI.getStrictFPOperationAction(
4261	Op: ISD::STRICT_FSUB, VT: Node->getValueType(ResNo: 0)) == TargetLowering::Legal)
4262	return true;
4263	if (TLI.getStrictFPOperationAction(
4264	Op: ISD::STRICT_FADD, VT: Node->getValueType(ResNo: 0)) != TargetLowering::Legal)
4265	break;
4266
4267	EVT VT = Node->getValueType(ResNo: 0);
4268	const SDNodeFlags Flags = Node->getFlags();
4269	SDValue Neg = DAG.getNode(Opcode: ISD::FNEG, DL: dl, VT, Operand: Node->getOperand(Num: 2), Flags);
4270	SDValue Fadd = DAG.getNode(Opcode: ISD::STRICT_FADD, DL: dl, VTList: Node->getVTList(),
4271	Ops: {Node->getOperand(Num: 0), Node->getOperand(Num: 1), Neg},
4272	Flags);
4273
4274	Results.push_back(Elt: Fadd);
4275	Results.push_back(Elt: Fadd.getValue(R: 1));
4276	break;
4277	}
4278	case ISD::STRICT_SINT_TO_FP:
4279	case ISD::STRICT_UINT_TO_FP:
4280	case ISD::STRICT_LRINT:
4281	case ISD::STRICT_LLRINT:
4282	case ISD::STRICT_LROUND:
4283	case ISD::STRICT_LLROUND:
4284	// These are registered by the operand type instead of the value
4285	// type. Reflect that here.
4286	if (TLI.getStrictFPOperationAction(Op: Node->getOpcode(),
4287	VT: Node->getOperand(Num: 1).getValueType())
4288	== TargetLowering::Legal)
4289	return true;
4290	break;
4291	}
4292	}
4293
4294	// Replace the original node with the legalized result.
4295	if (Results.empty()) {
4296	LLVM_DEBUG(dbgs() << "Cannot expand node\n");
4297	return false;
4298	}
4299
4300	LLVM_DEBUG(dbgs() << "Successfully expanded node\n");
4301	ReplaceNode(Old: Node, New: Results.data());
4302	return true;
4303	}
4304
4305	void SelectionDAGLegalize::ConvertNodeToLibcall(SDNode *Node) {
4306	LLVM_DEBUG(dbgs() << "Trying to convert node to libcall\n");
4307	SmallVector<SDValue, 8> Results;
4308	SDLoc dl(Node);
4309	// FIXME: Check flags on the node to see if we can use a finite call.
4310	unsigned Opc = Node->getOpcode();
4311	switch (Opc) {
4312	case ISD::ATOMIC_FENCE: {
4313	// If the target didn't lower this, lower it to '__sync_synchronize()' call
4314	// FIXME: handle "fence singlethread" more efficiently.
4315	TargetLowering::ArgListTy Args;
4316
4317	TargetLowering::CallLoweringInfo CLI(DAG);
4318	CLI.setDebugLoc(dl)
4319	.setChain(Node->getOperand(Num: 0))
4320	.setLibCallee(
4321	CC: CallingConv::C, ResultType: Type::getVoidTy(C&: *DAG.getContext()),
4322	Target: DAG.getExternalSymbol(Sym: "__sync_synchronize",
4323	VT: TLI.getPointerTy(DL: DAG.getDataLayout())),
4324	ArgsList: std::move(Args));
4325
4326	std::pair<SDValue, SDValue> CallResult = TLI.LowerCallTo(CLI);
4327
4328	Results.push_back(Elt: CallResult.second);
4329	break;
4330	}
4331	// By default, atomic intrinsics are marked Legal and lowered. Targets
4332	// which don't support them directly, however, may want libcalls, in which
4333	// case they mark them Expand, and we get here.
4334	case ISD::ATOMIC_SWAP:
4335	case ISD::ATOMIC_LOAD_ADD:
4336	case ISD::ATOMIC_LOAD_SUB:
4337	case ISD::ATOMIC_LOAD_AND:
4338	case ISD::ATOMIC_LOAD_CLR:
4339	case ISD::ATOMIC_LOAD_OR:
4340	case ISD::ATOMIC_LOAD_XOR:
4341	case ISD::ATOMIC_LOAD_NAND:
4342	case ISD::ATOMIC_LOAD_MIN:
4343	case ISD::ATOMIC_LOAD_MAX:
4344	case ISD::ATOMIC_LOAD_UMIN:
4345	case ISD::ATOMIC_LOAD_UMAX:
4346	case ISD::ATOMIC_CMP_SWAP: {
4347	MVT VT = cast<AtomicSDNode>(Val: Node)->getMemoryVT().getSimpleVT();
4348	AtomicOrdering Order = cast<AtomicSDNode>(Val: Node)->getMergedOrdering();
4349	RTLIB::Libcall LC = RTLIB::getOUTLINE_ATOMIC(Opc, Order, VT);
4350	EVT RetVT = Node->getValueType(ResNo: 0);
4351	TargetLowering::MakeLibCallOptions CallOptions;
4352	SmallVector<SDValue, 4> Ops;
4353	if (TLI.getLibcallName(Call: LC)) {
4354	// If outline atomic available, prepare its arguments and expand.
4355	Ops.append(in_start: Node->op_begin() + 2, in_end: Node->op_end());
4356	Ops.push_back(Elt: Node->getOperand(Num: 1));
4357
4358	} else {
4359	LC = RTLIB::getSYNC(Opc, VT);
4360	assert(LC != RTLIB::UNKNOWN_LIBCALL &&
4361	"Unexpected atomic op or value type!");
4362	// Arguments for expansion to sync libcall
4363	Ops.append(in_start: Node->op_begin() + 1, in_end: Node->op_end());
4364	}
4365	std::pair<SDValue, SDValue> Tmp = TLI.makeLibCall(DAG, LC, RetVT,
4366	Ops, CallOptions,
4367	dl: SDLoc(Node),
4368	Chain: Node->getOperand(Num: 0));
4369	Results.push_back(Elt: Tmp.first);
4370	Results.push_back(Elt: Tmp.second);
4371	break;
4372	}
4373	case ISD::TRAP: {
4374	// If this operation is not supported, lower it to 'abort()' call
4375	TargetLowering::ArgListTy Args;
4376	TargetLowering::CallLoweringInfo CLI(DAG);
4377	CLI.setDebugLoc(dl)
4378	.setChain(Node->getOperand(Num: 0))
4379	.setLibCallee(CC: CallingConv::C, ResultType: Type::getVoidTy(C&: *DAG.getContext()),
4380	Target: DAG.getExternalSymbol(
4381	Sym: "abort", VT: TLI.getPointerTy(DL: DAG.getDataLayout())),
4382	ArgsList: std::move(Args));
4383	std::pair<SDValue, SDValue> CallResult = TLI.LowerCallTo(CLI);
4384
4385	Results.push_back(Elt: CallResult.second);
4386	break;
4387	}
4388	case ISD::FMINNUM:
4389	case ISD::STRICT_FMINNUM:
4390	ExpandFPLibCall(Node, Call_F32: RTLIB::FMIN_F32, Call_F64: RTLIB::FMIN_F64,
4391	Call_F80: RTLIB::FMIN_F80, Call_F128: RTLIB::FMIN_F128,
4392	Call_PPCF128: RTLIB::FMIN_PPCF128, Results);
4393	break;
4394	// FIXME: We do not have libcalls for FMAXIMUM and FMINIMUM. So, we cannot use
4395	// libcall legalization for these nodes, but there is no default expasion for
4396	// these nodes either (see PR63267 for example).
4397	case ISD::FMAXNUM:
4398	case ISD::STRICT_FMAXNUM:
4399	ExpandFPLibCall(Node, Call_F32: RTLIB::FMAX_F32, Call_F64: RTLIB::FMAX_F64,
4400	Call_F80: RTLIB::FMAX_F80, Call_F128: RTLIB::FMAX_F128,
4401	Call_PPCF128: RTLIB::FMAX_PPCF128, Results);
4402	break;
4403	case ISD::FSQRT:
4404	case ISD::STRICT_FSQRT:
4405	ExpandFPLibCall(Node, Call_F32: RTLIB::SQRT_F32, Call_F64: RTLIB::SQRT_F64,
4406	Call_F80: RTLIB::SQRT_F80, Call_F128: RTLIB::SQRT_F128,
4407	Call_PPCF128: RTLIB::SQRT_PPCF128, Results);
4408	break;
4409	case ISD::FCBRT:
4410	ExpandFPLibCall(Node, Call_F32: RTLIB::CBRT_F32, Call_F64: RTLIB::CBRT_F64,
4411	Call_F80: RTLIB::CBRT_F80, Call_F128: RTLIB::CBRT_F128,
4412	Call_PPCF128: RTLIB::CBRT_PPCF128, Results);
4413	break;
4414	case ISD::FSIN:
4415	case ISD::STRICT_FSIN:
4416	ExpandFPLibCall(Node, Call_F32: RTLIB::SIN_F32, Call_F64: RTLIB::SIN_F64,
4417	Call_F80: RTLIB::SIN_F80, Call_F128: RTLIB::SIN_F128,
4418	Call_PPCF128: RTLIB::SIN_PPCF128, Results);
4419	break;
4420	case ISD::FCOS:
4421	case ISD::STRICT_FCOS:
4422	ExpandFPLibCall(Node, Call_F32: RTLIB::COS_F32, Call_F64: RTLIB::COS_F64,
4423	Call_F80: RTLIB::COS_F80, Call_F128: RTLIB::COS_F128,
4424	Call_PPCF128: RTLIB::COS_PPCF128, Results);
4425	break;
4426	case ISD::FSINCOS:
4427	// Expand into sincos libcall.
4428	ExpandSinCosLibCall(Node, Results);
4429	break;
4430	case ISD::FLOG:
4431	case ISD::STRICT_FLOG:
4432	ExpandFPLibCall(Node, Call_F32: RTLIB::LOG_F32, Call_F64: RTLIB::LOG_F64, Call_F80: RTLIB::LOG_F80,
4433	Call_F128: RTLIB::LOG_F128, Call_PPCF128: RTLIB::LOG_PPCF128, Results);
4434	break;
4435	case ISD::FLOG2:
4436	case ISD::STRICT_FLOG2:
4437	ExpandFPLibCall(Node, Call_F32: RTLIB::LOG2_F32, Call_F64: RTLIB::LOG2_F64, Call_F80: RTLIB::LOG2_F80,
4438	Call_F128: RTLIB::LOG2_F128, Call_PPCF128: RTLIB::LOG2_PPCF128, Results);
4439	break;
4440	case ISD::FLOG10:
4441	case ISD::STRICT_FLOG10:
4442	ExpandFPLibCall(Node, Call_F32: RTLIB::LOG10_F32, Call_F64: RTLIB::LOG10_F64, Call_F80: RTLIB::LOG10_F80,
4443	Call_F128: RTLIB::LOG10_F128, Call_PPCF128: RTLIB::LOG10_PPCF128, Results);
4444	break;
4445	case ISD::FEXP:
4446	case ISD::STRICT_FEXP:
4447	ExpandFPLibCall(Node, Call_F32: RTLIB::EXP_F32, Call_F64: RTLIB::EXP_F64, Call_F80: RTLIB::EXP_F80,
4448	Call_F128: RTLIB::EXP_F128, Call_PPCF128: RTLIB::EXP_PPCF128, Results);
4449	break;
4450	case ISD::FEXP2:
4451	case ISD::STRICT_FEXP2:
4452	ExpandFPLibCall(Node, Call_F32: RTLIB::EXP2_F32, Call_F64: RTLIB::EXP2_F64, Call_F80: RTLIB::EXP2_F80,
4453	Call_F128: RTLIB::EXP2_F128, Call_PPCF128: RTLIB::EXP2_PPCF128, Results);
4454	break;
4455	case ISD::FEXP10:
4456	ExpandFPLibCall(Node, Call_F32: RTLIB::EXP10_F32, Call_F64: RTLIB::EXP10_F64, Call_F80: RTLIB::EXP10_F80,
4457	Call_F128: RTLIB::EXP10_F128, Call_PPCF128: RTLIB::EXP10_PPCF128, Results);
4458	break;
4459	case ISD::FTRUNC:
4460	case ISD::STRICT_FTRUNC:
4461	ExpandFPLibCall(Node, Call_F32: RTLIB::TRUNC_F32, Call_F64: RTLIB::TRUNC_F64,
4462	Call_F80: RTLIB::TRUNC_F80, Call_F128: RTLIB::TRUNC_F128,
4463	Call_PPCF128: RTLIB::TRUNC_PPCF128, Results);
4464	break;
4465	case ISD::FFLOOR:
4466	case ISD::STRICT_FFLOOR:
4467	ExpandFPLibCall(Node, Call_F32: RTLIB::FLOOR_F32, Call_F64: RTLIB::FLOOR_F64,
4468	Call_F80: RTLIB::FLOOR_F80, Call_F128: RTLIB::FLOOR_F128,
4469	Call_PPCF128: RTLIB::FLOOR_PPCF128, Results);
4470	break;
4471	case ISD::FCEIL:
4472	case ISD::STRICT_FCEIL:
4473	ExpandFPLibCall(Node, Call_F32: RTLIB::CEIL_F32, Call_F64: RTLIB::CEIL_F64,
4474	Call_F80: RTLIB::CEIL_F80, Call_F128: RTLIB::CEIL_F128,
4475	Call_PPCF128: RTLIB::CEIL_PPCF128, Results);
4476	break;
4477	case ISD::FRINT:
4478	case ISD::STRICT_FRINT:
4479	ExpandFPLibCall(Node, Call_F32: RTLIB::RINT_F32, Call_F64: RTLIB::RINT_F64,
4480	Call_F80: RTLIB::RINT_F80, Call_F128: RTLIB::RINT_F128,
4481	Call_PPCF128: RTLIB::RINT_PPCF128, Results);
4482	break;
4483	case ISD::FNEARBYINT:
4484	case ISD::STRICT_FNEARBYINT:
4485	ExpandFPLibCall(Node, Call_F32: RTLIB::NEARBYINT_F32,
4486	Call_F64: RTLIB::NEARBYINT_F64,
4487	Call_F80: RTLIB::NEARBYINT_F80,
4488	Call_F128: RTLIB::NEARBYINT_F128,
4489	Call_PPCF128: RTLIB::NEARBYINT_PPCF128, Results);
4490	break;
4491	case ISD::FROUND:
4492	case ISD::STRICT_FROUND:
4493	ExpandFPLibCall(Node, Call_F32: RTLIB::ROUND_F32,
4494	Call_F64: RTLIB::ROUND_F64,
4495	Call_F80: RTLIB::ROUND_F80,
4496	Call_F128: RTLIB::ROUND_F128,
4497	Call_PPCF128: RTLIB::ROUND_PPCF128, Results);
4498	break;
4499	case ISD::FROUNDEVEN:
4500	case ISD::STRICT_FROUNDEVEN:
4501	ExpandFPLibCall(Node, Call_F32: RTLIB::ROUNDEVEN_F32,
4502	Call_F64: RTLIB::ROUNDEVEN_F64,
4503	Call_F80: RTLIB::ROUNDEVEN_F80,
4504	Call_F128: RTLIB::ROUNDEVEN_F128,
4505	Call_PPCF128: RTLIB::ROUNDEVEN_PPCF128, Results);
4506	break;
4507	case ISD::FLDEXP:
4508	case ISD::STRICT_FLDEXP:
4509	ExpandFPLibCall(Node, Call_F32: RTLIB::LDEXP_F32, Call_F64: RTLIB::LDEXP_F64, Call_F80: RTLIB::LDEXP_F80,
4510	Call_F128: RTLIB::LDEXP_F128, Call_PPCF128: RTLIB::LDEXP_PPCF128, Results);
4511	break;
4512	case ISD::FFREXP: {
4513	ExpandFrexpLibCall(Node, Results);
4514	break;
4515	}
4516	case ISD::FPOWI:
4517	case ISD::STRICT_FPOWI: {
4518	RTLIB::Libcall LC = RTLIB::getPOWI(RetVT: Node->getSimpleValueType(ResNo: 0));
4519	assert(LC != RTLIB::UNKNOWN_LIBCALL && "Unexpected fpowi.");
4520	if (!TLI.getLibcallName(Call: LC)) {
4521	// Some targets don't have a powi libcall; use pow instead.
4522	if (Node->isStrictFPOpcode()) {
4523	SDValue Exponent =
4524	DAG.getNode(Opcode: ISD::STRICT_SINT_TO_FP, DL: SDLoc(Node),
4525	ResultTys: {Node->getValueType(ResNo: 0), Node->getValueType(ResNo: 1)},
4526	Ops: {Node->getOperand(Num: 0), Node->getOperand(Num: 2)});
4527	SDValue FPOW =
4528	DAG.getNode(Opcode: ISD::STRICT_FPOW, DL: SDLoc(Node),
4529	ResultTys: {Node->getValueType(ResNo: 0), Node->getValueType(ResNo: 1)},
4530	Ops: {Exponent.getValue(R: 1), Node->getOperand(Num: 1), Exponent});
4531	Results.push_back(Elt: FPOW);
4532	Results.push_back(Elt: FPOW.getValue(R: 1));
4533	} else {
4534	SDValue Exponent =
4535	DAG.getNode(Opcode: ISD::SINT_TO_FP, DL: SDLoc(Node), VT: Node->getValueType(ResNo: 0),
4536	Operand: Node->getOperand(Num: 1));
4537	Results.push_back(Elt: DAG.getNode(Opcode: ISD::FPOW, DL: SDLoc(Node),
4538	VT: Node->getValueType(ResNo: 0),
4539	N1: Node->getOperand(Num: 0), N2: Exponent));
4540	}
4541	break;
4542	}
4543	unsigned Offset = Node->isStrictFPOpcode() ? 1 : 0;
4544	bool ExponentHasSizeOfInt =
4545	DAG.getLibInfo().getIntSize() ==
4546	Node->getOperand(Num: 1 + Offset).getValueType().getSizeInBits();
4547	if (!ExponentHasSizeOfInt) {
4548	// If the exponent does not match with sizeof(int) a libcall to
4549	// RTLIB::POWI would use the wrong type for the argument.
4550	DAG.getContext()->emitError(ErrorStr: "POWI exponent does not match sizeof(int)");
4551	Results.push_back(Elt: DAG.getUNDEF(VT: Node->getValueType(ResNo: 0)));
4552	break;
4553	}
4554	ExpandFPLibCall(Node, LC, Results);
4555	break;
4556	}
4557	case ISD::FPOW:
4558	case ISD::STRICT_FPOW:
4559	ExpandFPLibCall(Node, Call_F32: RTLIB::POW_F32, Call_F64: RTLIB::POW_F64, Call_F80: RTLIB::POW_F80,
4560	Call_F128: RTLIB::POW_F128, Call_PPCF128: RTLIB::POW_PPCF128, Results);
4561	break;
4562	case ISD::LROUND:
4563	case ISD::STRICT_LROUND:
4564	ExpandArgFPLibCall(Node, Call_F32: RTLIB::LROUND_F32,
4565	Call_F64: RTLIB::LROUND_F64, Call_F80: RTLIB::LROUND_F80,
4566	Call_F128: RTLIB::LROUND_F128,
4567	Call_PPCF128: RTLIB::LROUND_PPCF128, Results);
4568	break;
4569	case ISD::LLROUND:
4570	case ISD::STRICT_LLROUND:
4571	ExpandArgFPLibCall(Node, Call_F32: RTLIB::LLROUND_F32,
4572	Call_F64: RTLIB::LLROUND_F64, Call_F80: RTLIB::LLROUND_F80,
4573	Call_F128: RTLIB::LLROUND_F128,
4574	Call_PPCF128: RTLIB::LLROUND_PPCF128, Results);
4575	break;
4576	case ISD::LRINT:
4577	case ISD::STRICT_LRINT:
4578	ExpandArgFPLibCall(Node, Call_F32: RTLIB::LRINT_F32,
4579	Call_F64: RTLIB::LRINT_F64, Call_F80: RTLIB::LRINT_F80,
4580	Call_F128: RTLIB::LRINT_F128,
4581	Call_PPCF128: RTLIB::LRINT_PPCF128, Results);
4582	break;
4583	case ISD::LLRINT:
4584	case ISD::STRICT_LLRINT:
4585	ExpandArgFPLibCall(Node, Call_F32: RTLIB::LLRINT_F32,
4586	Call_F64: RTLIB::LLRINT_F64, Call_F80: RTLIB::LLRINT_F80,
4587	Call_F128: RTLIB::LLRINT_F128,
4588	Call_PPCF128: RTLIB::LLRINT_PPCF128, Results);
4589	break;
4590	case ISD::FDIV:
4591	case ISD::STRICT_FDIV:
4592	ExpandFPLibCall(Node, Call_F32: RTLIB::DIV_F32, Call_F64: RTLIB::DIV_F64,
4593	Call_F80: RTLIB::DIV_F80, Call_F128: RTLIB::DIV_F128,
4594	Call_PPCF128: RTLIB::DIV_PPCF128, Results);
4595	break;
4596	case ISD::FREM:
4597	case ISD::STRICT_FREM:
4598	ExpandFPLibCall(Node, Call_F32: RTLIB::REM_F32, Call_F64: RTLIB::REM_F64,
4599	Call_F80: RTLIB::REM_F80, Call_F128: RTLIB::REM_F128,
4600	Call_PPCF128: RTLIB::REM_PPCF128, Results);
4601	break;
4602	case ISD::FMA:
4603	case ISD::STRICT_FMA:
4604	ExpandFPLibCall(Node, Call_F32: RTLIB::FMA_F32, Call_F64: RTLIB::FMA_F64,
4605	Call_F80: RTLIB::FMA_F80, Call_F128: RTLIB::FMA_F128,
4606	Call_PPCF128: RTLIB::FMA_PPCF128, Results);
4607	break;
4608	case ISD::FADD:
4609	case ISD::STRICT_FADD:
4610	ExpandFPLibCall(Node, Call_F32: RTLIB::ADD_F32, Call_F64: RTLIB::ADD_F64,
4611	Call_F80: RTLIB::ADD_F80, Call_F128: RTLIB::ADD_F128,
4612	Call_PPCF128: RTLIB::ADD_PPCF128, Results);
4613	break;
4614	case ISD::FMUL:
4615	case ISD::STRICT_FMUL:
4616	ExpandFPLibCall(Node, Call_F32: RTLIB::MUL_F32, Call_F64: RTLIB::MUL_F64,
4617	Call_F80: RTLIB::MUL_F80, Call_F128: RTLIB::MUL_F128,
4618	Call_PPCF128: RTLIB::MUL_PPCF128, Results);
4619	break;
4620	case ISD::FP16_TO_FP:
4621	if (Node->getValueType(0) == MVT::f32) {
4622	Results.push_back(Elt: ExpandLibCall(LC: RTLIB::FPEXT_F16_F32, Node, isSigned: false).first);
4623	}
4624	break;
4625	case ISD::STRICT_BF16_TO_FP:
4626	if (Node->getValueType(0) == MVT::f32) {
4627	TargetLowering::MakeLibCallOptions CallOptions;
4628	std::pair<SDValue, SDValue> Tmp = TLI.makeLibCall(
4629	DAG, RTLIB::FPEXT_BF16_F32, MVT::f32, Node->getOperand(1),
4630	CallOptions, SDLoc(Node), Node->getOperand(0));
4631	Results.push_back(Elt: Tmp.first);
4632	Results.push_back(Elt: Tmp.second);
4633	}
4634	break;
4635	case ISD::STRICT_FP16_TO_FP: {
4636	if (Node->getValueType(0) == MVT::f32) {
4637	TargetLowering::MakeLibCallOptions CallOptions;
4638	std::pair<SDValue, SDValue> Tmp = TLI.makeLibCall(
4639	DAG, RTLIB::FPEXT_F16_F32, MVT::f32, Node->getOperand(1), CallOptions,
4640	SDLoc(Node), Node->getOperand(0));
4641	Results.push_back(Elt: Tmp.first);
4642	Results.push_back(Elt: Tmp.second);
4643	}
4644	break;
4645	}
4646	case ISD::FP_TO_FP16: {
4647	RTLIB::Libcall LC =
4648	RTLIB::getFPROUND(Node->getOperand(0).getValueType(), MVT::f16);
4649	assert(LC != RTLIB::UNKNOWN_LIBCALL && "Unable to expand fp_to_fp16");
4650	Results.push_back(Elt: ExpandLibCall(LC, Node, isSigned: false).first);
4651	break;
4652	}
4653	case ISD::FP_TO_BF16: {
4654	RTLIB::Libcall LC =
4655	RTLIB::getFPROUND(Node->getOperand(0).getValueType(), MVT::bf16);
4656	assert(LC != RTLIB::UNKNOWN_LIBCALL && "Unable to expand fp_to_bf16");
4657	Results.push_back(Elt: ExpandLibCall(LC, Node, isSigned: false).first);
4658	break;
4659	}
4660	case ISD::STRICT_SINT_TO_FP:
4661	case ISD::STRICT_UINT_TO_FP:
4662	case ISD::SINT_TO_FP:
4663	case ISD::UINT_TO_FP: {
4664	// TODO - Common the code with DAGTypeLegalizer::SoftenFloatRes_XINT_TO_FP
4665	bool IsStrict = Node->isStrictFPOpcode();
4666	bool Signed = Node->getOpcode() == ISD::SINT_TO_FP ||
4667	Node->getOpcode() == ISD::STRICT_SINT_TO_FP;
4668	EVT SVT = Node->getOperand(Num: IsStrict ? 1 : 0).getValueType();
4669	EVT RVT = Node->getValueType(ResNo: 0);
4670	EVT NVT = EVT();
4671	SDLoc dl(Node);
4672
4673	// Even if the input is legal, no libcall may exactly match, eg. we don't
4674	// have i1 -> fp conversions. So, it needs to be promoted to a larger type,
4675	// eg: i13 -> fp. Then, look for an appropriate libcall.
4676	RTLIB::Libcall LC = RTLIB::UNKNOWN_LIBCALL;
4677	for (unsigned t = MVT::FIRST_INTEGER_VALUETYPE;
4678	t <= MVT::LAST_INTEGER_VALUETYPE && LC == RTLIB::UNKNOWN_LIBCALL;
4679	++t) {
4680	NVT = (MVT::SimpleValueType)t;
4681	// The source needs to big enough to hold the operand.
4682	if (NVT.bitsGE(SVT))
4683	LC = Signed ? RTLIB::getSINTTOFP(NVT, RVT)
4684	: RTLIB::getUINTTOFP(NVT, RVT);
4685	}
4686	assert(LC != RTLIB::UNKNOWN_LIBCALL && "Unable to legalize as libcall");
4687
4688	SDValue Chain = IsStrict ? Node->getOperand(Num: 0) : SDValue();
4689	// Sign/zero extend the argument if the libcall takes a larger type.
4690	SDValue Op = DAG.getNode(Opcode: Signed ? ISD::SIGN_EXTEND : ISD::ZERO_EXTEND, DL: dl,
4691	VT: NVT, Operand: Node->getOperand(Num: IsStrict ? 1 : 0));
4692	TargetLowering::MakeLibCallOptions CallOptions;
4693	CallOptions.setSExt(Signed);
4694	std::pair<SDValue, SDValue> Tmp =
4695	TLI.makeLibCall(DAG, LC, RetVT: RVT, Ops: Op, CallOptions, dl, Chain);
4696	Results.push_back(Elt: Tmp.first);
4697	if (IsStrict)
4698	Results.push_back(Elt: Tmp.second);
4699	break;
4700	}
4701	case ISD::FP_TO_SINT:
4702	case ISD::FP_TO_UINT:
4703	case ISD::STRICT_FP_TO_SINT:
4704	case ISD::STRICT_FP_TO_UINT: {
4705	// TODO - Common the code with DAGTypeLegalizer::SoftenFloatOp_FP_TO_XINT.
4706	bool IsStrict = Node->isStrictFPOpcode();
4707	bool Signed = Node->getOpcode() == ISD::FP_TO_SINT ||
4708	Node->getOpcode() == ISD::STRICT_FP_TO_SINT;
4709
4710	SDValue Op = Node->getOperand(Num: IsStrict ? 1 : 0);
4711	EVT SVT = Op.getValueType();
4712	EVT RVT = Node->getValueType(ResNo: 0);
4713	EVT NVT = EVT();
4714	SDLoc dl(Node);
4715
4716	// Even if the result is legal, no libcall may exactly match, eg. we don't
4717	// have fp -> i1 conversions. So, it needs to be promoted to a larger type,
4718	// eg: fp -> i32. Then, look for an appropriate libcall.
4719	RTLIB::Libcall LC = RTLIB::UNKNOWN_LIBCALL;
4720	for (unsigned IntVT = MVT::FIRST_INTEGER_VALUETYPE;
4721	IntVT <= MVT::LAST_INTEGER_VALUETYPE && LC == RTLIB::UNKNOWN_LIBCALL;
4722	++IntVT) {
4723	NVT = (MVT::SimpleValueType)IntVT;
4724	// The type needs to big enough to hold the result.
4725	if (NVT.bitsGE(RVT))
4726	LC = Signed ? RTLIB::getFPTOSINT(SVT, NVT)
4727	: RTLIB::getFPTOUINT(SVT, NVT);
4728	}
4729	assert(LC != RTLIB::UNKNOWN_LIBCALL && "Unable to legalize as libcall");
4730
4731	SDValue Chain = IsStrict ? Node->getOperand(Num: 0) : SDValue();
4732	TargetLowering::MakeLibCallOptions CallOptions;
4733	std::pair<SDValue, SDValue> Tmp =
4734	TLI.makeLibCall(DAG, LC, RetVT: NVT, Ops: Op, CallOptions, dl, Chain);
4735
4736	// Truncate the result if the libcall returns a larger type.
4737	Results.push_back(Elt: DAG.getNode(Opcode: ISD::TRUNCATE, DL: dl, VT: RVT, Operand: Tmp.first));
4738	if (IsStrict)
4739	Results.push_back(Elt: Tmp.second);
4740	break;
4741	}
4742
4743	case ISD::FP_ROUND:
4744	case ISD::STRICT_FP_ROUND: {
4745	// X = FP_ROUND(Y, TRUNC)
4746	// TRUNC is a flag, which is always an integer that is zero or one.
4747	// If TRUNC is 0, this is a normal rounding, if it is 1, this FP_ROUND
4748	// is known to not change the value of Y.
4749	// We can only expand it into libcall if the TRUNC is 0.
4750	bool IsStrict = Node->isStrictFPOpcode();
4751	SDValue Op = Node->getOperand(Num: IsStrict ? 1 : 0);
4752	SDValue Chain = IsStrict ? Node->getOperand(Num: 0) : SDValue();
4753	EVT VT = Node->getValueType(ResNo: 0);
4754	assert(cast<ConstantSDNode>(Node->getOperand(IsStrict ? 2 : 1))->isZero() &&
4755	"Unable to expand as libcall if it is not normal rounding");
4756
4757	RTLIB::Libcall LC = RTLIB::getFPROUND(OpVT: Op.getValueType(), RetVT: VT);
4758	assert(LC != RTLIB::UNKNOWN_LIBCALL && "Unable to legalize as libcall");
4759
4760	TargetLowering::MakeLibCallOptions CallOptions;
4761	std::pair<SDValue, SDValue> Tmp =
4762	TLI.makeLibCall(DAG, LC, RetVT: VT, Ops: Op, CallOptions, dl: SDLoc(Node), Chain);
4763	Results.push_back(Elt: Tmp.first);
4764	if (IsStrict)
4765	Results.push_back(Elt: Tmp.second);
4766	break;
4767	}
4768	case ISD::FP_EXTEND: {
4769	Results.push_back(
4770	Elt: ExpandLibCall(LC: RTLIB::getFPEXT(OpVT: Node->getOperand(Num: 0).getValueType(),
4771	RetVT: Node->getValueType(ResNo: 0)),
4772	Node, isSigned: false).first);
4773	break;
4774	}
4775	case ISD::STRICT_FP_EXTEND:
4776	case ISD::STRICT_FP_TO_FP16:
4777	case ISD::STRICT_FP_TO_BF16: {
4778	RTLIB::Libcall LC = RTLIB::UNKNOWN_LIBCALL;
4779	if (Node->getOpcode() == ISD::STRICT_FP_TO_FP16)
4780	LC = RTLIB::getFPROUND(Node->getOperand(1).getValueType(), MVT::f16);
4781	else if (Node->getOpcode() == ISD::STRICT_FP_TO_BF16)
4782	LC = RTLIB::getFPROUND(Node->getOperand(1).getValueType(), MVT::bf16);
4783	else
4784	LC = RTLIB::getFPEXT(OpVT: Node->getOperand(Num: 1).getValueType(),
4785	RetVT: Node->getValueType(ResNo: 0));
4786
4787	assert(LC != RTLIB::UNKNOWN_LIBCALL && "Unable to legalize as libcall");
4788
4789	TargetLowering::MakeLibCallOptions CallOptions;
4790	std::pair<SDValue, SDValue> Tmp =
4791	TLI.makeLibCall(DAG, LC, RetVT: Node->getValueType(ResNo: 0), Ops: Node->getOperand(Num: 1),
4792	CallOptions, dl: SDLoc(Node), Chain: Node->getOperand(Num: 0));
4793	Results.push_back(Elt: Tmp.first);
4794	Results.push_back(Elt: Tmp.second);
4795	break;
4796	}
4797	case ISD::FSUB:
4798	case ISD::STRICT_FSUB:
4799	ExpandFPLibCall(Node, Call_F32: RTLIB::SUB_F32, Call_F64: RTLIB::SUB_F64,
4800	Call_F80: RTLIB::SUB_F80, Call_F128: RTLIB::SUB_F128,
4801	Call_PPCF128: RTLIB::SUB_PPCF128, Results);
4802	break;
4803	case ISD::SREM:
4804	Results.push_back(Elt: ExpandIntLibCall(Node, isSigned: true,
4805	Call_I8: RTLIB::SREM_I8,
4806	Call_I16: RTLIB::SREM_I16, Call_I32: RTLIB::SREM_I32,
4807	Call_I64: RTLIB::SREM_I64, Call_I128: RTLIB::SREM_I128));
4808	break;
4809	case ISD::UREM:
4810	Results.push_back(Elt: ExpandIntLibCall(Node, isSigned: false,
4811	Call_I8: RTLIB::UREM_I8,
4812	Call_I16: RTLIB::UREM_I16, Call_I32: RTLIB::UREM_I32,
4813	Call_I64: RTLIB::UREM_I64, Call_I128: RTLIB::UREM_I128));
4814	break;
4815	case ISD::SDIV:
4816	Results.push_back(Elt: ExpandIntLibCall(Node, isSigned: true,
4817	Call_I8: RTLIB::SDIV_I8,
4818	Call_I16: RTLIB::SDIV_I16, Call_I32: RTLIB::SDIV_I32,
4819	Call_I64: RTLIB::SDIV_I64, Call_I128: RTLIB::SDIV_I128));
4820	break;
4821	case ISD::UDIV:
4822	Results.push_back(Elt: ExpandIntLibCall(Node, isSigned: false,
4823	Call_I8: RTLIB::UDIV_I8,
4824	Call_I16: RTLIB::UDIV_I16, Call_I32: RTLIB::UDIV_I32,
4825	Call_I64: RTLIB::UDIV_I64, Call_I128: RTLIB::UDIV_I128));
4826	break;
4827	case ISD::SDIVREM:
4828	case ISD::UDIVREM:
4829	// Expand into divrem libcall
4830	ExpandDivRemLibCall(Node, Results);
4831	break;
4832	case ISD::MUL:
4833	Results.push_back(Elt: ExpandIntLibCall(Node, isSigned: false,
4834	Call_I8: RTLIB::MUL_I8,
4835	Call_I16: RTLIB::MUL_I16, Call_I32: RTLIB::MUL_I32,
4836	Call_I64: RTLIB::MUL_I64, Call_I128: RTLIB::MUL_I128));
4837	break;
4838	case ISD::CTLZ_ZERO_UNDEF:
4839	switch (Node->getSimpleValueType(ResNo: 0).SimpleTy) {
4840	default:
4841	llvm_unreachable("LibCall explicitly requested, but not available");
4842	case MVT::i32:
4843	Results.push_back(Elt: ExpandLibCall(LC: RTLIB::CTLZ_I32, Node, isSigned: false).first);
4844	break;
4845	case MVT::i64:
4846	Results.push_back(Elt: ExpandLibCall(LC: RTLIB::CTLZ_I64, Node, isSigned: false).first);
4847	break;
4848	case MVT::i128:
4849	Results.push_back(Elt: ExpandLibCall(LC: RTLIB::CTLZ_I128, Node, isSigned: false).first);
4850	break;
4851	}
4852	break;
4853	case ISD::RESET_FPENV: {
4854	// It is legalized to call 'fesetenv(FE_DFL_ENV)'. On most targets
4855	// FE_DFL_ENV is defined as '((const fenv_t *) -1)' in glibc.
4856	SDValue Ptr = DAG.getIntPtrConstant(Val: -1LL, DL: dl);
4857	SDValue Chain = Node->getOperand(Num: 0);
4858	Results.push_back(
4859	Elt: DAG.makeStateFunctionCall(LibFunc: RTLIB::FESETENV, Ptr, InChain: Chain, DLoc: dl));
4860	break;
4861	}
4862	case ISD::GET_FPENV_MEM: {
4863	SDValue Chain = Node->getOperand(Num: 0);
4864	SDValue EnvPtr = Node->getOperand(Num: 1);
4865	Results.push_back(
4866	Elt: DAG.makeStateFunctionCall(LibFunc: RTLIB::FEGETENV, Ptr: EnvPtr, InChain: Chain, DLoc: dl));
4867	break;
4868	}
4869	case ISD::SET_FPENV_MEM: {
4870	SDValue Chain = Node->getOperand(Num: 0);
4871	SDValue EnvPtr = Node->getOperand(Num: 1);
4872	Results.push_back(
4873	Elt: DAG.makeStateFunctionCall(LibFunc: RTLIB::FESETENV, Ptr: EnvPtr, InChain: Chain, DLoc: dl));
4874	break;
4875	}
4876	case ISD::GET_FPMODE: {
4877	// Call fegetmode, which saves control modes into a stack slot. Then load
4878	// the value to return from the stack.
4879	EVT ModeVT = Node->getValueType(ResNo: 0);
4880	SDValue StackPtr = DAG.CreateStackTemporary(VT: ModeVT);
4881	int SPFI = cast<FrameIndexSDNode>(Val: StackPtr.getNode())->getIndex();
4882	SDValue Chain = DAG.makeStateFunctionCall(LibFunc: RTLIB::FEGETMODE, Ptr: StackPtr,
4883	InChain: Node->getOperand(Num: 0), DLoc: dl);
4884	SDValue LdInst = DAG.getLoad(
4885	VT: ModeVT, dl, Chain, Ptr: StackPtr,
4886	PtrInfo: MachinePointerInfo::getFixedStack(MF&: DAG.getMachineFunction(), FI: SPFI));
4887	Results.push_back(Elt: LdInst);
4888	Results.push_back(Elt: LdInst.getValue(R: 1));
4889	break;
4890	}
4891	case ISD::SET_FPMODE: {
4892	// Move control modes to stack slot and then call fesetmode with the pointer
4893	// to the slot as argument.
4894	SDValue Mode = Node->getOperand(Num: 1);
4895	EVT ModeVT = Mode.getValueType();
4896	SDValue StackPtr = DAG.CreateStackTemporary(VT: ModeVT);
4897	int SPFI = cast<FrameIndexSDNode>(Val: StackPtr.getNode())->getIndex();
4898	SDValue StInst = DAG.getStore(
4899	Chain: Node->getOperand(Num: 0), dl, Val: Mode, Ptr: StackPtr,
4900	PtrInfo: MachinePointerInfo::getFixedStack(MF&: DAG.getMachineFunction(), FI: SPFI));
4901	Results.push_back(
4902	Elt: DAG.makeStateFunctionCall(LibFunc: RTLIB::FESETMODE, Ptr: StackPtr, InChain: StInst, DLoc: dl));
4903	break;
4904	}
4905	case ISD::RESET_FPMODE: {
4906	// It is legalized to a call 'fesetmode(FE_DFL_MODE)'. On most targets
4907	// FE_DFL_MODE is defined as '((const femode_t *) -1)' in glibc. If not, the
4908	// target must provide custom lowering.
4909	const DataLayout &DL = DAG.getDataLayout();
4910	EVT PtrTy = TLI.getPointerTy(DL);
4911	SDValue Mode = DAG.getConstant(Val: -1LL, DL: dl, VT: PtrTy);
4912	Results.push_back(Elt: DAG.makeStateFunctionCall(LibFunc: RTLIB::FESETMODE, Ptr: Mode,
4913	InChain: Node->getOperand(Num: 0), DLoc: dl));
4914	break;
4915	}
4916	}
4917
4918	// Replace the original node with the legalized result.
4919	if (!Results.empty()) {
4920	LLVM_DEBUG(dbgs() << "Successfully converted node to libcall\n");
4921	ReplaceNode(Old: Node, New: Results.data());
4922	} else
4923	LLVM_DEBUG(dbgs() << "Could not convert node to libcall\n");
4924	}
4925
4926	// Determine the vector type to use in place of an original scalar element when
4927	// promoting equally sized vectors.
4928	static MVT getPromotedVectorElementType(const TargetLowering &TLI,
4929	MVT EltVT, MVT NewEltVT) {
4930	unsigned OldEltsPerNewElt = EltVT.getSizeInBits() / NewEltVT.getSizeInBits();
4931	MVT MidVT = OldEltsPerNewElt == 1
4932	? NewEltVT
4933	: MVT::getVectorVT(VT: NewEltVT, NumElements: OldEltsPerNewElt);
4934	assert(TLI.isTypeLegal(MidVT) && "unexpected");
4935	return MidVT;
4936	}
4937
4938	void SelectionDAGLegalize::PromoteNode(SDNode *Node) {
4939	LLVM_DEBUG(dbgs() << "Trying to promote node\n");
4940	SmallVector<SDValue, 8> Results;
4941	MVT OVT = Node->getSimpleValueType(ResNo: 0);
4942	if (Node->getOpcode() == ISD::UINT_TO_FP ||
4943	Node->getOpcode() == ISD::SINT_TO_FP ||
4944	Node->getOpcode() == ISD::SETCC ||
4945	Node->getOpcode() == ISD::EXTRACT_VECTOR_ELT ||
4946	Node->getOpcode() == ISD::INSERT_VECTOR_ELT) {
4947	OVT = Node->getOperand(Num: 0).getSimpleValueType();
4948	}
4949	if (Node->getOpcode() == ISD::STRICT_UINT_TO_FP ||
4950	Node->getOpcode() == ISD::STRICT_SINT_TO_FP ||
4951	Node->getOpcode() == ISD::STRICT_FSETCC ||
4952	Node->getOpcode() == ISD::STRICT_FSETCCS)
4953	OVT = Node->getOperand(Num: 1).getSimpleValueType();
4954	if (Node->getOpcode() == ISD::BR_CC ||
4955	Node->getOpcode() == ISD::SELECT_CC)
4956	OVT = Node->getOperand(Num: 2).getSimpleValueType();
4957	MVT NVT = TLI.getTypeToPromoteTo(Op: Node->getOpcode(), VT: OVT);
4958	SDLoc dl(Node);
4959	SDValue Tmp1, Tmp2, Tmp3, Tmp4;
4960	switch (Node->getOpcode()) {
4961	case ISD::CTTZ:
4962	case ISD::CTTZ_ZERO_UNDEF:
4963	case ISD::CTLZ:
4964	case ISD::CTLZ_ZERO_UNDEF:
4965	case ISD::CTPOP:
4966	// Zero extend the argument unless its cttz, then use any_extend.
4967	if (Node->getOpcode() == ISD::CTTZ ||
4968	Node->getOpcode() == ISD::CTTZ_ZERO_UNDEF)
4969	Tmp1 = DAG.getNode(Opcode: ISD::ANY_EXTEND, DL: dl, VT: NVT, Operand: Node->getOperand(Num: 0));
4970	else
4971	Tmp1 = DAG.getNode(Opcode: ISD::ZERO_EXTEND, DL: dl, VT: NVT, Operand: Node->getOperand(Num: 0));
4972
4973	if (Node->getOpcode() == ISD::CTTZ) {
4974	// The count is the same in the promoted type except if the original
4975	// value was zero. This can be handled by setting the bit just off
4976	// the top of the original type.
4977	auto TopBit = APInt::getOneBitSet(numBits: NVT.getSizeInBits(),
4978	BitNo: OVT.getSizeInBits());
4979	Tmp1 = DAG.getNode(Opcode: ISD::OR, DL: dl, VT: NVT, N1: Tmp1,
4980	N2: DAG.getConstant(Val: TopBit, DL: dl, VT: NVT));
4981	}
4982	// Perform the larger operation. For CTPOP and CTTZ_ZERO_UNDEF, this is
4983	// already the correct result.
4984	Tmp1 = DAG.getNode(Opcode: Node->getOpcode(), DL: dl, VT: NVT, Operand: Tmp1);
4985	if (Node->getOpcode() == ISD::CTLZ ||
4986	Node->getOpcode() == ISD::CTLZ_ZERO_UNDEF) {
4987	// Tmp1 = Tmp1 - (sizeinbits(NVT) - sizeinbits(Old VT))
4988	Tmp1 = DAG.getNode(Opcode: ISD::SUB, DL: dl, VT: NVT, N1: Tmp1,
4989	N2: DAG.getConstant(Val: NVT.getSizeInBits() -
4990	OVT.getSizeInBits(), DL: dl, VT: NVT));
4991	}
4992	Results.push_back(Elt: DAG.getNode(Opcode: ISD::TRUNCATE, DL: dl, VT: OVT, Operand: Tmp1));
4993	break;
4994	case ISD::BITREVERSE:
4995	case ISD::BSWAP: {
4996	unsigned DiffBits = NVT.getSizeInBits() - OVT.getSizeInBits();
4997	Tmp1 = DAG.getNode(Opcode: ISD::ZERO_EXTEND, DL: dl, VT: NVT, Operand: Node->getOperand(Num: 0));
4998	Tmp1 = DAG.getNode(Opcode: Node->getOpcode(), DL: dl, VT: NVT, Operand: Tmp1);
4999	Tmp1 = DAG.getNode(
5000	Opcode: ISD::SRL, DL: dl, VT: NVT, N1: Tmp1,
5001	N2: DAG.getConstant(Val: DiffBits, DL: dl,
5002	VT: TLI.getShiftAmountTy(LHSTy: NVT, DL: DAG.getDataLayout())));
5003
5004	Results.push_back(Elt: DAG.getNode(Opcode: ISD::TRUNCATE, DL: dl, VT: OVT, Operand: Tmp1));
5005	break;
5006	}
5007	case ISD::FP_TO_UINT:
5008	case ISD::STRICT_FP_TO_UINT:
5009	case ISD::FP_TO_SINT:
5010	case ISD::STRICT_FP_TO_SINT:
5011	PromoteLegalFP_TO_INT(N: Node, dl, Results);
5012	break;
5013	case ISD::FP_TO_UINT_SAT:
5014	case ISD::FP_TO_SINT_SAT:
5015	Results.push_back(Elt: PromoteLegalFP_TO_INT_SAT(Node, dl));
5016	break;
5017	case ISD::UINT_TO_FP:
5018	case ISD::STRICT_UINT_TO_FP:
5019	case ISD::SINT_TO_FP:
5020	case ISD::STRICT_SINT_TO_FP:
5021	PromoteLegalINT_TO_FP(N: Node, dl, Results);
5022	break;
5023	case ISD::VAARG: {
5024	SDValue Chain = Node->getOperand(Num: 0); // Get the chain.
5025	SDValue Ptr = Node->getOperand(Num: 1); // Get the pointer.
5026
5027	unsigned TruncOp;
5028	if (OVT.isVector()) {
5029	TruncOp = ISD::BITCAST;
5030	} else {
5031	assert(OVT.isInteger()
5032	&& "VAARG promotion is supported only for vectors or integer types");
5033	TruncOp = ISD::TRUNCATE;
5034	}
5035
5036	// Perform the larger operation, then convert back
5037	Tmp1 = DAG.getVAArg(VT: NVT, dl, Chain, Ptr, SV: Node->getOperand(Num: 2),
5038	Align: Node->getConstantOperandVal(Num: 3));
5039	Chain = Tmp1.getValue(R: 1);
5040
5041	Tmp2 = DAG.getNode(Opcode: TruncOp, DL: dl, VT: OVT, Operand: Tmp1);
5042
5043	// Modified the chain result - switch anything that used the old chain to
5044	// use the new one.
5045	DAG.ReplaceAllUsesOfValueWith(From: SDValue(Node, 0), To: Tmp2);
5046	DAG.ReplaceAllUsesOfValueWith(From: SDValue(Node, 1), To: Chain);
5047	if (UpdatedNodes) {
5048	UpdatedNodes->insert(X: Tmp2.getNode());
5049	UpdatedNodes->insert(X: Chain.getNode());
5050	}
5051	ReplacedNode(N: Node);
5052	break;
5053	}
5054	case ISD::MUL:
5055	case ISD::SDIV:
5056	case ISD::SREM:
5057	case ISD::UDIV:
5058	case ISD::UREM:
5059	case ISD::SMIN:
5060	case ISD::SMAX:
5061	case ISD::UMIN:
5062	case ISD::UMAX:
5063	case ISD::AND:
5064	case ISD::OR:
5065	case ISD::XOR: {
5066	unsigned ExtOp, TruncOp;
5067	if (OVT.isVector()) {
5068	ExtOp = ISD::BITCAST;
5069	TruncOp = ISD::BITCAST;
5070	} else {
5071	assert(OVT.isInteger() && "Cannot promote logic operation");
5072
5073	switch (Node->getOpcode()) {
5074	default:
5075	ExtOp = ISD::ANY_EXTEND;
5076	break;
5077	case ISD::SDIV:
5078	case ISD::SREM:
5079	case ISD::SMIN:
5080	case ISD::SMAX:
5081	ExtOp = ISD::SIGN_EXTEND;
5082	break;
5083	case ISD::UDIV:
5084	case ISD::UREM:
5085	ExtOp = ISD::ZERO_EXTEND;
5086	break;
5087	case ISD::UMIN:
5088	case ISD::UMAX:
5089	if (TLI.isSExtCheaperThanZExt(FromTy: OVT, ToTy: NVT))
5090	ExtOp = ISD::SIGN_EXTEND;
5091	else
5092	ExtOp = ISD::ZERO_EXTEND;
5093	break;
5094	}
5095	TruncOp = ISD::TRUNCATE;
5096	}
5097	// Promote each of the values to the new type.
5098	Tmp1 = DAG.getNode(Opcode: ExtOp, DL: dl, VT: NVT, Operand: Node->getOperand(Num: 0));
5099	Tmp2 = DAG.getNode(Opcode: ExtOp, DL: dl, VT: NVT, Operand: Node->getOperand(Num: 1));
5100	// Perform the larger operation, then convert back
5101	Tmp1 = DAG.getNode(Opcode: Node->getOpcode(), DL: dl, VT: NVT, N1: Tmp1, N2: Tmp2);
5102	Results.push_back(Elt: DAG.getNode(Opcode: TruncOp, DL: dl, VT: OVT, Operand: Tmp1));
5103	break;
5104	}
5105	case ISD::UMUL_LOHI:
5106	case ISD::SMUL_LOHI: {
5107	// Promote to a multiply in a wider integer type.
5108	unsigned ExtOp = Node->getOpcode() == ISD::UMUL_LOHI ? ISD::ZERO_EXTEND
5109	: ISD::SIGN_EXTEND;
5110	Tmp1 = DAG.getNode(Opcode: ExtOp, DL: dl, VT: NVT, Operand: Node->getOperand(Num: 0));
5111	Tmp2 = DAG.getNode(Opcode: ExtOp, DL: dl, VT: NVT, Operand: Node->getOperand(Num: 1));
5112	Tmp1 = DAG.getNode(Opcode: ISD::MUL, DL: dl, VT: NVT, N1: Tmp1, N2: Tmp2);
5113
5114	auto &DL = DAG.getDataLayout();
5115	unsigned OriginalSize = OVT.getScalarSizeInBits();
5116	Tmp2 = DAG.getNode(
5117	Opcode: ISD::SRL, DL: dl, VT: NVT, N1: Tmp1,
5118	N2: DAG.getConstant(Val: OriginalSize, DL: dl, VT: TLI.getScalarShiftAmountTy(DL, NVT)));
5119	Results.push_back(Elt: DAG.getNode(Opcode: ISD::TRUNCATE, DL: dl, VT: OVT, Operand: Tmp1));
5120	Results.push_back(Elt: DAG.getNode(Opcode: ISD::TRUNCATE, DL: dl, VT: OVT, Operand: Tmp2));
5121	break;
5122	}
5123	case ISD::SELECT: {
5124	unsigned ExtOp, TruncOp;
5125	if (Node->getValueType(ResNo: 0).isVector() ||
5126	Node->getValueType(ResNo: 0).getSizeInBits() == NVT.getSizeInBits()) {
5127	ExtOp = ISD::BITCAST;
5128	TruncOp = ISD::BITCAST;
5129	} else if (Node->getValueType(ResNo: 0).isInteger()) {
5130	ExtOp = ISD::ANY_EXTEND;
5131	TruncOp = ISD::TRUNCATE;
5132	} else {
5133	ExtOp = ISD::FP_EXTEND;
5134	TruncOp = ISD::FP_ROUND;
5135	}
5136	Tmp1 = Node->getOperand(Num: 0);
5137	// Promote each of the values to the new type.
5138	Tmp2 = DAG.getNode(Opcode: ExtOp, DL: dl, VT: NVT, Operand: Node->getOperand(Num: 1));
5139	Tmp3 = DAG.getNode(Opcode: ExtOp, DL: dl, VT: NVT, Operand: Node->getOperand(Num: 2));
5140	// Perform the larger operation, then round down.
5141	Tmp1 = DAG.getSelect(DL: dl, VT: NVT, Cond: Tmp1, LHS: Tmp2, RHS: Tmp3);
5142	Tmp1->setFlags(Node->getFlags());
5143	if (TruncOp != ISD::FP_ROUND)
5144	Tmp1 = DAG.getNode(Opcode: TruncOp, DL: dl, VT: Node->getValueType(ResNo: 0), Operand: Tmp1);
5145	else
5146	Tmp1 = DAG.getNode(Opcode: TruncOp, DL: dl, VT: Node->getValueType(ResNo: 0), N1: Tmp1,
5147	N2: DAG.getIntPtrConstant(Val: 0, DL: dl));
5148	Results.push_back(Elt: Tmp1);
5149	break;
5150	}
5151	case ISD::VECTOR_SHUFFLE: {
5152	ArrayRef<int> Mask = cast<ShuffleVectorSDNode>(Val: Node)->getMask();
5153
5154	// Cast the two input vectors.
5155	Tmp1 = DAG.getNode(Opcode: ISD::BITCAST, DL: dl, VT: NVT, Operand: Node->getOperand(Num: 0));
5156	Tmp2 = DAG.getNode(Opcode: ISD::BITCAST, DL: dl, VT: NVT, Operand: Node->getOperand(Num: 1));
5157
5158	// Convert the shuffle mask to the right # elements.
5159	Tmp1 = ShuffleWithNarrowerEltType(NVT, VT: OVT, dl, N1: Tmp1, N2: Tmp2, Mask);
5160	Tmp1 = DAG.getNode(Opcode: ISD::BITCAST, DL: dl, VT: OVT, Operand: Tmp1);
5161	Results.push_back(Elt: Tmp1);
5162	break;
5163	}
5164	case ISD::VECTOR_SPLICE: {
5165	Tmp1 = DAG.getNode(Opcode: ISD::ANY_EXTEND, DL: dl, VT: NVT, Operand: Node->getOperand(Num: 0));
5166	Tmp2 = DAG.getNode(Opcode: ISD::ANY_EXTEND, DL: dl, VT: NVT, Operand: Node->getOperand(Num: 1));
5167	Tmp3 = DAG.getNode(Opcode: ISD::VECTOR_SPLICE, DL: dl, VT: NVT, N1: Tmp1, N2: Tmp2,
5168	N3: Node->getOperand(Num: 2));
5169	Results.push_back(Elt: DAG.getNode(Opcode: ISD::TRUNCATE, DL: dl, VT: OVT, Operand: Tmp3));
5170	break;
5171	}
5172	case ISD::SELECT_CC: {
5173	SDValue Cond = Node->getOperand(Num: 4);
5174	ISD::CondCode CCCode = cast<CondCodeSDNode>(Val&: Cond)->get();
5175	// Type of the comparison operands.
5176	MVT CVT = Node->getSimpleValueType(ResNo: 0);
5177	assert(CVT == OVT && "not handled");
5178
5179	unsigned ExtOp = ISD::FP_EXTEND;
5180	if (NVT.isInteger()) {
5181	ExtOp = isSignedIntSetCC(Code: CCCode) ? ISD::SIGN_EXTEND : ISD::ZERO_EXTEND;
5182	}
5183
5184	// Promote the comparison operands, if needed.
5185	if (TLI.isCondCodeLegal(CC: CCCode, VT: CVT)) {
5186	Tmp1 = Node->getOperand(Num: 0);
5187	Tmp2 = Node->getOperand(Num: 1);
5188	} else {
5189	Tmp1 = DAG.getNode(Opcode: ExtOp, DL: dl, VT: NVT, Operand: Node->getOperand(Num: 0));
5190	Tmp2 = DAG.getNode(Opcode: ExtOp, DL: dl, VT: NVT, Operand: Node->getOperand(Num: 1));
5191	}
5192	// Cast the true/false operands.
5193	Tmp3 = DAG.getNode(Opcode: ExtOp, DL: dl, VT: NVT, Operand: Node->getOperand(Num: 2));
5194	Tmp4 = DAG.getNode(Opcode: ExtOp, DL: dl, VT: NVT, Operand: Node->getOperand(Num: 3));
5195
5196	Tmp1 = DAG.getNode(Opcode: ISD::SELECT_CC, DL: dl, VT: NVT, Ops: {Tmp1, Tmp2, Tmp3, Tmp4, Cond},
5197	Flags: Node->getFlags());
5198
5199	// Cast the result back to the original type.
5200	if (ExtOp != ISD::FP_EXTEND)
5201	Tmp1 = DAG.getNode(Opcode: ISD::TRUNCATE, DL: dl, VT: OVT, Operand: Tmp1);
5202	else
5203	Tmp1 = DAG.getNode(Opcode: ISD::FP_ROUND, DL: dl, VT: OVT, N1: Tmp1,
5204	N2: DAG.getIntPtrConstant(Val: 0, DL: dl, /*isTarget=*/true));
5205
5206	Results.push_back(Elt: Tmp1);
5207	break;
5208	}
5209	case ISD::SETCC:
5210	case ISD::STRICT_FSETCC:
5211	case ISD::STRICT_FSETCCS: {
5212	unsigned ExtOp = ISD::FP_EXTEND;
5213	if (NVT.isInteger()) {
5214	ISD::CondCode CCCode = cast<CondCodeSDNode>(Val: Node->getOperand(Num: 2))->get();
5215	if (isSignedIntSetCC(Code: CCCode) ||
5216	TLI.isSExtCheaperThanZExt(FromTy: Node->getOperand(Num: 0).getValueType(), ToTy: NVT))
5217	ExtOp = ISD::SIGN_EXTEND;
5218	else
5219	ExtOp = ISD::ZERO_EXTEND;
5220	}
5221	if (Node->isStrictFPOpcode()) {
5222	SDValue InChain = Node->getOperand(Num: 0);
5223	std::tie(args&: Tmp1, args: std::ignore) =
5224	DAG.getStrictFPExtendOrRound(Op: Node->getOperand(Num: 1), Chain: InChain, DL: dl, VT: NVT);
5225	std::tie(args&: Tmp2, args: std::ignore) =
5226	DAG.getStrictFPExtendOrRound(Op: Node->getOperand(Num: 2), Chain: InChain, DL: dl, VT: NVT);
5227	SmallVector<SDValue, 2> TmpChains = {Tmp1.getValue(R: 1), Tmp2.getValue(R: 1)};
5228	SDValue OutChain = DAG.getTokenFactor(DL: dl, Vals&: TmpChains);
5229	SDVTList VTs = DAG.getVTList(Node->getValueType(0), MVT::Other);
5230	Results.push_back(Elt: DAG.getNode(Opcode: Node->getOpcode(), DL: dl, VTList: VTs,
5231	Ops: {OutChain, Tmp1, Tmp2, Node->getOperand(Num: 3)},
5232	Flags: Node->getFlags()));
5233	Results.push_back(Elt: Results.back().getValue(R: 1));
5234	break;
5235	}
5236	Tmp1 = DAG.getNode(Opcode: ExtOp, DL: dl, VT: NVT, Operand: Node->getOperand(Num: 0));
5237	Tmp2 = DAG.getNode(Opcode: ExtOp, DL: dl, VT: NVT, Operand: Node->getOperand(Num: 1));
5238	Results.push_back(Elt: DAG.getNode(Opcode: ISD::SETCC, DL: dl, VT: Node->getValueType(ResNo: 0), N1: Tmp1,
5239	N2: Tmp2, N3: Node->getOperand(Num: 2), Flags: Node->getFlags()));
5240	break;
5241	}
5242	case ISD::BR_CC: {
5243	unsigned ExtOp = ISD::FP_EXTEND;
5244	if (NVT.isInteger()) {
5245	ISD::CondCode CCCode =
5246	cast<CondCodeSDNode>(Val: Node->getOperand(Num: 1))->get();
5247	ExtOp = isSignedIntSetCC(Code: CCCode) ? ISD::SIGN_EXTEND : ISD::ZERO_EXTEND;
5248	}
5249	Tmp1 = DAG.getNode(Opcode: ExtOp, DL: dl, VT: NVT, Operand: Node->getOperand(Num: 2));
5250	Tmp2 = DAG.getNode(Opcode: ExtOp, DL: dl, VT: NVT, Operand: Node->getOperand(Num: 3));
5251	Results.push_back(Elt: DAG.getNode(Opcode: ISD::BR_CC, DL: dl, VT: Node->getValueType(ResNo: 0),
5252	N1: Node->getOperand(Num: 0), N2: Node->getOperand(Num: 1),
5253	N3: Tmp1, N4: Tmp2, N5: Node->getOperand(Num: 4)));
5254	break;
5255	}
5256	case ISD::FADD:
5257	case ISD::FSUB:
5258	case ISD::FMUL:
5259	case ISD::FDIV:
5260	case ISD::FREM:
5261	case ISD::FMINNUM:
5262	case ISD::FMAXNUM:
5263	case ISD::FMINIMUM:
5264	case ISD::FMAXIMUM:
5265	case ISD::FPOW:
5266	Tmp1 = DAG.getNode(Opcode: ISD::FP_EXTEND, DL: dl, VT: NVT, Operand: Node->getOperand(Num: 0));
5267	Tmp2 = DAG.getNode(Opcode: ISD::FP_EXTEND, DL: dl, VT: NVT, Operand: Node->getOperand(Num: 1));
5268	Tmp3 = DAG.getNode(Opcode: Node->getOpcode(), DL: dl, VT: NVT, N1: Tmp1, N2: Tmp2,
5269	Flags: Node->getFlags());
5270	Results.push_back(
5271	Elt: DAG.getNode(Opcode: ISD::FP_ROUND, DL: dl, VT: OVT, N1: Tmp3,
5272	N2: DAG.getIntPtrConstant(Val: 0, DL: dl, /*isTarget=*/true)));
5273	break;
5274	case ISD::STRICT_FADD:
5275	case ISD::STRICT_FSUB:
5276	case ISD::STRICT_FMUL:
5277	case ISD::STRICT_FDIV:
5278	case ISD::STRICT_FMINNUM:
5279	case ISD::STRICT_FMAXNUM:
5280	case ISD::STRICT_FREM:
5281	case ISD::STRICT_FPOW:
5282	Tmp1 = DAG.getNode(ISD::STRICT_FP_EXTEND, dl, {NVT, MVT::Other},
5283	{Node->getOperand(0), Node->getOperand(1)});
5284	Tmp2 = DAG.getNode(ISD::STRICT_FP_EXTEND, dl, {NVT, MVT::Other},
5285	{Node->getOperand(0), Node->getOperand(2)});
5286	Tmp3 = DAG.getNode(ISD::TokenFactor, dl, MVT::Other, Tmp1.getValue(1),
5287	Tmp2.getValue(1));
5288	Tmp1 = DAG.getNode(Node->getOpcode(), dl, {NVT, MVT::Other},
5289	{Tmp3, Tmp1, Tmp2});
5290	Tmp1 = DAG.getNode(ISD::STRICT_FP_ROUND, dl, {OVT, MVT::Other},
5291	{Tmp1.getValue(1), Tmp1, DAG.getIntPtrConstant(0, dl)});
5292	Results.push_back(Elt: Tmp1);
5293	Results.push_back(Elt: Tmp1.getValue(R: 1));
5294	break;
5295	case ISD::FMA:
5296	Tmp1 = DAG.getNode(Opcode: ISD::FP_EXTEND, DL: dl, VT: NVT, Operand: Node->getOperand(Num: 0));
5297	Tmp2 = DAG.getNode(Opcode: ISD::FP_EXTEND, DL: dl, VT: NVT, Operand: Node->getOperand(Num: 1));
5298	Tmp3 = DAG.getNode(Opcode: ISD::FP_EXTEND, DL: dl, VT: NVT, Operand: Node->getOperand(Num: 2));
5299	Results.push_back(
5300	Elt: DAG.getNode(Opcode: ISD::FP_ROUND, DL: dl, VT: OVT,
5301	N1: DAG.getNode(Opcode: Node->getOpcode(), DL: dl, VT: NVT, N1: Tmp1, N2: Tmp2, N3: Tmp3),
5302	N2: DAG.getIntPtrConstant(Val: 0, DL: dl, /*isTarget=*/true)));
5303	break;
5304	case ISD::STRICT_FMA:
5305	Tmp1 = DAG.getNode(ISD::STRICT_FP_EXTEND, dl, {NVT, MVT::Other},
5306	{Node->getOperand(0), Node->getOperand(1)});
5307	Tmp2 = DAG.getNode(ISD::STRICT_FP_EXTEND, dl, {NVT, MVT::Other},
5308	{Node->getOperand(0), Node->getOperand(2)});
5309	Tmp3 = DAG.getNode(ISD::STRICT_FP_EXTEND, dl, {NVT, MVT::Other},
5310	{Node->getOperand(0), Node->getOperand(3)});
5311	Tmp4 = DAG.getNode(ISD::TokenFactor, dl, MVT::Other, Tmp1.getValue(1),
5312	Tmp2.getValue(1), Tmp3.getValue(1));
5313	Tmp4 = DAG.getNode(Node->getOpcode(), dl, {NVT, MVT::Other},
5314	{Tmp4, Tmp1, Tmp2, Tmp3});
5315	Tmp4 = DAG.getNode(ISD::STRICT_FP_ROUND, dl, {OVT, MVT::Other},
5316	{Tmp4.getValue(1), Tmp4, DAG.getIntPtrConstant(0, dl)});
5317	Results.push_back(Elt: Tmp4);
5318	Results.push_back(Elt: Tmp4.getValue(R: 1));
5319	break;
5320	case ISD::FCOPYSIGN:
5321	case ISD::FLDEXP:
5322	case ISD::FPOWI: {
5323	Tmp1 = DAG.getNode(Opcode: ISD::FP_EXTEND, DL: dl, VT: NVT, Operand: Node->getOperand(Num: 0));
5324	Tmp2 = Node->getOperand(Num: 1);
5325	Tmp3 = DAG.getNode(Opcode: Node->getOpcode(), DL: dl, VT: NVT, N1: Tmp1, N2: Tmp2);
5326
5327	// fcopysign doesn't change anything but the sign bit, so
5328	// (fp_round (fcopysign (fpext a), b))
5329	// is as precise as
5330	// (fp_round (fpext a))
5331	// which is a no-op. Mark it as a TRUNCating FP_ROUND.
5332	const bool isTrunc = (Node->getOpcode() == ISD::FCOPYSIGN);
5333	Results.push_back(
5334	Elt: DAG.getNode(Opcode: ISD::FP_ROUND, DL: dl, VT: OVT, N1: Tmp3,
5335	N2: DAG.getIntPtrConstant(Val: isTrunc, DL: dl, /*isTarget=*/true)));
5336	break;
5337	}
5338	case ISD::STRICT_FPOWI:
5339	Tmp1 = DAG.getNode(ISD::STRICT_FP_EXTEND, dl, {NVT, MVT::Other},
5340	{Node->getOperand(0), Node->getOperand(1)});
5341	Tmp2 = DAG.getNode(Node->getOpcode(), dl, {NVT, MVT::Other},
5342	{Tmp1.getValue(1), Tmp1, Node->getOperand(2)});
5343	Tmp3 = DAG.getNode(ISD::STRICT_FP_ROUND, dl, {OVT, MVT::Other},
5344	{Tmp2.getValue(1), Tmp2, DAG.getIntPtrConstant(0, dl)});
5345	Results.push_back(Elt: Tmp3);
5346	Results.push_back(Elt: Tmp3.getValue(R: 1));
5347	break;
5348	case ISD::FFREXP: {
5349	Tmp1 = DAG.getNode(Opcode: ISD::FP_EXTEND, DL: dl, VT: NVT, Operand: Node->getOperand(Num: 0));
5350	Tmp2 = DAG.getNode(Opcode: ISD::FFREXP, DL: dl, ResultTys: {NVT, Node->getValueType(ResNo: 1)}, Ops: Tmp1);
5351
5352	Results.push_back(
5353	Elt: DAG.getNode(Opcode: ISD::FP_ROUND, DL: dl, VT: OVT, N1: Tmp2,
5354	N2: DAG.getIntPtrConstant(Val: 0, DL: dl, /*isTarget=*/true)));
5355
5356	Results.push_back(Elt: Tmp2.getValue(R: 1));
5357	break;
5358	}
5359	case ISD::FFLOOR:
5360	case ISD::FCEIL:
5361	case ISD::FRINT:
5362	case ISD::FNEARBYINT:
5363	case ISD::FROUND:
5364	case ISD::FROUNDEVEN:
5365	case ISD::FTRUNC:
5366	case ISD::FNEG:
5367	case ISD::FSQRT:
5368	case ISD::FSIN:
5369	case ISD::FCOS:
5370	case ISD::FLOG:
5371	case ISD::FLOG2:
5372	case ISD::FLOG10:
5373	case ISD::FABS:
5374	case ISD::FEXP:
5375	case ISD::FEXP2:
5376	case ISD::FEXP10:
5377	case ISD::FCANONICALIZE:
5378	Tmp1 = DAG.getNode(Opcode: ISD::FP_EXTEND, DL: dl, VT: NVT, Operand: Node->getOperand(Num: 0));
5379	Tmp2 = DAG.getNode(Opcode: Node->getOpcode(), DL: dl, VT: NVT, Operand: Tmp1);
5380	Results.push_back(
5381	Elt: DAG.getNode(Opcode: ISD::FP_ROUND, DL: dl, VT: OVT, N1: Tmp2,
5382	N2: DAG.getIntPtrConstant(Val: 0, DL: dl, /*isTarget=*/true)));
5383	break;
5384	case ISD::STRICT_FFLOOR:
5385	case ISD::STRICT_FCEIL:
5386	case ISD::STRICT_FRINT:
5387	case ISD::STRICT_FNEARBYINT:
5388	case ISD::STRICT_FROUND:
5389	case ISD::STRICT_FROUNDEVEN:
5390	case ISD::STRICT_FTRUNC:
5391	case ISD::STRICT_FSQRT:
5392	case ISD::STRICT_FSIN:
5393	case ISD::STRICT_FCOS:
5394	case ISD::STRICT_FLOG:
5395	case ISD::STRICT_FLOG2:
5396	case ISD::STRICT_FLOG10:
5397	case ISD::STRICT_FEXP:
5398	case ISD::STRICT_FEXP2:
5399	Tmp1 = DAG.getNode(ISD::STRICT_FP_EXTEND, dl, {NVT, MVT::Other},
5400	{Node->getOperand(0), Node->getOperand(1)});
5401	Tmp2 = DAG.getNode(Node->getOpcode(), dl, {NVT, MVT::Other},
5402	{Tmp1.getValue(1), Tmp1});
5403	Tmp3 = DAG.getNode(ISD::STRICT_FP_ROUND, dl, {OVT, MVT::Other},
5404	{Tmp2.getValue(1), Tmp2, DAG.getIntPtrConstant(0, dl)});
5405	Results.push_back(Elt: Tmp3);
5406	Results.push_back(Elt: Tmp3.getValue(R: 1));
5407	break;
5408	case ISD::BUILD_VECTOR: {
5409	MVT EltVT = OVT.getVectorElementType();
5410	MVT NewEltVT = NVT.getVectorElementType();
5411
5412	// Handle bitcasts to a different vector type with the same total bit size
5413	//
5414	// e.g. v2i64 = build_vector i64:x, i64:y => v4i32
5415	// =>
5416	// v4i32 = concat_vectors (v2i32 (bitcast i64:x)), (v2i32 (bitcast i64:y))
5417
5418	assert(NVT.isVector() && OVT.getSizeInBits() == NVT.getSizeInBits() &&
5419	"Invalid promote type for build_vector");
5420	assert(NewEltVT.bitsLE(EltVT) && "not handled");
5421
5422	MVT MidVT = getPromotedVectorElementType(TLI, EltVT, NewEltVT);
5423
5424	SmallVector<SDValue, 8> NewOps;
5425	for (unsigned I = 0, E = Node->getNumOperands(); I != E; ++I) {
5426	SDValue Op = Node->getOperand(Num: I);
5427	NewOps.push_back(Elt: DAG.getNode(Opcode: ISD::BITCAST, DL: SDLoc(Op), VT: MidVT, Operand: Op));
5428	}
5429
5430	SDLoc SL(Node);
5431	SDValue Concat =
5432	DAG.getNode(Opcode: MidVT == NewEltVT ? ISD::BUILD_VECTOR : ISD::CONCAT_VECTORS,
5433	DL: SL, VT: NVT, Ops: NewOps);
5434	SDValue CvtVec = DAG.getNode(Opcode: ISD::BITCAST, DL: SL, VT: OVT, Operand: Concat);
5435	Results.push_back(Elt: CvtVec);
5436	break;
5437	}
5438	case ISD::EXTRACT_VECTOR_ELT: {
5439	MVT EltVT = OVT.getVectorElementType();
5440	MVT NewEltVT = NVT.getVectorElementType();
5441
5442	// Handle bitcasts to a different vector type with the same total bit size.
5443	//
5444	// e.g. v2i64 = extract_vector_elt x:v2i64, y:i32
5445	// =>
5446	// v4i32:castx = bitcast x:v2i64
5447	//
5448	// i64 = bitcast
5449	// (v2i32 build_vector (i32 (extract_vector_elt castx, (2 * y))),
5450	// (i32 (extract_vector_elt castx, (2 * y + 1)))
5451	//
5452
5453	assert(NVT.isVector() && OVT.getSizeInBits() == NVT.getSizeInBits() &&
5454	"Invalid promote type for extract_vector_elt");
5455	assert(NewEltVT.bitsLT(EltVT) && "not handled");
5456
5457	MVT MidVT = getPromotedVectorElementType(TLI, EltVT, NewEltVT);
5458	unsigned NewEltsPerOldElt = MidVT.getVectorNumElements();
5459
5460	SDValue Idx = Node->getOperand(Num: 1);
5461	EVT IdxVT = Idx.getValueType();
5462	SDLoc SL(Node);
5463	SDValue Factor = DAG.getConstant(Val: NewEltsPerOldElt, DL: SL, VT: IdxVT);
5464	SDValue NewBaseIdx = DAG.getNode(Opcode: ISD::MUL, DL: SL, VT: IdxVT, N1: Idx, N2: Factor);
5465
5466	SDValue CastVec = DAG.getNode(Opcode: ISD::BITCAST, DL: SL, VT: NVT, Operand: Node->getOperand(Num: 0));
5467
5468	SmallVector<SDValue, 8> NewOps;
5469	for (unsigned I = 0; I < NewEltsPerOldElt; ++I) {
5470	SDValue IdxOffset = DAG.getConstant(Val: I, DL: SL, VT: IdxVT);
5471	SDValue TmpIdx = DAG.getNode(Opcode: ISD::ADD, DL: SL, VT: IdxVT, N1: NewBaseIdx, N2: IdxOffset);
5472
5473	SDValue Elt = DAG.getNode(Opcode: ISD::EXTRACT_VECTOR_ELT, DL: SL, VT: NewEltVT,
5474	N1: CastVec, N2: TmpIdx);
5475	NewOps.push_back(Elt);
5476	}
5477
5478	SDValue NewVec = DAG.getBuildVector(VT: MidVT, DL: SL, Ops: NewOps);
5479	Results.push_back(Elt: DAG.getNode(Opcode: ISD::BITCAST, DL: SL, VT: EltVT, Operand: NewVec));
5480	break;
5481	}
5482	case ISD::INSERT_VECTOR_ELT: {
5483	MVT EltVT = OVT.getVectorElementType();
5484	MVT NewEltVT = NVT.getVectorElementType();
5485
5486	// Handle bitcasts to a different vector type with the same total bit size
5487	//
5488	// e.g. v2i64 = insert_vector_elt x:v2i64, y:i64, z:i32
5489	// =>
5490	// v4i32:castx = bitcast x:v2i64
5491	// v2i32:casty = bitcast y:i64
5492	//
5493	// v2i64 = bitcast
5494	// (v4i32 insert_vector_elt
5495	// (v4i32 insert_vector_elt v4i32:castx,
5496	// (extract_vector_elt casty, 0), 2 * z),
5497	// (extract_vector_elt casty, 1), (2 * z + 1))
5498
5499	assert(NVT.isVector() && OVT.getSizeInBits() == NVT.getSizeInBits() &&
5500	"Invalid promote type for insert_vector_elt");
5501	assert(NewEltVT.bitsLT(EltVT) && "not handled");
5502
5503	MVT MidVT = getPromotedVectorElementType(TLI, EltVT, NewEltVT);
5504	unsigned NewEltsPerOldElt = MidVT.getVectorNumElements();
5505
5506	SDValue Val = Node->getOperand(Num: 1);
5507	SDValue Idx = Node->getOperand(Num: 2);
5508	EVT IdxVT = Idx.getValueType();
5509	SDLoc SL(Node);
5510
5511	SDValue Factor = DAG.getConstant(Val: NewEltsPerOldElt, DL: SDLoc(), VT: IdxVT);
5512	SDValue NewBaseIdx = DAG.getNode(Opcode: ISD::MUL, DL: SL, VT: IdxVT, N1: Idx, N2: Factor);
5513
5514	SDValue CastVec = DAG.getNode(Opcode: ISD::BITCAST, DL: SL, VT: NVT, Operand: Node->getOperand(Num: 0));
5515	SDValue CastVal = DAG.getNode(Opcode: ISD::BITCAST, DL: SL, VT: MidVT, Operand: Val);
5516
5517	SDValue NewVec = CastVec;
5518	for (unsigned I = 0; I < NewEltsPerOldElt; ++I) {
5519	SDValue IdxOffset = DAG.getConstant(Val: I, DL: SL, VT: IdxVT);
5520	SDValue InEltIdx = DAG.getNode(Opcode: ISD::ADD, DL: SL, VT: IdxVT, N1: NewBaseIdx, N2: IdxOffset);
5521
5522	SDValue Elt = DAG.getNode(Opcode: ISD::EXTRACT_VECTOR_ELT, DL: SL, VT: NewEltVT,
5523	N1: CastVal, N2: IdxOffset);
5524
5525	NewVec = DAG.getNode(Opcode: ISD::INSERT_VECTOR_ELT, DL: SL, VT: NVT,
5526	N1: NewVec, N2: Elt, N3: InEltIdx);
5527	}
5528
5529	Results.push_back(Elt: DAG.getNode(Opcode: ISD::BITCAST, DL: SL, VT: OVT, Operand: NewVec));
5530	break;
5531	}
5532	case ISD::SCALAR_TO_VECTOR: {
5533	MVT EltVT = OVT.getVectorElementType();
5534	MVT NewEltVT = NVT.getVectorElementType();
5535
5536	// Handle bitcasts to different vector type with the same total bit size.
5537	//
5538	// e.g. v2i64 = scalar_to_vector x:i64
5539	// =>
5540	// concat_vectors (v2i32 bitcast x:i64), (v2i32 undef)
5541	//
5542
5543	MVT MidVT = getPromotedVectorElementType(TLI, EltVT, NewEltVT);
5544	SDValue Val = Node->getOperand(Num: 0);
5545	SDLoc SL(Node);
5546
5547	SDValue CastVal = DAG.getNode(Opcode: ISD::BITCAST, DL: SL, VT: MidVT, Operand: Val);
5548	SDValue Undef = DAG.getUNDEF(VT: MidVT);
5549
5550	SmallVector<SDValue, 8> NewElts;
5551	NewElts.push_back(Elt: CastVal);
5552	for (unsigned I = 1, NElts = OVT.getVectorNumElements(); I != NElts; ++I)
5553	NewElts.push_back(Elt: Undef);
5554
5555	SDValue Concat = DAG.getNode(Opcode: ISD::CONCAT_VECTORS, DL: SL, VT: NVT, Ops: NewElts);
5556	SDValue CvtVec = DAG.getNode(Opcode: ISD::BITCAST, DL: SL, VT: OVT, Operand: Concat);
5557	Results.push_back(Elt: CvtVec);
5558	break;
5559	}
5560	case ISD::ATOMIC_SWAP: {
5561	AtomicSDNode *AM = cast<AtomicSDNode>(Val: Node);
5562	SDLoc SL(Node);
5563	SDValue CastVal = DAG.getNode(Opcode: ISD::BITCAST, DL: SL, VT: NVT, Operand: AM->getVal());
5564	assert(NVT.getSizeInBits() == OVT.getSizeInBits() &&
5565	"unexpected promotion type");
5566	assert(AM->getMemoryVT().getSizeInBits() == NVT.getSizeInBits() &&
5567	"unexpected atomic_swap with illegal type");
5568
5569	SDValue NewAtomic
5570	= DAG.getAtomic(ISD::ATOMIC_SWAP, SL, NVT,
5571	DAG.getVTList(NVT, MVT::Other),
5572	{ AM->getChain(), AM->getBasePtr(), CastVal },
5573	AM->getMemOperand());
5574	Results.push_back(Elt: DAG.getNode(Opcode: ISD::BITCAST, DL: SL, VT: OVT, Operand: NewAtomic));
5575	Results.push_back(Elt: NewAtomic.getValue(R: 1));
5576	break;
5577	}
5578	case ISD::SPLAT_VECTOR: {
5579	SDValue Scalar = Node->getOperand(Num: 0);
5580	MVT ScalarType = Scalar.getSimpleValueType();
5581	MVT NewScalarType = NVT.getVectorElementType();
5582	if (ScalarType.isInteger()) {
5583	Tmp1 = DAG.getNode(Opcode: ISD::ANY_EXTEND, DL: dl, VT: NewScalarType, Operand: Scalar);
5584	Tmp2 = DAG.getNode(Opcode: Node->getOpcode(), DL: dl, VT: NVT, Operand: Tmp1);
5585	Results.push_back(Elt: DAG.getNode(Opcode: ISD::TRUNCATE, DL: dl, VT: OVT, Operand: Tmp2));
5586	break;
5587	}
5588	Tmp1 = DAG.getNode(Opcode: ISD::FP_EXTEND, DL: dl, VT: NewScalarType, Operand: Scalar);
5589	Tmp2 = DAG.getNode(Opcode: Node->getOpcode(), DL: dl, VT: NVT, Operand: Tmp1);
5590	Results.push_back(
5591	Elt: DAG.getNode(Opcode: ISD::FP_ROUND, DL: dl, VT: OVT, N1: Tmp2,
5592	N2: DAG.getIntPtrConstant(Val: 0, DL: dl, /*isTarget=*/true)));
5593	break;
5594	}
5595	}
5596
5597	// Replace the original node with the legalized result.
5598	if (!Results.empty()) {
5599	LLVM_DEBUG(dbgs() << "Successfully promoted node\n");
5600	ReplaceNode(Old: Node, New: Results.data());
5601	} else
5602	LLVM_DEBUG(dbgs() << "Could not promote node\n");
5603	}
5604
5605	/// This is the entry point for the file.
5606	void SelectionDAG::Legalize() {
5607	AssignTopologicalOrder();
5608
5609	SmallPtrSet<SDNode *, 16> LegalizedNodes;
5610	// Use a delete listener to remove nodes which were deleted during
5611	// legalization from LegalizeNodes. This is needed to handle the situation
5612	// where a new node is allocated by the object pool to the same address of a
5613	// previously deleted node.
5614	DAGNodeDeletedListener DeleteListener(
5615	*this,
5616	[&LegalizedNodes](SDNode *N, SDNode *E) { LegalizedNodes.erase(Ptr: N); });
5617
5618	SelectionDAGLegalize Legalizer(*this, LegalizedNodes);
5619
5620	// Visit all the nodes. We start in topological order, so that we see
5621	// nodes with their original operands intact. Legalization can produce
5622	// new nodes which may themselves need to be legalized. Iterate until all
5623	// nodes have been legalized.
5624	while (true) {
5625	bool AnyLegalized = false;
5626	for (auto NI = allnodes_end(); NI != allnodes_begin();) {
5627	--NI;
5628
5629	SDNode *N = &*NI;
5630	if (N->use_empty() && N != getRoot().getNode()) {
5631	++NI;
5632	DeleteNode(N);
5633	continue;
5634	}
5635
5636	if (LegalizedNodes.insert(Ptr: N).second) {
5637	AnyLegalized = true;
5638	Legalizer.LegalizeOp(Node: N);
5639
5640	if (N->use_empty() && N != getRoot().getNode()) {
5641	++NI;
5642	DeleteNode(N);
5643	}
5644	}
5645	}
5646	if (!AnyLegalized)
5647	break;
5648
5649	}
5650
5651	// Remove dead nodes now.
5652	RemoveDeadNodes();
5653	}
5654
5655	bool SelectionDAG::LegalizeOp(SDNode *N,
5656	SmallSetVector<SDNode *, 16> &UpdatedNodes) {
5657	SmallPtrSet<SDNode *, 16> LegalizedNodes;
5658	SelectionDAGLegalize Legalizer(*this, LegalizedNodes, &UpdatedNodes);
5659
5660	// Directly insert the node in question, and legalize it. This will recurse
5661	// as needed through operands.
5662	LegalizedNodes.insert(Ptr: N);
5663	Legalizer.LegalizeOp(Node: N);
5664
5665	return LegalizedNodes.count(Ptr: N);
5666	}
5667