1//===-------- LegalizeFloatTypes.cpp - Legalization of float types --------===//
2//
3// Part of the LLVM Project, under the Apache License v2.0 with LLVM Exceptions.
4// See https://llvm.org/LICENSE.txt for license information.
5// SPDX-License-Identifier: Apache-2.0 WITH LLVM-exception
6//
7//===----------------------------------------------------------------------===//
8//
9// This file implements float type expansion and softening for LegalizeTypes.
10// Softening is the act of turning a computation in an illegal floating point
11// type into a computation in an integer type of the same size; also known as
12// "soft float". For example, turning f32 arithmetic into operations using i32.
13// The resulting integer value is the same as what you would get by performing
14// the floating point operation and bitcasting the result to the integer type.
15// Expansion is the act of changing a computation in an illegal type to be a
16// computation in two identical registers of a smaller type. For example,
17// implementing ppcf128 arithmetic in two f64 registers.
18//
19//===----------------------------------------------------------------------===//
20
21#include "LegalizeTypes.h"
22#include "llvm/Analysis/TargetLibraryInfo.h"
23#include "llvm/Support/ErrorHandling.h"
24#include "llvm/Support/raw_ostream.h"
25using namespace llvm;
26
27#define DEBUG_TYPE "legalize-types"
28
29/// GetFPLibCall - Return the right libcall for the given floating point type.
30/// FIXME: This is a local version of RTLIB::getFPLibCall that should be
31/// refactored away (see RTLIB::getPOWI for an example).
32static RTLIB::Libcall GetFPLibCall(EVT VT,
33 RTLIB::Libcall Call_F32,
34 RTLIB::Libcall Call_F64,
35 RTLIB::Libcall Call_F80,
36 RTLIB::Libcall Call_F128,
37 RTLIB::Libcall Call_PPCF128) {
38 return
39 VT == MVT::f32 ? Call_F32 :
40 VT == MVT::f64 ? Call_F64 :
41 VT == MVT::f80 ? Call_F80 :
42 VT == MVT::f128 ? Call_F128 :
43 VT == MVT::ppcf128 ? Call_PPCF128 :
44 RTLIB::UNKNOWN_LIBCALL;
45}
46
47//===----------------------------------------------------------------------===//
48// Convert Float Results to Integer
49//===----------------------------------------------------------------------===//
50
51void DAGTypeLegalizer::SoftenFloatResult(SDNode *N, unsigned ResNo) {
52 LLVM_DEBUG(dbgs() << "Soften float result " << ResNo << ": "; N->dump(&DAG));
53 SDValue R = SDValue();
54
55 switch (N->getOpcode()) {
56 // clang-format off
57 default:
58#ifndef NDEBUG
59 dbgs() << "SoftenFloatResult #" << ResNo << ": ";
60 N->dump(G: &DAG); dbgs() << "\n";
61#endif
62 report_fatal_error(reason: "Do not know how to soften the result of this "
63 "operator!");
64 case ISD::EXTRACT_ELEMENT: R = SoftenFloatRes_EXTRACT_ELEMENT(N); break;
65 case ISD::ARITH_FENCE: R = SoftenFloatRes_ARITH_FENCE(N); break;
66 case ISD::MERGE_VALUES:R = SoftenFloatRes_MERGE_VALUES(N, ResNo); break;
67 case ISD::BITCAST: R = SoftenFloatRes_BITCAST(N); break;
68 case ISD::BUILD_PAIR: R = SoftenFloatRes_BUILD_PAIR(N); break;
69 case ISD::ConstantFP: R = SoftenFloatRes_ConstantFP(N); break;
70 case ISD::EXTRACT_VECTOR_ELT:
71 R = SoftenFloatRes_EXTRACT_VECTOR_ELT(N, ResNo); break;
72 case ISD::FABS: R = SoftenFloatRes_FABS(N); break;
73 case ISD::STRICT_FMINNUM:
74 case ISD::FMINNUM: R = SoftenFloatRes_FMINNUM(N); break;
75 case ISD::STRICT_FMAXNUM:
76 case ISD::FMAXNUM: R = SoftenFloatRes_FMAXNUM(N); break;
77 case ISD::STRICT_FADD:
78 case ISD::FADD: R = SoftenFloatRes_FADD(N); break;
79 case ISD::FCBRT: R = SoftenFloatRes_FCBRT(N); break;
80 case ISD::STRICT_FCEIL:
81 case ISD::FCEIL: R = SoftenFloatRes_FCEIL(N); break;
82 case ISD::FCOPYSIGN: R = SoftenFloatRes_FCOPYSIGN(N); break;
83 case ISD::STRICT_FCOS:
84 case ISD::FCOS: R = SoftenFloatRes_FCOS(N); break;
85 case ISD::STRICT_FDIV:
86 case ISD::FDIV: R = SoftenFloatRes_FDIV(N); break;
87 case ISD::STRICT_FEXP:
88 case ISD::FEXP: R = SoftenFloatRes_FEXP(N); break;
89 case ISD::STRICT_FEXP2:
90 case ISD::FEXP2: R = SoftenFloatRes_FEXP2(N); break;
91 case ISD::FEXP10: R = SoftenFloatRes_FEXP10(N); break;
92 case ISD::STRICT_FFLOOR:
93 case ISD::FFLOOR: R = SoftenFloatRes_FFLOOR(N); break;
94 case ISD::STRICT_FLOG:
95 case ISD::FLOG: R = SoftenFloatRes_FLOG(N); break;
96 case ISD::STRICT_FLOG2:
97 case ISD::FLOG2: R = SoftenFloatRes_FLOG2(N); break;
98 case ISD::STRICT_FLOG10:
99 case ISD::FLOG10: R = SoftenFloatRes_FLOG10(N); break;
100 case ISD::STRICT_FMA:
101 case ISD::FMA: R = SoftenFloatRes_FMA(N); break;
102 case ISD::STRICT_FMUL:
103 case ISD::FMUL: R = SoftenFloatRes_FMUL(N); break;
104 case ISD::STRICT_FNEARBYINT:
105 case ISD::FNEARBYINT: R = SoftenFloatRes_FNEARBYINT(N); break;
106 case ISD::FNEG: R = SoftenFloatRes_FNEG(N); break;
107 case ISD::STRICT_FP_EXTEND:
108 case ISD::FP_EXTEND: R = SoftenFloatRes_FP_EXTEND(N); break;
109 case ISD::STRICT_FP_ROUND:
110 case ISD::FP_ROUND: R = SoftenFloatRes_FP_ROUND(N); break;
111 case ISD::FP16_TO_FP: R = SoftenFloatRes_FP16_TO_FP(N); break;
112 case ISD::BF16_TO_FP: R = SoftenFloatRes_BF16_TO_FP(N); break;
113 case ISD::STRICT_FPOW:
114 case ISD::FPOW: R = SoftenFloatRes_FPOW(N); break;
115 case ISD::STRICT_FPOWI:
116 case ISD::FPOWI:
117 case ISD::FLDEXP:
118 case ISD::STRICT_FLDEXP: R = SoftenFloatRes_ExpOp(N); break;
119 case ISD::FFREXP: R = SoftenFloatRes_FFREXP(N); break;
120 case ISD::STRICT_FREM:
121 case ISD::FREM: R = SoftenFloatRes_FREM(N); break;
122 case ISD::STRICT_FRINT:
123 case ISD::FRINT: R = SoftenFloatRes_FRINT(N); break;
124 case ISD::STRICT_FROUND:
125 case ISD::FROUND: R = SoftenFloatRes_FROUND(N); break;
126 case ISD::STRICT_FROUNDEVEN:
127 case ISD::FROUNDEVEN: R = SoftenFloatRes_FROUNDEVEN(N); break;
128 case ISD::STRICT_FSIN:
129 case ISD::FSIN: R = SoftenFloatRes_FSIN(N); break;
130 case ISD::STRICT_FSQRT:
131 case ISD::FSQRT: R = SoftenFloatRes_FSQRT(N); break;
132 case ISD::STRICT_FSUB:
133 case ISD::FSUB: R = SoftenFloatRes_FSUB(N); break;
134 case ISD::STRICT_FTRUNC:
135 case ISD::FTRUNC: R = SoftenFloatRes_FTRUNC(N); break;
136 case ISD::LOAD: R = SoftenFloatRes_LOAD(N); break;
137 case ISD::ATOMIC_SWAP: R = BitcastToInt_ATOMIC_SWAP(N); break;
138 case ISD::SELECT: R = SoftenFloatRes_SELECT(N); break;
139 case ISD::SELECT_CC: R = SoftenFloatRes_SELECT_CC(N); break;
140 case ISD::FREEZE: R = SoftenFloatRes_FREEZE(N); break;
141 case ISD::STRICT_SINT_TO_FP:
142 case ISD::STRICT_UINT_TO_FP:
143 case ISD::SINT_TO_FP:
144 case ISD::UINT_TO_FP: R = SoftenFloatRes_XINT_TO_FP(N); break;
145 case ISD::UNDEF: R = SoftenFloatRes_UNDEF(N); break;
146 case ISD::VAARG: R = SoftenFloatRes_VAARG(N); break;
147 case ISD::VECREDUCE_FADD:
148 case ISD::VECREDUCE_FMUL:
149 case ISD::VECREDUCE_FMIN:
150 case ISD::VECREDUCE_FMAX:
151 case ISD::VECREDUCE_FMAXIMUM:
152 case ISD::VECREDUCE_FMINIMUM: R = SoftenFloatRes_VECREDUCE(N); break;
153 case ISD::VECREDUCE_SEQ_FADD:
154 case ISD::VECREDUCE_SEQ_FMUL: R = SoftenFloatRes_VECREDUCE_SEQ(N); break;
155 // clang-format on
156 }
157
158 // If R is null, the sub-method took care of registering the result.
159 if (R.getNode()) {
160 assert(R.getNode() != N);
161 SetSoftenedFloat(Op: SDValue(N, ResNo), Result: R);
162 }
163}
164
165SDValue DAGTypeLegalizer::SoftenFloatRes_Unary(SDNode *N, RTLIB::Libcall LC) {
166 bool IsStrict = N->isStrictFPOpcode();
167 EVT NVT = TLI.getTypeToTransformTo(Context&: *DAG.getContext(), VT: N->getValueType(ResNo: 0));
168 unsigned Offset = IsStrict ? 1 : 0;
169 assert(N->getNumOperands() == (1 + Offset) &&
170 "Unexpected number of operands!");
171 SDValue Op = GetSoftenedFloat(Op: N->getOperand(Num: 0 + Offset));
172 SDValue Chain = IsStrict ? N->getOperand(Num: 0) : SDValue();
173 TargetLowering::MakeLibCallOptions CallOptions;
174 EVT OpVT = N->getOperand(Num: 0 + Offset).getValueType();
175 CallOptions.setTypeListBeforeSoften(OpsVT: OpVT, RetVT: N->getValueType(ResNo: 0), Value: true);
176 std::pair<SDValue, SDValue> Tmp = TLI.makeLibCall(DAG, LC, RetVT: NVT, Ops: Op,
177 CallOptions, dl: SDLoc(N),
178 Chain);
179 if (IsStrict)
180 ReplaceValueWith(From: SDValue(N, 1), To: Tmp.second);
181 return Tmp.first;
182}
183
184SDValue DAGTypeLegalizer::SoftenFloatRes_Binary(SDNode *N, RTLIB::Libcall LC) {
185 bool IsStrict = N->isStrictFPOpcode();
186 EVT NVT = TLI.getTypeToTransformTo(Context&: *DAG.getContext(), VT: N->getValueType(ResNo: 0));
187 unsigned Offset = IsStrict ? 1 : 0;
188 assert(N->getNumOperands() == (2 + Offset) &&
189 "Unexpected number of operands!");
190 SDValue Ops[2] = { GetSoftenedFloat(Op: N->getOperand(Num: 0 + Offset)),
191 GetSoftenedFloat(Op: N->getOperand(Num: 1 + Offset)) };
192 SDValue Chain = IsStrict ? N->getOperand(Num: 0) : SDValue();
193 TargetLowering::MakeLibCallOptions CallOptions;
194 EVT OpsVT[2] = { N->getOperand(Num: 0 + Offset).getValueType(),
195 N->getOperand(Num: 1 + Offset).getValueType() };
196 CallOptions.setTypeListBeforeSoften(OpsVT, RetVT: N->getValueType(ResNo: 0), Value: true);
197 std::pair<SDValue, SDValue> Tmp = TLI.makeLibCall(DAG, LC, RetVT: NVT, Ops,
198 CallOptions, dl: SDLoc(N),
199 Chain);
200 if (IsStrict)
201 ReplaceValueWith(From: SDValue(N, 1), To: Tmp.second);
202 return Tmp.first;
203}
204
205SDValue DAGTypeLegalizer::SoftenFloatRes_BITCAST(SDNode *N) {
206 return BitConvertToInteger(Op: N->getOperand(Num: 0));
207}
208
209SDValue DAGTypeLegalizer::SoftenFloatRes_FREEZE(SDNode *N) {
210 EVT Ty = TLI.getTypeToTransformTo(Context&: *DAG.getContext(), VT: N->getValueType(ResNo: 0));
211 return DAG.getNode(Opcode: ISD::FREEZE, DL: SDLoc(N), VT: Ty,
212 Operand: GetSoftenedFloat(Op: N->getOperand(Num: 0)));
213}
214
215SDValue DAGTypeLegalizer::SoftenFloatRes_ARITH_FENCE(SDNode *N) {
216 EVT Ty = TLI.getTypeToTransformTo(Context&: *DAG.getContext(), VT: N->getValueType(ResNo: 0));
217 SDValue NewFence = DAG.getNode(Opcode: ISD::ARITH_FENCE, DL: SDLoc(N), VT: Ty,
218 Operand: GetSoftenedFloat(Op: N->getOperand(Num: 0)));
219 return NewFence;
220}
221
222SDValue DAGTypeLegalizer::SoftenFloatRes_MERGE_VALUES(SDNode *N,
223 unsigned ResNo) {
224 SDValue Op = DisintegrateMERGE_VALUES(N, ResNo);
225 return BitConvertToInteger(Op);
226}
227
228SDValue DAGTypeLegalizer::SoftenFloatRes_BUILD_PAIR(SDNode *N) {
229 // Convert the inputs to integers, and build a new pair out of them.
230 return DAG.getNode(Opcode: ISD::BUILD_PAIR, DL: SDLoc(N),
231 VT: TLI.getTypeToTransformTo(Context&: *DAG.getContext(),
232 VT: N->getValueType(ResNo: 0)),
233 N1: BitConvertToInteger(Op: N->getOperand(Num: 0)),
234 N2: BitConvertToInteger(Op: N->getOperand(Num: 1)));
235}
236
237SDValue DAGTypeLegalizer::SoftenFloatRes_ConstantFP(SDNode *N) {
238 ConstantFPSDNode *CN = cast<ConstantFPSDNode>(Val: N);
239 // In ppcf128, the high 64 bits are always first in memory regardless
240 // of Endianness. LLVM's APFloat representation is not Endian sensitive,
241 // and so always converts into a 128-bit APInt in a non-Endian-sensitive
242 // way. However, APInt's are serialized in an Endian-sensitive fashion,
243 // so on big-Endian targets, the two doubles are output in the wrong
244 // order. Fix this by manually flipping the order of the high 64 bits
245 // and the low 64 bits here.
246 if (DAG.getDataLayout().isBigEndian() &&
247 CN->getValueType(ResNo: 0).getSimpleVT() == llvm::MVT::ppcf128) {
248 uint64_t words[2] = { CN->getValueAPF().bitcastToAPInt().getRawData()[1],
249 CN->getValueAPF().bitcastToAPInt().getRawData()[0] };
250 APInt Val(128, words);
251 return DAG.getConstant(Val, DL: SDLoc(CN),
252 VT: TLI.getTypeToTransformTo(Context&: *DAG.getContext(),
253 VT: CN->getValueType(ResNo: 0)));
254 } else {
255 return DAG.getConstant(Val: CN->getValueAPF().bitcastToAPInt(), DL: SDLoc(CN),
256 VT: TLI.getTypeToTransformTo(Context&: *DAG.getContext(),
257 VT: CN->getValueType(ResNo: 0)));
258 }
259}
260
261SDValue DAGTypeLegalizer::SoftenFloatRes_EXTRACT_ELEMENT(SDNode *N) {
262 SDValue Src = N->getOperand(Num: 0);
263 assert(Src.getValueType() == MVT::ppcf128 &&
264 "In floats only ppcf128 can be extracted by element!");
265 return DAG.getNode(ISD::EXTRACT_ELEMENT, SDLoc(N),
266 N->getValueType(ResNo: 0).changeTypeToInteger(),
267 DAG.getBitcast(MVT::VT: i128, V: Src), N->getOperand(Num: 1));
268}
269
270SDValue DAGTypeLegalizer::SoftenFloatRes_EXTRACT_VECTOR_ELT(SDNode *N, unsigned ResNo) {
271 SDValue NewOp = BitConvertVectorToIntegerVector(Op: N->getOperand(Num: 0));
272 return DAG.getNode(Opcode: ISD::EXTRACT_VECTOR_ELT, DL: SDLoc(N),
273 VT: NewOp.getValueType().getVectorElementType(),
274 N1: NewOp, N2: N->getOperand(Num: 1));
275}
276
277SDValue DAGTypeLegalizer::SoftenFloatRes_FABS(SDNode *N) {
278 EVT NVT = TLI.getTypeToTransformTo(Context&: *DAG.getContext(), VT: N->getValueType(ResNo: 0));
279 unsigned Size = NVT.getSizeInBits();
280
281 // Mask = ~(1 << (Size-1))
282 APInt API = APInt::getAllOnes(numBits: Size);
283 API.clearBit(BitPosition: Size - 1);
284 SDValue Mask = DAG.getConstant(Val: API, DL: SDLoc(N), VT: NVT);
285 SDValue Op = GetSoftenedFloat(Op: N->getOperand(Num: 0));
286 return DAG.getNode(Opcode: ISD::AND, DL: SDLoc(N), VT: NVT, N1: Op, N2: Mask);
287}
288
289SDValue DAGTypeLegalizer::SoftenFloatRes_FMINNUM(SDNode *N) {
290 if (SDValue SelCC = TLI.createSelectForFMINNUM_FMAXNUM(Node: N, DAG))
291 return SoftenFloatRes_SELECT_CC(N: SelCC.getNode());
292 return SoftenFloatRes_Binary(N, LC: GetFPLibCall(VT: N->getValueType(ResNo: 0),
293 Call_F32: RTLIB::FMIN_F32,
294 Call_F64: RTLIB::FMIN_F64,
295 Call_F80: RTLIB::FMIN_F80,
296 Call_F128: RTLIB::FMIN_F128,
297 Call_PPCF128: RTLIB::FMIN_PPCF128));
298}
299
300SDValue DAGTypeLegalizer::SoftenFloatRes_FMAXNUM(SDNode *N) {
301 if (SDValue SelCC = TLI.createSelectForFMINNUM_FMAXNUM(Node: N, DAG))
302 return SoftenFloatRes_SELECT_CC(N: SelCC.getNode());
303 return SoftenFloatRes_Binary(N, LC: GetFPLibCall(VT: N->getValueType(ResNo: 0),
304 Call_F32: RTLIB::FMAX_F32,
305 Call_F64: RTLIB::FMAX_F64,
306 Call_F80: RTLIB::FMAX_F80,
307 Call_F128: RTLIB::FMAX_F128,
308 Call_PPCF128: RTLIB::FMAX_PPCF128));
309}
310
311SDValue DAGTypeLegalizer::SoftenFloatRes_FADD(SDNode *N) {
312 return SoftenFloatRes_Binary(N, LC: GetFPLibCall(VT: N->getValueType(ResNo: 0),
313 Call_F32: RTLIB::ADD_F32,
314 Call_F64: RTLIB::ADD_F64,
315 Call_F80: RTLIB::ADD_F80,
316 Call_F128: RTLIB::ADD_F128,
317 Call_PPCF128: RTLIB::ADD_PPCF128));
318}
319
320SDValue DAGTypeLegalizer::SoftenFloatRes_FCBRT(SDNode *N) {
321 return SoftenFloatRes_Unary(N, LC: GetFPLibCall(VT: N->getValueType(ResNo: 0),
322 Call_F32: RTLIB::CBRT_F32,
323 Call_F64: RTLIB::CBRT_F64,
324 Call_F80: RTLIB::CBRT_F80,
325 Call_F128: RTLIB::CBRT_F128,
326 Call_PPCF128: RTLIB::CBRT_PPCF128));
327}
328
329SDValue DAGTypeLegalizer::SoftenFloatRes_FCEIL(SDNode *N) {
330 return SoftenFloatRes_Unary(N, LC: GetFPLibCall(VT: N->getValueType(ResNo: 0),
331 Call_F32: RTLIB::CEIL_F32,
332 Call_F64: RTLIB::CEIL_F64,
333 Call_F80: RTLIB::CEIL_F80,
334 Call_F128: RTLIB::CEIL_F128,
335 Call_PPCF128: RTLIB::CEIL_PPCF128));
336}
337
338SDValue DAGTypeLegalizer::SoftenFloatRes_FCOPYSIGN(SDNode *N) {
339 SDValue LHS = GetSoftenedFloat(Op: N->getOperand(Num: 0));
340 SDValue RHS = BitConvertToInteger(Op: N->getOperand(Num: 1));
341 SDLoc dl(N);
342
343 EVT LVT = LHS.getValueType();
344 EVT RVT = RHS.getValueType();
345
346 unsigned LSize = LVT.getSizeInBits();
347 unsigned RSize = RVT.getSizeInBits();
348
349 // First get the sign bit of second operand.
350 SDValue SignBit = DAG.getNode(
351 Opcode: ISD::SHL, DL: dl, VT: RVT, N1: DAG.getConstant(Val: 1, DL: dl, VT: RVT),
352 N2: DAG.getConstant(Val: RSize - 1, DL: dl,
353 VT: TLI.getShiftAmountTy(LHSTy: RVT, DL: DAG.getDataLayout())));
354 SignBit = DAG.getNode(Opcode: ISD::AND, DL: dl, VT: RVT, N1: RHS, N2: SignBit);
355
356 // Shift right or sign-extend it if the two operands have different types.
357 int SizeDiff = RVT.getSizeInBits() - LVT.getSizeInBits();
358 if (SizeDiff > 0) {
359 SignBit =
360 DAG.getNode(Opcode: ISD::SRL, DL: dl, VT: RVT, N1: SignBit,
361 N2: DAG.getConstant(Val: SizeDiff, DL: dl,
362 VT: TLI.getShiftAmountTy(LHSTy: SignBit.getValueType(),
363 DL: DAG.getDataLayout())));
364 SignBit = DAG.getNode(Opcode: ISD::TRUNCATE, DL: dl, VT: LVT, Operand: SignBit);
365 } else if (SizeDiff < 0) {
366 SignBit = DAG.getNode(Opcode: ISD::ANY_EXTEND, DL: dl, VT: LVT, Operand: SignBit);
367 SignBit =
368 DAG.getNode(Opcode: ISD::SHL, DL: dl, VT: LVT, N1: SignBit,
369 N2: DAG.getConstant(Val: -SizeDiff, DL: dl,
370 VT: TLI.getShiftAmountTy(LHSTy: SignBit.getValueType(),
371 DL: DAG.getDataLayout())));
372 }
373
374 // Clear the sign bit of the first operand.
375 SDValue Mask = DAG.getNode(
376 Opcode: ISD::SHL, DL: dl, VT: LVT, N1: DAG.getConstant(Val: 1, DL: dl, VT: LVT),
377 N2: DAG.getConstant(Val: LSize - 1, DL: dl,
378 VT: TLI.getShiftAmountTy(LHSTy: LVT, DL: DAG.getDataLayout())));
379 Mask = DAG.getNode(Opcode: ISD::SUB, DL: dl, VT: LVT, N1: Mask, N2: DAG.getConstant(Val: 1, DL: dl, VT: LVT));
380 LHS = DAG.getNode(Opcode: ISD::AND, DL: dl, VT: LVT, N1: LHS, N2: Mask);
381
382 // Or the value with the sign bit.
383 return DAG.getNode(Opcode: ISD::OR, DL: dl, VT: LVT, N1: LHS, N2: SignBit);
384}
385
386SDValue DAGTypeLegalizer::SoftenFloatRes_FCOS(SDNode *N) {
387 return SoftenFloatRes_Unary(N, LC: GetFPLibCall(VT: N->getValueType(ResNo: 0),
388 Call_F32: RTLIB::COS_F32,
389 Call_F64: RTLIB::COS_F64,
390 Call_F80: RTLIB::COS_F80,
391 Call_F128: RTLIB::COS_F128,
392 Call_PPCF128: RTLIB::COS_PPCF128));
393}
394
395SDValue DAGTypeLegalizer::SoftenFloatRes_FDIV(SDNode *N) {
396 return SoftenFloatRes_Binary(N, LC: GetFPLibCall(VT: N->getValueType(ResNo: 0),
397 Call_F32: RTLIB::DIV_F32,
398 Call_F64: RTLIB::DIV_F64,
399 Call_F80: RTLIB::DIV_F80,
400 Call_F128: RTLIB::DIV_F128,
401 Call_PPCF128: RTLIB::DIV_PPCF128));
402}
403
404SDValue DAGTypeLegalizer::SoftenFloatRes_FEXP(SDNode *N) {
405 return SoftenFloatRes_Unary(N, LC: GetFPLibCall(VT: N->getValueType(ResNo: 0),
406 Call_F32: RTLIB::EXP_F32,
407 Call_F64: RTLIB::EXP_F64,
408 Call_F80: RTLIB::EXP_F80,
409 Call_F128: RTLIB::EXP_F128,
410 Call_PPCF128: RTLIB::EXP_PPCF128));
411}
412
413SDValue DAGTypeLegalizer::SoftenFloatRes_FEXP2(SDNode *N) {
414 return SoftenFloatRes_Unary(N, LC: GetFPLibCall(VT: N->getValueType(ResNo: 0),
415 Call_F32: RTLIB::EXP2_F32,
416 Call_F64: RTLIB::EXP2_F64,
417 Call_F80: RTLIB::EXP2_F80,
418 Call_F128: RTLIB::EXP2_F128,
419 Call_PPCF128: RTLIB::EXP2_PPCF128));
420}
421
422SDValue DAGTypeLegalizer::SoftenFloatRes_FEXP10(SDNode *N) {
423 return SoftenFloatRes_Unary(
424 N,
425 LC: GetFPLibCall(VT: N->getValueType(ResNo: 0), Call_F32: RTLIB::EXP10_F32, Call_F64: RTLIB::EXP10_F64,
426 Call_F80: RTLIB::EXP10_F80, Call_F128: RTLIB::EXP10_F128, Call_PPCF128: RTLIB::EXP10_PPCF128));
427}
428
429SDValue DAGTypeLegalizer::SoftenFloatRes_FFLOOR(SDNode *N) {
430 return SoftenFloatRes_Unary(N, LC: GetFPLibCall(VT: N->getValueType(ResNo: 0),
431 Call_F32: RTLIB::FLOOR_F32,
432 Call_F64: RTLIB::FLOOR_F64,
433 Call_F80: RTLIB::FLOOR_F80,
434 Call_F128: RTLIB::FLOOR_F128,
435 Call_PPCF128: RTLIB::FLOOR_PPCF128));
436}
437
438SDValue DAGTypeLegalizer::SoftenFloatRes_FLOG(SDNode *N) {
439 return SoftenFloatRes_Unary(N, LC: GetFPLibCall(VT: N->getValueType(ResNo: 0),
440 Call_F32: RTLIB::LOG_F32,
441 Call_F64: RTLIB::LOG_F64,
442 Call_F80: RTLIB::LOG_F80,
443 Call_F128: RTLIB::LOG_F128,
444 Call_PPCF128: RTLIB::LOG_PPCF128));
445}
446
447SDValue DAGTypeLegalizer::SoftenFloatRes_FLOG2(SDNode *N) {
448 return SoftenFloatRes_Unary(N, LC: GetFPLibCall(VT: N->getValueType(ResNo: 0),
449 Call_F32: RTLIB::LOG2_F32,
450 Call_F64: RTLIB::LOG2_F64,
451 Call_F80: RTLIB::LOG2_F80,
452 Call_F128: RTLIB::LOG2_F128,
453 Call_PPCF128: RTLIB::LOG2_PPCF128));
454}
455
456SDValue DAGTypeLegalizer::SoftenFloatRes_FLOG10(SDNode *N) {
457 return SoftenFloatRes_Unary(N, LC: GetFPLibCall(VT: N->getValueType(ResNo: 0),
458 Call_F32: RTLIB::LOG10_F32,
459 Call_F64: RTLIB::LOG10_F64,
460 Call_F80: RTLIB::LOG10_F80,
461 Call_F128: RTLIB::LOG10_F128,
462 Call_PPCF128: RTLIB::LOG10_PPCF128));
463}
464
465SDValue DAGTypeLegalizer::SoftenFloatRes_FMA(SDNode *N) {
466 bool IsStrict = N->isStrictFPOpcode();
467 EVT NVT = TLI.getTypeToTransformTo(Context&: *DAG.getContext(), VT: N->getValueType(ResNo: 0));
468 unsigned Offset = IsStrict ? 1 : 0;
469 SDValue Ops[3] = { GetSoftenedFloat(Op: N->getOperand(Num: 0 + Offset)),
470 GetSoftenedFloat(Op: N->getOperand(Num: 1 + Offset)),
471 GetSoftenedFloat(Op: N->getOperand(Num: 2 + Offset)) };
472 SDValue Chain = IsStrict ? N->getOperand(Num: 0) : SDValue();
473 TargetLowering::MakeLibCallOptions CallOptions;
474 EVT OpsVT[3] = { N->getOperand(Num: 0 + Offset).getValueType(),
475 N->getOperand(Num: 1 + Offset).getValueType(),
476 N->getOperand(Num: 2 + Offset).getValueType() };
477 CallOptions.setTypeListBeforeSoften(OpsVT, RetVT: N->getValueType(ResNo: 0), Value: true);
478 std::pair<SDValue, SDValue> Tmp = TLI.makeLibCall(DAG,
479 LC: GetFPLibCall(VT: N->getValueType(ResNo: 0),
480 Call_F32: RTLIB::FMA_F32,
481 Call_F64: RTLIB::FMA_F64,
482 Call_F80: RTLIB::FMA_F80,
483 Call_F128: RTLIB::FMA_F128,
484 Call_PPCF128: RTLIB::FMA_PPCF128),
485 RetVT: NVT, Ops, CallOptions, dl: SDLoc(N), Chain);
486 if (IsStrict)
487 ReplaceValueWith(From: SDValue(N, 1), To: Tmp.second);
488 return Tmp.first;
489}
490
491SDValue DAGTypeLegalizer::SoftenFloatRes_FMUL(SDNode *N) {
492 return SoftenFloatRes_Binary(N, LC: GetFPLibCall(VT: N->getValueType(ResNo: 0),
493 Call_F32: RTLIB::MUL_F32,
494 Call_F64: RTLIB::MUL_F64,
495 Call_F80: RTLIB::MUL_F80,
496 Call_F128: RTLIB::MUL_F128,
497 Call_PPCF128: RTLIB::MUL_PPCF128));
498}
499
500SDValue DAGTypeLegalizer::SoftenFloatRes_FNEARBYINT(SDNode *N) {
501 return SoftenFloatRes_Unary(N, LC: GetFPLibCall(VT: N->getValueType(ResNo: 0),
502 Call_F32: RTLIB::NEARBYINT_F32,
503 Call_F64: RTLIB::NEARBYINT_F64,
504 Call_F80: RTLIB::NEARBYINT_F80,
505 Call_F128: RTLIB::NEARBYINT_F128,
506 Call_PPCF128: RTLIB::NEARBYINT_PPCF128));
507}
508
509SDValue DAGTypeLegalizer::SoftenFloatRes_FNEG(SDNode *N) {
510 EVT NVT = TLI.getTypeToTransformTo(Context&: *DAG.getContext(), VT: N->getValueType(ResNo: 0));
511 SDLoc dl(N);
512
513 // Expand Y = FNEG(X) -> Y = X ^ sign mask
514 APInt SignMask = APInt::getSignMask(BitWidth: NVT.getSizeInBits());
515 return DAG.getNode(Opcode: ISD::XOR, DL: dl, VT: NVT, N1: GetSoftenedFloat(Op: N->getOperand(Num: 0)),
516 N2: DAG.getConstant(Val: SignMask, DL: dl, VT: NVT));
517}
518
519SDValue DAGTypeLegalizer::SoftenFloatRes_FP_EXTEND(SDNode *N) {
520 bool IsStrict = N->isStrictFPOpcode();
521 EVT NVT = TLI.getTypeToTransformTo(Context&: *DAG.getContext(), VT: N->getValueType(ResNo: 0));
522 SDValue Op = N->getOperand(Num: IsStrict ? 1 : 0);
523
524 SDValue Chain = IsStrict ? N->getOperand(Num: 0) : SDValue();
525
526 if (getTypeAction(VT: Op.getValueType()) == TargetLowering::TypePromoteFloat) {
527 Op = GetPromotedFloat(Op);
528 // If the promotion did the FP_EXTEND to the destination type for us,
529 // there's nothing left to do here.
530 if (Op.getValueType() == N->getValueType(ResNo: 0)) {
531 if (IsStrict)
532 ReplaceValueWith(From: SDValue(N, 1), To: Chain);
533 return BitConvertToInteger(Op);
534 }
535 }
536
537 // There's only a libcall for f16 -> f32 and shifting is only valid for bf16
538 // -> f32, so proceed in two stages. Also, it's entirely possible for both
539 // f16 and f32 to be legal, so use the fully hard-float FP_EXTEND rather
540 // than FP16_TO_FP.
541 if ((Op.getValueType() == MVT::f16 || Op.getValueType() == MVT::bf16) &&
542 N->getValueType(ResNo: 0) != MVT::f32) {
543 if (IsStrict) {
544 Op = DAG.getNode(ISD::STRICT_FP_EXTEND, SDLoc(N),
545 { MVT::f32, MVT::Other }, { Chain, Op });
546 Chain = Op.getValue(R: 1);
547 } else {
548 Op = DAG.getNode(ISD::FP_EXTEND, SDLoc(N), MVT::f32, Op);
549 }
550 }
551
552 if (Op.getValueType() == MVT::bf16) {
553 // FIXME: Need ReplaceValueWith on chain in strict case
554 return SoftenFloatRes_BF16_TO_FP(N);
555 }
556
557 RTLIB::Libcall LC = RTLIB::getFPEXT(OpVT: Op.getValueType(), RetVT: N->getValueType(ResNo: 0));
558 assert(LC != RTLIB::UNKNOWN_LIBCALL && "Unsupported FP_EXTEND!");
559 TargetLowering::MakeLibCallOptions CallOptions;
560 EVT OpVT = N->getOperand(Num: IsStrict ? 1 : 0).getValueType();
561 CallOptions.setTypeListBeforeSoften(OpsVT: OpVT, RetVT: N->getValueType(ResNo: 0), Value: true);
562 std::pair<SDValue, SDValue> Tmp = TLI.makeLibCall(DAG, LC, RetVT: NVT, Ops: Op,
563 CallOptions, dl: SDLoc(N),
564 Chain);
565 if (IsStrict)
566 ReplaceValueWith(From: SDValue(N, 1), To: Tmp.second);
567 return Tmp.first;
568}
569
570// FIXME: Should we just use 'normal' FP_EXTEND / FP_TRUNC instead of special
571// nodes?
572SDValue DAGTypeLegalizer::SoftenFloatRes_FP16_TO_FP(SDNode *N) {
573 EVT MidVT = TLI.getTypeToTransformTo(Context&: *DAG.getContext(), MVT::VT: f32);
574 SDValue Op = N->getOperand(Num: 0);
575 TargetLowering::MakeLibCallOptions CallOptions;
576 EVT OpsVT[1] = { N->getOperand(Num: 0).getValueType() };
577 CallOptions.setTypeListBeforeSoften(OpsVT, RetVT: N->getValueType(ResNo: 0), Value: true);
578 SDValue Res32 = TLI.makeLibCall(DAG, LC: RTLIB::FPEXT_F16_F32, RetVT: MidVT, Ops: Op,
579 CallOptions, dl: SDLoc(N)).first;
580 if (N->getValueType(ResNo: 0) == MVT::f32)
581 return Res32;
582
583 EVT NVT = TLI.getTypeToTransformTo(Context&: *DAG.getContext(), VT: N->getValueType(ResNo: 0));
584 RTLIB::Libcall LC = RTLIB::getFPEXT(MVT::OpVT: f32, RetVT: N->getValueType(ResNo: 0));
585 assert(LC != RTLIB::UNKNOWN_LIBCALL && "Unsupported FP_EXTEND!");
586 return TLI.makeLibCall(DAG, LC, RetVT: NVT, Ops: Res32, CallOptions, dl: SDLoc(N)).first;
587}
588
589// FIXME: Should we just use 'normal' FP_EXTEND / FP_TRUNC instead of special
590// nodes?
591SDValue DAGTypeLegalizer::SoftenFloatRes_BF16_TO_FP(SDNode *N) {
592 assert(N->getValueType(0) == MVT::f32 &&
593 "Can only soften BF16_TO_FP with f32 result");
594 EVT NVT = TLI.getTypeToTransformTo(Context&: *DAG.getContext(), MVT::VT: f32);
595 SDValue Op = N->getOperand(Num: 0);
596 SDLoc DL(N);
597 Op = DAG.getNode(ISD::ANY_EXTEND, DL, NVT,
598 DAG.getNode(ISD::BITCAST, DL, MVT::i16, Op));
599 SDValue Res = DAG.getNode(Opcode: ISD::SHL, DL, VT: NVT, N1: Op,
600 N2: DAG.getShiftAmountConstant(Val: 16, VT: NVT, DL));
601 return Res;
602}
603
604SDValue DAGTypeLegalizer::SoftenFloatRes_FP_ROUND(SDNode *N) {
605 bool IsStrict = N->isStrictFPOpcode();
606 EVT NVT = TLI.getTypeToTransformTo(Context&: *DAG.getContext(), VT: N->getValueType(ResNo: 0));
607 SDValue Op = N->getOperand(Num: IsStrict ? 1 : 0);
608 SDValue Chain = IsStrict ? N->getOperand(Num: 0) : SDValue();
609 RTLIB::Libcall LC = RTLIB::getFPROUND(OpVT: Op.getValueType(), RetVT: N->getValueType(ResNo: 0));
610 assert(LC != RTLIB::UNKNOWN_LIBCALL && "Unsupported FP_ROUND!");
611 TargetLowering::MakeLibCallOptions CallOptions;
612 EVT OpVT = N->getOperand(Num: IsStrict ? 1 : 0).getValueType();
613 CallOptions.setTypeListBeforeSoften(OpsVT: OpVT, RetVT: N->getValueType(ResNo: 0), Value: true);
614 std::pair<SDValue, SDValue> Tmp = TLI.makeLibCall(DAG, LC, RetVT: NVT, Ops: Op,
615 CallOptions, dl: SDLoc(N),
616 Chain);
617 if (IsStrict)
618 ReplaceValueWith(From: SDValue(N, 1), To: Tmp.second);
619 return Tmp.first;
620}
621
622SDValue DAGTypeLegalizer::SoftenFloatRes_FPOW(SDNode *N) {
623 return SoftenFloatRes_Binary(N, LC: GetFPLibCall(VT: N->getValueType(ResNo: 0),
624 Call_F32: RTLIB::POW_F32,
625 Call_F64: RTLIB::POW_F64,
626 Call_F80: RTLIB::POW_F80,
627 Call_F128: RTLIB::POW_F128,
628 Call_PPCF128: RTLIB::POW_PPCF128));
629}
630
631SDValue DAGTypeLegalizer::SoftenFloatRes_ExpOp(SDNode *N) {
632 bool IsStrict = N->isStrictFPOpcode();
633 unsigned Offset = IsStrict ? 1 : 0;
634 assert((N->getOperand(1 + Offset).getValueType() == MVT::i16 ||
635 N->getOperand(1 + Offset).getValueType() == MVT::i32) &&
636 "Unsupported power type!");
637 bool IsPowI =
638 N->getOpcode() == ISD::FPOWI || N->getOpcode() == ISD::STRICT_FPOWI;
639
640 RTLIB::Libcall LC = IsPowI ? RTLIB::getPOWI(RetVT: N->getValueType(ResNo: 0))
641 : RTLIB::getLDEXP(RetVT: N->getValueType(ResNo: 0));
642 assert(LC != RTLIB::UNKNOWN_LIBCALL && "Unexpected fpowi.");
643 if (!TLI.getLibcallName(Call: LC)) {
644 // Some targets don't have a powi libcall; use pow instead.
645 // FIXME: Implement this if some target needs it.
646 DAG.getContext()->emitError(ErrorStr: "Don't know how to soften fpowi to fpow");
647 return DAG.getUNDEF(VT: N->getValueType(ResNo: 0));
648 }
649
650 if (DAG.getLibInfo().getIntSize() !=
651 N->getOperand(Num: 1 + Offset).getValueType().getSizeInBits()) {
652 // If the exponent does not match with sizeof(int) a libcall to RTLIB::POWI
653 // would use the wrong type for the argument.
654 DAG.getContext()->emitError(ErrorStr: "POWI exponent does not match sizeof(int)");
655 return DAG.getUNDEF(VT: N->getValueType(ResNo: 0));
656 }
657
658 EVT NVT = TLI.getTypeToTransformTo(Context&: *DAG.getContext(), VT: N->getValueType(ResNo: 0));
659 SDValue Ops[2] = { GetSoftenedFloat(Op: N->getOperand(Num: 0 + Offset)),
660 N->getOperand(Num: 1 + Offset) };
661 SDValue Chain = IsStrict ? N->getOperand(Num: 0) : SDValue();
662 TargetLowering::MakeLibCallOptions CallOptions;
663 EVT OpsVT[2] = { N->getOperand(Num: 0 + Offset).getValueType(),
664 N->getOperand(Num: 1 + Offset).getValueType() };
665 CallOptions.setTypeListBeforeSoften(OpsVT, RetVT: N->getValueType(ResNo: 0), Value: true);
666 std::pair<SDValue, SDValue> Tmp = TLI.makeLibCall(DAG, LC, RetVT: NVT, Ops,
667 CallOptions, dl: SDLoc(N),
668 Chain);
669 if (IsStrict)
670 ReplaceValueWith(From: SDValue(N, 1), To: Tmp.second);
671 return Tmp.first;
672}
673
674SDValue DAGTypeLegalizer::SoftenFloatRes_FFREXP(SDNode *N) {
675 assert(!N->isStrictFPOpcode() && "strictfp not implemented for frexp");
676 EVT VT0 = N->getValueType(ResNo: 0);
677 EVT VT1 = N->getValueType(ResNo: 1);
678 RTLIB::Libcall LC = RTLIB::getFREXP(RetVT: VT0);
679
680 if (DAG.getLibInfo().getIntSize() != VT1.getSizeInBits()) {
681 // If the exponent does not match with sizeof(int) a libcall would use the
682 // wrong type for the argument.
683 // TODO: Should be able to handle mismatches.
684 DAG.getContext()->emitError(ErrorStr: "ffrexp exponent does not match sizeof(int)");
685 return DAG.getUNDEF(VT: N->getValueType(ResNo: 0));
686 }
687
688 EVT NVT0 = TLI.getTypeToTransformTo(Context&: *DAG.getContext(), VT: VT0);
689 SDValue StackSlot = DAG.CreateStackTemporary(VT: VT1);
690
691 SDLoc DL(N);
692
693 TargetLowering::MakeLibCallOptions CallOptions;
694 SDValue Ops[2] = {GetSoftenedFloat(Op: N->getOperand(Num: 0)), StackSlot};
695 EVT OpsVT[2] = {VT0, StackSlot.getValueType()};
696
697 // TODO: setTypeListBeforeSoften can't properly express multiple return types,
698 // but we only really need to handle the 0th one for softening anyway.
699 CallOptions.setTypeListBeforeSoften(OpsVT: {OpsVT}, RetVT: VT0, Value: true);
700
701 auto [ReturnVal, Chain] = TLI.makeLibCall(DAG, LC, RetVT: NVT0, Ops, CallOptions, dl: DL,
702 /*Chain=*/SDValue());
703 int FrameIdx = cast<FrameIndexSDNode>(Val&: StackSlot)->getIndex();
704 auto PtrInfo =
705 MachinePointerInfo::getFixedStack(MF&: DAG.getMachineFunction(), FI: FrameIdx);
706
707 SDValue LoadExp = DAG.getLoad(VT: VT1, dl: DL, Chain, Ptr: StackSlot, PtrInfo);
708
709 ReplaceValueWith(From: SDValue(N, 1), To: LoadExp);
710 return ReturnVal;
711}
712
713SDValue DAGTypeLegalizer::SoftenFloatRes_FREM(SDNode *N) {
714 return SoftenFloatRes_Binary(N, LC: GetFPLibCall(VT: N->getValueType(ResNo: 0),
715 Call_F32: RTLIB::REM_F32,
716 Call_F64: RTLIB::REM_F64,
717 Call_F80: RTLIB::REM_F80,
718 Call_F128: RTLIB::REM_F128,
719 Call_PPCF128: RTLIB::REM_PPCF128));
720}
721
722SDValue DAGTypeLegalizer::SoftenFloatRes_FRINT(SDNode *N) {
723 return SoftenFloatRes_Unary(N, LC: GetFPLibCall(VT: N->getValueType(ResNo: 0),
724 Call_F32: RTLIB::RINT_F32,
725 Call_F64: RTLIB::RINT_F64,
726 Call_F80: RTLIB::RINT_F80,
727 Call_F128: RTLIB::RINT_F128,
728 Call_PPCF128: RTLIB::RINT_PPCF128));
729}
730
731SDValue DAGTypeLegalizer::SoftenFloatRes_FROUND(SDNode *N) {
732 return SoftenFloatRes_Unary(N, LC: GetFPLibCall(VT: N->getValueType(ResNo: 0),
733 Call_F32: RTLIB::ROUND_F32,
734 Call_F64: RTLIB::ROUND_F64,
735 Call_F80: RTLIB::ROUND_F80,
736 Call_F128: RTLIB::ROUND_F128,
737 Call_PPCF128: RTLIB::ROUND_PPCF128));
738}
739
740SDValue DAGTypeLegalizer::SoftenFloatRes_FROUNDEVEN(SDNode *N) {
741 return SoftenFloatRes_Unary(N, LC: GetFPLibCall(VT: N->getValueType(ResNo: 0),
742 Call_F32: RTLIB::ROUNDEVEN_F32,
743 Call_F64: RTLIB::ROUNDEVEN_F64,
744 Call_F80: RTLIB::ROUNDEVEN_F80,
745 Call_F128: RTLIB::ROUNDEVEN_F128,
746 Call_PPCF128: RTLIB::ROUNDEVEN_PPCF128));
747}
748
749SDValue DAGTypeLegalizer::SoftenFloatRes_FSIN(SDNode *N) {
750 return SoftenFloatRes_Unary(N, LC: GetFPLibCall(VT: N->getValueType(ResNo: 0),
751 Call_F32: RTLIB::SIN_F32,
752 Call_F64: RTLIB::SIN_F64,
753 Call_F80: RTLIB::SIN_F80,
754 Call_F128: RTLIB::SIN_F128,
755 Call_PPCF128: RTLIB::SIN_PPCF128));
756}
757
758SDValue DAGTypeLegalizer::SoftenFloatRes_FSQRT(SDNode *N) {
759 return SoftenFloatRes_Unary(N, LC: GetFPLibCall(VT: N->getValueType(ResNo: 0),
760 Call_F32: RTLIB::SQRT_F32,
761 Call_F64: RTLIB::SQRT_F64,
762 Call_F80: RTLIB::SQRT_F80,
763 Call_F128: RTLIB::SQRT_F128,
764 Call_PPCF128: RTLIB::SQRT_PPCF128));
765}
766
767SDValue DAGTypeLegalizer::SoftenFloatRes_FSUB(SDNode *N) {
768 return SoftenFloatRes_Binary(N, LC: GetFPLibCall(VT: N->getValueType(ResNo: 0),
769 Call_F32: RTLIB::SUB_F32,
770 Call_F64: RTLIB::SUB_F64,
771 Call_F80: RTLIB::SUB_F80,
772 Call_F128: RTLIB::SUB_F128,
773 Call_PPCF128: RTLIB::SUB_PPCF128));
774}
775
776SDValue DAGTypeLegalizer::SoftenFloatRes_FTRUNC(SDNode *N) {
777 return SoftenFloatRes_Unary(N, LC: GetFPLibCall(VT: N->getValueType(ResNo: 0),
778 Call_F32: RTLIB::TRUNC_F32,
779 Call_F64: RTLIB::TRUNC_F64,
780 Call_F80: RTLIB::TRUNC_F80,
781 Call_F128: RTLIB::TRUNC_F128,
782 Call_PPCF128: RTLIB::TRUNC_PPCF128));
783}
784
785SDValue DAGTypeLegalizer::SoftenFloatRes_LOAD(SDNode *N) {
786 LoadSDNode *L = cast<LoadSDNode>(Val: N);
787 EVT VT = N->getValueType(ResNo: 0);
788 EVT NVT = TLI.getTypeToTransformTo(Context&: *DAG.getContext(), VT);
789 SDLoc dl(N);
790
791 auto MMOFlags =
792 L->getMemOperand()->getFlags() &
793 ~(MachineMemOperand::MOInvariant | MachineMemOperand::MODereferenceable);
794 SDValue NewL;
795 if (L->getExtensionType() == ISD::NON_EXTLOAD) {
796 NewL = DAG.getLoad(AM: L->getAddressingMode(), ExtType: L->getExtensionType(), VT: NVT, dl,
797 Chain: L->getChain(), Ptr: L->getBasePtr(), Offset: L->getOffset(),
798 PtrInfo: L->getPointerInfo(), MemVT: NVT, Alignment: L->getOriginalAlign(),
799 MMOFlags, AAInfo: L->getAAInfo());
800 // Legalized the chain result - switch anything that used the old chain to
801 // use the new one.
802 ReplaceValueWith(From: SDValue(N, 1), To: NewL.getValue(R: 1));
803 return NewL;
804 }
805
806 // Do a non-extending load followed by FP_EXTEND.
807 NewL = DAG.getLoad(AM: L->getAddressingMode(), ExtType: ISD::NON_EXTLOAD, VT: L->getMemoryVT(),
808 dl, Chain: L->getChain(), Ptr: L->getBasePtr(), Offset: L->getOffset(),
809 PtrInfo: L->getPointerInfo(), MemVT: L->getMemoryVT(),
810 Alignment: L->getOriginalAlign(), MMOFlags, AAInfo: L->getAAInfo());
811 // Legalized the chain result - switch anything that used the old chain to
812 // use the new one.
813 ReplaceValueWith(From: SDValue(N, 1), To: NewL.getValue(R: 1));
814 auto ExtendNode = DAG.getNode(Opcode: ISD::FP_EXTEND, DL: dl, VT, Operand: NewL);
815 return BitConvertToInteger(Op: ExtendNode);
816}
817
818SDValue DAGTypeLegalizer::SoftenFloatRes_SELECT(SDNode *N) {
819 SDValue LHS = GetSoftenedFloat(Op: N->getOperand(Num: 1));
820 SDValue RHS = GetSoftenedFloat(Op: N->getOperand(Num: 2));
821 return DAG.getSelect(DL: SDLoc(N),
822 VT: LHS.getValueType(), Cond: N->getOperand(Num: 0), LHS, RHS);
823}
824
825SDValue DAGTypeLegalizer::SoftenFloatRes_SELECT_CC(SDNode *N) {
826 SDValue LHS = GetSoftenedFloat(Op: N->getOperand(Num: 2));
827 SDValue RHS = GetSoftenedFloat(Op: N->getOperand(Num: 3));
828 return DAG.getNode(Opcode: ISD::SELECT_CC, DL: SDLoc(N),
829 VT: LHS.getValueType(), N1: N->getOperand(Num: 0),
830 N2: N->getOperand(Num: 1), N3: LHS, N4: RHS, N5: N->getOperand(Num: 4));
831}
832
833SDValue DAGTypeLegalizer::SoftenFloatRes_UNDEF(SDNode *N) {
834 return DAG.getUNDEF(VT: TLI.getTypeToTransformTo(Context&: *DAG.getContext(),
835 VT: N->getValueType(ResNo: 0)));
836}
837
838SDValue DAGTypeLegalizer::SoftenFloatRes_VAARG(SDNode *N) {
839 SDValue Chain = N->getOperand(Num: 0); // Get the chain.
840 SDValue Ptr = N->getOperand(Num: 1); // Get the pointer.
841 EVT VT = N->getValueType(ResNo: 0);
842 EVT NVT = TLI.getTypeToTransformTo(Context&: *DAG.getContext(), VT);
843 SDLoc dl(N);
844
845 SDValue NewVAARG;
846 NewVAARG = DAG.getVAArg(VT: NVT, dl, Chain, Ptr, SV: N->getOperand(Num: 2),
847 Align: N->getConstantOperandVal(Num: 3));
848
849 // Legalized the chain result - switch anything that used the old chain to
850 // use the new one.
851 if (N != NewVAARG.getValue(R: 1).getNode())
852 ReplaceValueWith(From: SDValue(N, 1), To: NewVAARG.getValue(R: 1));
853 return NewVAARG;
854}
855
856SDValue DAGTypeLegalizer::SoftenFloatRes_XINT_TO_FP(SDNode *N) {
857 bool IsStrict = N->isStrictFPOpcode();
858 bool Signed = N->getOpcode() == ISD::SINT_TO_FP ||
859 N->getOpcode() == ISD::STRICT_SINT_TO_FP;
860 EVT SVT = N->getOperand(Num: IsStrict ? 1 : 0).getValueType();
861 EVT RVT = N->getValueType(ResNo: 0);
862 EVT NVT = EVT();
863 SDLoc dl(N);
864
865 // If the input is not legal, eg: i1 -> fp, then it needs to be promoted to
866 // a larger type, eg: i8 -> fp. Even if it is legal, no libcall may exactly
867 // match. Look for an appropriate libcall.
868 RTLIB::Libcall LC = RTLIB::UNKNOWN_LIBCALL;
869 for (unsigned t = MVT::FIRST_INTEGER_VALUETYPE;
870 t <= MVT::LAST_INTEGER_VALUETYPE && LC == RTLIB::UNKNOWN_LIBCALL; ++t) {
871 NVT = (MVT::SimpleValueType)t;
872 // The source needs to big enough to hold the operand.
873 if (NVT.bitsGE(VT: SVT))
874 LC = Signed ? RTLIB::getSINTTOFP(OpVT: NVT, RetVT: RVT):RTLIB::getUINTTOFP (OpVT: NVT, RetVT: RVT);
875 }
876 assert(LC != RTLIB::UNKNOWN_LIBCALL && "Unsupported XINT_TO_FP!");
877
878 SDValue Chain = IsStrict ? N->getOperand(Num: 0) : SDValue();
879 // Sign/zero extend the argument if the libcall takes a larger type.
880 SDValue Op = DAG.getNode(Opcode: Signed ? ISD::SIGN_EXTEND : ISD::ZERO_EXTEND, DL: dl,
881 VT: NVT, Operand: N->getOperand(Num: IsStrict ? 1 : 0));
882 TargetLowering::MakeLibCallOptions CallOptions;
883 CallOptions.setSExt(Signed);
884 CallOptions.setTypeListBeforeSoften(OpsVT: SVT, RetVT: RVT, Value: true);
885 std::pair<SDValue, SDValue> Tmp =
886 TLI.makeLibCall(DAG, LC, RetVT: TLI.getTypeToTransformTo(Context&: *DAG.getContext(), VT: RVT),
887 Ops: Op, CallOptions, dl, Chain);
888
889 if (IsStrict)
890 ReplaceValueWith(From: SDValue(N, 1), To: Tmp.second);
891 return Tmp.first;
892}
893
894SDValue DAGTypeLegalizer::SoftenFloatRes_VECREDUCE(SDNode *N) {
895 // Expand and soften recursively.
896 ReplaceValueWith(From: SDValue(N, 0), To: TLI.expandVecReduce(Node: N, DAG));
897 return SDValue();
898}
899
900SDValue DAGTypeLegalizer::SoftenFloatRes_VECREDUCE_SEQ(SDNode *N) {
901 ReplaceValueWith(From: SDValue(N, 0), To: TLI.expandVecReduceSeq(Node: N, DAG));
902 return SDValue();
903}
904
905//===----------------------------------------------------------------------===//
906// Convert Float Operand to Integer
907//===----------------------------------------------------------------------===//
908
909bool DAGTypeLegalizer::SoftenFloatOperand(SDNode *N, unsigned OpNo) {
910 LLVM_DEBUG(dbgs() << "Soften float operand " << OpNo << ": "; N->dump(&DAG));
911 SDValue Res = SDValue();
912
913 switch (N->getOpcode()) {
914 default:
915#ifndef NDEBUG
916 dbgs() << "SoftenFloatOperand Op #" << OpNo << ": ";
917 N->dump(G: &DAG); dbgs() << "\n";
918#endif
919 report_fatal_error(reason: "Do not know how to soften this operator's operand!");
920
921 case ISD::BITCAST: Res = SoftenFloatOp_BITCAST(N); break;
922 case ISD::BR_CC: Res = SoftenFloatOp_BR_CC(N); break;
923 case ISD::STRICT_FP_TO_FP16:
924 case ISD::FP_TO_FP16: // Same as FP_ROUND for softening purposes
925 case ISD::FP_TO_BF16:
926 case ISD::STRICT_FP_TO_BF16:
927 case ISD::STRICT_FP_ROUND:
928 case ISD::FP_ROUND: Res = SoftenFloatOp_FP_ROUND(N); break;
929 case ISD::STRICT_FP_TO_SINT:
930 case ISD::STRICT_FP_TO_UINT:
931 case ISD::FP_TO_SINT:
932 case ISD::FP_TO_UINT: Res = SoftenFloatOp_FP_TO_XINT(N); break;
933 case ISD::FP_TO_SINT_SAT:
934 case ISD::FP_TO_UINT_SAT:
935 Res = SoftenFloatOp_FP_TO_XINT_SAT(N); break;
936 case ISD::STRICT_LROUND:
937 case ISD::LROUND: Res = SoftenFloatOp_LROUND(N); break;
938 case ISD::STRICT_LLROUND:
939 case ISD::LLROUND: Res = SoftenFloatOp_LLROUND(N); break;
940 case ISD::STRICT_LRINT:
941 case ISD::LRINT: Res = SoftenFloatOp_LRINT(N); break;
942 case ISD::STRICT_LLRINT:
943 case ISD::LLRINT: Res = SoftenFloatOp_LLRINT(N); break;
944 case ISD::SELECT_CC: Res = SoftenFloatOp_SELECT_CC(N); break;
945 case ISD::STRICT_FSETCC:
946 case ISD::STRICT_FSETCCS:
947 case ISD::SETCC: Res = SoftenFloatOp_SETCC(N); break;
948 case ISD::STORE: Res = SoftenFloatOp_STORE(N, OpNo); break;
949 case ISD::FCOPYSIGN: Res = SoftenFloatOp_FCOPYSIGN(N); break;
950 }
951
952 // If the result is null, the sub-method took care of registering results etc.
953 if (!Res.getNode()) return false;
954
955 // If the result is N, the sub-method updated N in place. Tell the legalizer
956 // core about this to re-analyze.
957 if (Res.getNode() == N)
958 return true;
959
960 assert(Res.getValueType() == N->getValueType(0) && N->getNumValues() == 1 &&
961 "Invalid operand softening");
962
963 ReplaceValueWith(From: SDValue(N, 0), To: Res);
964 return false;
965}
966
967SDValue DAGTypeLegalizer::SoftenFloatOp_BITCAST(SDNode *N) {
968 SDValue Op0 = GetSoftenedFloat(Op: N->getOperand(Num: 0));
969
970 return DAG.getNode(Opcode: ISD::BITCAST, DL: SDLoc(N), VT: N->getValueType(ResNo: 0), Operand: Op0);
971}
972
973SDValue DAGTypeLegalizer::SoftenFloatOp_FP_ROUND(SDNode *N) {
974 // We actually deal with the partially-softened FP_TO_FP16 node too, which
975 // returns an i16 so doesn't meet the constraints necessary for FP_ROUND.
976 assert(N->getOpcode() == ISD::FP_ROUND || N->getOpcode() == ISD::FP_TO_FP16 ||
977 N->getOpcode() == ISD::STRICT_FP_TO_FP16 ||
978 N->getOpcode() == ISD::FP_TO_BF16 ||
979 N->getOpcode() == ISD::STRICT_FP_TO_BF16 ||
980 N->getOpcode() == ISD::STRICT_FP_ROUND);
981
982 bool IsStrict = N->isStrictFPOpcode();
983 SDValue Op = N->getOperand(Num: IsStrict ? 1 : 0);
984 EVT SVT = Op.getValueType();
985 EVT RVT = N->getValueType(ResNo: 0);
986 EVT FloatRVT = RVT;
987 if (N->getOpcode() == ISD::FP_TO_FP16 ||
988 N->getOpcode() == ISD::STRICT_FP_TO_FP16)
989 FloatRVT = MVT::f16;
990 else if (N->getOpcode() == ISD::FP_TO_BF16 ||
991 N->getOpcode() == ISD::STRICT_FP_TO_BF16)
992 FloatRVT = MVT::bf16;
993
994 RTLIB::Libcall LC = RTLIB::getFPROUND(OpVT: SVT, RetVT: FloatRVT);
995 assert(LC != RTLIB::UNKNOWN_LIBCALL && "Unsupported FP_ROUND libcall");
996
997 SDValue Chain = IsStrict ? N->getOperand(Num: 0) : SDValue();
998 Op = GetSoftenedFloat(Op);
999 TargetLowering::MakeLibCallOptions CallOptions;
1000 CallOptions.setTypeListBeforeSoften(OpsVT: SVT, RetVT: RVT, Value: true);
1001 std::pair<SDValue, SDValue> Tmp = TLI.makeLibCall(DAG, LC, RetVT: RVT, Ops: Op,
1002 CallOptions, dl: SDLoc(N),
1003 Chain);
1004 if (IsStrict) {
1005 ReplaceValueWith(From: SDValue(N, 1), To: Tmp.second);
1006 ReplaceValueWith(From: SDValue(N, 0), To: Tmp.first);
1007 return SDValue();
1008 }
1009 return Tmp.first;
1010}
1011
1012SDValue DAGTypeLegalizer::SoftenFloatOp_BR_CC(SDNode *N) {
1013 SDValue NewLHS = N->getOperand(Num: 2), NewRHS = N->getOperand(Num: 3);
1014 ISD::CondCode CCCode = cast<CondCodeSDNode>(Val: N->getOperand(Num: 1))->get();
1015
1016 EVT VT = NewLHS.getValueType();
1017 NewLHS = GetSoftenedFloat(Op: NewLHS);
1018 NewRHS = GetSoftenedFloat(Op: NewRHS);
1019 TLI.softenSetCCOperands(DAG, VT, NewLHS, NewRHS, CCCode, DL: SDLoc(N),
1020 OldLHS: N->getOperand(Num: 2), OldRHS: N->getOperand(Num: 3));
1021
1022 // If softenSetCCOperands returned a scalar, we need to compare the result
1023 // against zero to select between true and false values.
1024 if (!NewRHS.getNode()) {
1025 NewRHS = DAG.getConstant(Val: 0, DL: SDLoc(N), VT: NewLHS.getValueType());
1026 CCCode = ISD::SETNE;
1027 }
1028
1029 // Update N to have the operands specified.
1030 return SDValue(DAG.UpdateNodeOperands(N, Op1: N->getOperand(Num: 0),
1031 Op2: DAG.getCondCode(Cond: CCCode), Op3: NewLHS, Op4: NewRHS,
1032 Op5: N->getOperand(Num: 4)),
1033 0);
1034}
1035
1036// Even if the result type is legal, no libcall may exactly match. (e.g. We
1037// don't have FP-i8 conversions) This helper method looks for an appropriate
1038// promoted libcall.
1039static RTLIB::Libcall findFPToIntLibcall(EVT SrcVT, EVT RetVT, EVT &Promoted,
1040 bool Signed) {
1041 RTLIB::Libcall LC = RTLIB::UNKNOWN_LIBCALL;
1042 for (unsigned IntVT = MVT::FIRST_INTEGER_VALUETYPE;
1043 IntVT <= MVT::LAST_INTEGER_VALUETYPE && LC == RTLIB::UNKNOWN_LIBCALL;
1044 ++IntVT) {
1045 Promoted = (MVT::SimpleValueType)IntVT;
1046 // The type needs to big enough to hold the result.
1047 if (Promoted.bitsGE(VT: RetVT))
1048 LC = Signed ? RTLIB::getFPTOSINT(OpVT: SrcVT, RetVT: Promoted)
1049 : RTLIB::getFPTOUINT(OpVT: SrcVT, RetVT: Promoted);
1050 }
1051 return LC;
1052}
1053
1054SDValue DAGTypeLegalizer::SoftenFloatOp_FP_TO_XINT(SDNode *N) {
1055 bool IsStrict = N->isStrictFPOpcode();
1056 bool Signed = N->getOpcode() == ISD::FP_TO_SINT ||
1057 N->getOpcode() == ISD::STRICT_FP_TO_SINT;
1058
1059 SDValue Op = N->getOperand(Num: IsStrict ? 1 : 0);
1060 EVT SVT = Op.getValueType();
1061 EVT RVT = N->getValueType(ResNo: 0);
1062 EVT NVT = EVT();
1063 SDLoc dl(N);
1064
1065 // If the result is not legal, eg: fp -> i1, then it needs to be promoted to
1066 // a larger type, eg: fp -> i32. Even if it is legal, no libcall may exactly
1067 // match, eg. we don't have fp -> i8 conversions.
1068 // Look for an appropriate libcall.
1069 RTLIB::Libcall LC = findFPToIntLibcall(SrcVT: SVT, RetVT: RVT, Promoted&: NVT, Signed);
1070 assert(LC != RTLIB::UNKNOWN_LIBCALL && NVT.isSimple() &&
1071 "Unsupported FP_TO_XINT!");
1072
1073 Op = GetSoftenedFloat(Op);
1074 SDValue Chain = IsStrict ? N->getOperand(Num: 0) : SDValue();
1075 TargetLowering::MakeLibCallOptions CallOptions;
1076 CallOptions.setTypeListBeforeSoften(OpsVT: SVT, RetVT: RVT, Value: true);
1077 std::pair<SDValue, SDValue> Tmp = TLI.makeLibCall(DAG, LC, RetVT: NVT, Ops: Op,
1078 CallOptions, dl, Chain);
1079
1080 // Truncate the result if the libcall returns a larger type.
1081 SDValue Res = DAG.getNode(Opcode: ISD::TRUNCATE, DL: dl, VT: RVT, Operand: Tmp.first);
1082
1083 if (!IsStrict)
1084 return Res;
1085
1086 ReplaceValueWith(From: SDValue(N, 1), To: Tmp.second);
1087 ReplaceValueWith(From: SDValue(N, 0), To: Res);
1088 return SDValue();
1089}
1090
1091SDValue DAGTypeLegalizer::SoftenFloatOp_FP_TO_XINT_SAT(SDNode *N) {
1092 SDValue Res = TLI.expandFP_TO_INT_SAT(N, DAG);
1093 return Res;
1094}
1095
1096SDValue DAGTypeLegalizer::SoftenFloatOp_SELECT_CC(SDNode *N) {
1097 SDValue NewLHS = N->getOperand(Num: 0), NewRHS = N->getOperand(Num: 1);
1098 ISD::CondCode CCCode = cast<CondCodeSDNode>(Val: N->getOperand(Num: 4))->get();
1099
1100 EVT VT = NewLHS.getValueType();
1101 NewLHS = GetSoftenedFloat(Op: NewLHS);
1102 NewRHS = GetSoftenedFloat(Op: NewRHS);
1103 TLI.softenSetCCOperands(DAG, VT, NewLHS, NewRHS, CCCode, DL: SDLoc(N),
1104 OldLHS: N->getOperand(Num: 0), OldRHS: N->getOperand(Num: 1));
1105
1106 // If softenSetCCOperands returned a scalar, we need to compare the result
1107 // against zero to select between true and false values.
1108 if (!NewRHS.getNode()) {
1109 NewRHS = DAG.getConstant(Val: 0, DL: SDLoc(N), VT: NewLHS.getValueType());
1110 CCCode = ISD::SETNE;
1111 }
1112
1113 // Update N to have the operands specified.
1114 return SDValue(DAG.UpdateNodeOperands(N, Op1: NewLHS, Op2: NewRHS,
1115 Op3: N->getOperand(Num: 2), Op4: N->getOperand(Num: 3),
1116 Op5: DAG.getCondCode(Cond: CCCode)),
1117 0);
1118}
1119
1120SDValue DAGTypeLegalizer::SoftenFloatOp_SETCC(SDNode *N) {
1121 bool IsStrict = N->isStrictFPOpcode();
1122 SDValue Op0 = N->getOperand(Num: IsStrict ? 1 : 0);
1123 SDValue Op1 = N->getOperand(Num: IsStrict ? 2 : 1);
1124 SDValue Chain = IsStrict ? N->getOperand(Num: 0) : SDValue();
1125 ISD::CondCode CCCode =
1126 cast<CondCodeSDNode>(Val: N->getOperand(Num: IsStrict ? 3 : 2))->get();
1127
1128 EVT VT = Op0.getValueType();
1129 SDValue NewLHS = GetSoftenedFloat(Op: Op0);
1130 SDValue NewRHS = GetSoftenedFloat(Op: Op1);
1131 TLI.softenSetCCOperands(DAG, VT, NewLHS, NewRHS, CCCode, DL: SDLoc(N), OldLHS: Op0, OldRHS: Op1,
1132 Chain, IsSignaling: N->getOpcode() == ISD::STRICT_FSETCCS);
1133
1134 // Update N to have the operands specified.
1135 if (NewRHS.getNode()) {
1136 if (IsStrict)
1137 NewLHS = DAG.getNode(Opcode: ISD::SETCC, DL: SDLoc(N), VT: N->getValueType(ResNo: 0), N1: NewLHS,
1138 N2: NewRHS, N3: DAG.getCondCode(Cond: CCCode));
1139 else
1140 return SDValue(DAG.UpdateNodeOperands(N, Op1: NewLHS, Op2: NewRHS,
1141 Op3: DAG.getCondCode(Cond: CCCode)), 0);
1142 }
1143
1144 // Otherwise, softenSetCCOperands returned a scalar, use it.
1145 assert((NewRHS.getNode() || NewLHS.getValueType() == N->getValueType(0)) &&
1146 "Unexpected setcc expansion!");
1147
1148 if (IsStrict) {
1149 ReplaceValueWith(From: SDValue(N, 0), To: NewLHS);
1150 ReplaceValueWith(From: SDValue(N, 1), To: Chain);
1151 return SDValue();
1152 }
1153 return NewLHS;
1154}
1155
1156SDValue DAGTypeLegalizer::SoftenFloatOp_STORE(SDNode *N, unsigned OpNo) {
1157 assert(ISD::isUNINDEXEDStore(N) && "Indexed store during type legalization!");
1158 assert(OpNo == 1 && "Can only soften the stored value!");
1159 StoreSDNode *ST = cast<StoreSDNode>(Val: N);
1160 SDValue Val = ST->getValue();
1161 SDLoc dl(N);
1162
1163 if (ST->isTruncatingStore())
1164 // Do an FP_ROUND followed by a non-truncating store.
1165 Val = BitConvertToInteger(
1166 Op: DAG.getNode(Opcode: ISD::FP_ROUND, DL: dl, VT: ST->getMemoryVT(), N1: Val,
1167 N2: DAG.getIntPtrConstant(Val: 0, DL: dl, /*isTarget=*/true)));
1168 else
1169 Val = GetSoftenedFloat(Op: Val);
1170
1171 return DAG.getStore(Chain: ST->getChain(), dl, Val, Ptr: ST->getBasePtr(),
1172 MMO: ST->getMemOperand());
1173}
1174
1175SDValue DAGTypeLegalizer::SoftenFloatOp_FCOPYSIGN(SDNode *N) {
1176 SDValue LHS = N->getOperand(Num: 0);
1177 SDValue RHS = BitConvertToInteger(Op: N->getOperand(Num: 1));
1178 SDLoc dl(N);
1179
1180 EVT LVT = LHS.getValueType();
1181 EVT ILVT = EVT::getIntegerVT(Context&: *DAG.getContext(), BitWidth: LVT.getSizeInBits());
1182 EVT RVT = RHS.getValueType();
1183
1184 unsigned LSize = LVT.getSizeInBits();
1185 unsigned RSize = RVT.getSizeInBits();
1186
1187 // Shift right or sign-extend it if the two operands have different types.
1188 int SizeDiff = RSize - LSize;
1189 if (SizeDiff > 0) {
1190 RHS =
1191 DAG.getNode(Opcode: ISD::SRL, DL: dl, VT: RVT, N1: RHS,
1192 N2: DAG.getConstant(Val: SizeDiff, DL: dl,
1193 VT: TLI.getShiftAmountTy(LHSTy: RHS.getValueType(),
1194 DL: DAG.getDataLayout())));
1195 RHS = DAG.getNode(Opcode: ISD::TRUNCATE, DL: dl, VT: ILVT, Operand: RHS);
1196 } else if (SizeDiff < 0) {
1197 RHS = DAG.getNode(Opcode: ISD::ANY_EXTEND, DL: dl, VT: LVT, Operand: RHS);
1198 RHS =
1199 DAG.getNode(Opcode: ISD::SHL, DL: dl, VT: ILVT, N1: RHS,
1200 N2: DAG.getConstant(Val: -SizeDiff, DL: dl,
1201 VT: TLI.getShiftAmountTy(LHSTy: RHS.getValueType(),
1202 DL: DAG.getDataLayout())));
1203 }
1204
1205 RHS = DAG.getBitcast(VT: LVT, V: RHS);
1206 return DAG.getNode(Opcode: ISD::FCOPYSIGN, DL: dl, VT: LVT, N1: LHS, N2: RHS);
1207}
1208
1209SDValue DAGTypeLegalizer::SoftenFloatOp_Unary(SDNode *N, RTLIB::Libcall LC) {
1210 EVT NVT = TLI.getTypeToTransformTo(Context&: *DAG.getContext(), VT: N->getValueType(ResNo: 0));
1211 bool IsStrict = N->isStrictFPOpcode();
1212 unsigned Offset = IsStrict ? 1 : 0;
1213 SDValue Op = GetSoftenedFloat(Op: N->getOperand(Num: 0 + Offset));
1214 SDValue Chain = IsStrict ? N->getOperand(Num: 0) : SDValue();
1215 TargetLowering::MakeLibCallOptions CallOptions;
1216 EVT OpVT = N->getOperand(Num: 0 + Offset).getValueType();
1217 CallOptions.setTypeListBeforeSoften(OpsVT: OpVT, RetVT: N->getValueType(ResNo: 0), Value: true);
1218 std::pair<SDValue, SDValue> Tmp = TLI.makeLibCall(DAG, LC, RetVT: NVT, Ops: Op,
1219 CallOptions, dl: SDLoc(N),
1220 Chain);
1221 if (IsStrict) {
1222 ReplaceValueWith(From: SDValue(N, 1), To: Tmp.second);
1223 ReplaceValueWith(From: SDValue(N, 0), To: Tmp.first);
1224 return SDValue();
1225 }
1226
1227 return Tmp.first;
1228}
1229
1230SDValue DAGTypeLegalizer::SoftenFloatOp_LROUND(SDNode *N) {
1231 EVT OpVT = N->getOperand(Num: N->isStrictFPOpcode() ? 1 : 0).getValueType();
1232 return SoftenFloatOp_Unary(N, LC: GetFPLibCall(VT: OpVT,
1233 Call_F32: RTLIB::LROUND_F32,
1234 Call_F64: RTLIB::LROUND_F64,
1235 Call_F80: RTLIB::LROUND_F80,
1236 Call_F128: RTLIB::LROUND_F128,
1237 Call_PPCF128: RTLIB::LROUND_PPCF128));
1238}
1239
1240SDValue DAGTypeLegalizer::SoftenFloatOp_LLROUND(SDNode *N) {
1241 EVT OpVT = N->getOperand(Num: N->isStrictFPOpcode() ? 1 : 0).getValueType();
1242 return SoftenFloatOp_Unary(N, LC: GetFPLibCall(VT: OpVT,
1243 Call_F32: RTLIB::LLROUND_F32,
1244 Call_F64: RTLIB::LLROUND_F64,
1245 Call_F80: RTLIB::LLROUND_F80,
1246 Call_F128: RTLIB::LLROUND_F128,
1247 Call_PPCF128: RTLIB::LLROUND_PPCF128));
1248}
1249
1250SDValue DAGTypeLegalizer::SoftenFloatOp_LRINT(SDNode *N) {
1251 EVT OpVT = N->getOperand(Num: N->isStrictFPOpcode() ? 1 : 0).getValueType();
1252 return SoftenFloatOp_Unary(N, LC: GetFPLibCall(VT: OpVT,
1253 Call_F32: RTLIB::LRINT_F32,
1254 Call_F64: RTLIB::LRINT_F64,
1255 Call_F80: RTLIB::LRINT_F80,
1256 Call_F128: RTLIB::LRINT_F128,
1257 Call_PPCF128: RTLIB::LRINT_PPCF128));
1258}
1259
1260SDValue DAGTypeLegalizer::SoftenFloatOp_LLRINT(SDNode *N) {
1261 EVT OpVT = N->getOperand(Num: N->isStrictFPOpcode() ? 1 : 0).getValueType();
1262 return SoftenFloatOp_Unary(N, LC: GetFPLibCall(VT: OpVT,
1263 Call_F32: RTLIB::LLRINT_F32,
1264 Call_F64: RTLIB::LLRINT_F64,
1265 Call_F80: RTLIB::LLRINT_F80,
1266 Call_F128: RTLIB::LLRINT_F128,
1267 Call_PPCF128: RTLIB::LLRINT_PPCF128));
1268}
1269
1270//===----------------------------------------------------------------------===//
1271// Float Result Expansion
1272//===----------------------------------------------------------------------===//
1273
1274/// ExpandFloatResult - This method is called when the specified result of the
1275/// specified node is found to need expansion. At this point, the node may also
1276/// have invalid operands or may have other results that need promotion, we just
1277/// know that (at least) one result needs expansion.
1278void DAGTypeLegalizer::ExpandFloatResult(SDNode *N, unsigned ResNo) {
1279 LLVM_DEBUG(dbgs() << "Expand float result: "; N->dump(&DAG));
1280 SDValue Lo, Hi;
1281 Lo = Hi = SDValue();
1282
1283 // See if the target wants to custom expand this node.
1284 if (CustomLowerNode(N, VT: N->getValueType(ResNo), LegalizeResult: true))
1285 return;
1286
1287 switch (N->getOpcode()) {
1288 default:
1289#ifndef NDEBUG
1290 dbgs() << "ExpandFloatResult #" << ResNo << ": ";
1291 N->dump(G: &DAG); dbgs() << "\n";
1292#endif
1293 report_fatal_error(reason: "Do not know how to expand the result of this "
1294 "operator!");
1295
1296 case ISD::UNDEF: SplitRes_UNDEF(N, Lo, Hi); break;
1297 case ISD::SELECT: SplitRes_Select(N, Lo, Hi); break;
1298 case ISD::SELECT_CC: SplitRes_SELECT_CC(N, Lo, Hi); break;
1299
1300 case ISD::MERGE_VALUES: ExpandRes_MERGE_VALUES(N, ResNo, Lo, Hi); break;
1301 case ISD::BITCAST: ExpandRes_BITCAST(N, Lo, Hi); break;
1302 case ISD::BUILD_PAIR: ExpandRes_BUILD_PAIR(N, Lo, Hi); break;
1303 case ISD::EXTRACT_ELEMENT: ExpandRes_EXTRACT_ELEMENT(N, Lo, Hi); break;
1304 case ISD::EXTRACT_VECTOR_ELT: ExpandRes_EXTRACT_VECTOR_ELT(N, Lo, Hi); break;
1305 case ISD::VAARG: ExpandRes_VAARG(N, Lo, Hi); break;
1306
1307 case ISD::ConstantFP: ExpandFloatRes_ConstantFP(N, Lo, Hi); break;
1308 case ISD::FABS: ExpandFloatRes_FABS(N, Lo, Hi); break;
1309 case ISD::STRICT_FMINNUM:
1310 case ISD::FMINNUM: ExpandFloatRes_FMINNUM(N, Lo, Hi); break;
1311 case ISD::STRICT_FMAXNUM:
1312 case ISD::FMAXNUM: ExpandFloatRes_FMAXNUM(N, Lo, Hi); break;
1313 case ISD::STRICT_FADD:
1314 case ISD::FADD: ExpandFloatRes_FADD(N, Lo, Hi); break;
1315 case ISD::FCBRT: ExpandFloatRes_FCBRT(N, Lo, Hi); break;
1316 case ISD::STRICT_FCEIL:
1317 case ISD::FCEIL: ExpandFloatRes_FCEIL(N, Lo, Hi); break;
1318 case ISD::FCOPYSIGN: ExpandFloatRes_FCOPYSIGN(N, Lo, Hi); break;
1319 case ISD::STRICT_FCOS:
1320 case ISD::FCOS: ExpandFloatRes_FCOS(N, Lo, Hi); break;
1321 case ISD::STRICT_FDIV:
1322 case ISD::FDIV: ExpandFloatRes_FDIV(N, Lo, Hi); break;
1323 case ISD::STRICT_FEXP:
1324 case ISD::FEXP: ExpandFloatRes_FEXP(N, Lo, Hi); break;
1325 case ISD::STRICT_FEXP2:
1326 case ISD::FEXP2: ExpandFloatRes_FEXP2(N, Lo, Hi); break;
1327 case ISD::FEXP10: ExpandFloatRes_FEXP10(N, Lo, Hi); break;
1328 case ISD::STRICT_FFLOOR:
1329 case ISD::FFLOOR: ExpandFloatRes_FFLOOR(N, Lo, Hi); break;
1330 case ISD::STRICT_FLOG:
1331 case ISD::FLOG: ExpandFloatRes_FLOG(N, Lo, Hi); break;
1332 case ISD::STRICT_FLOG2:
1333 case ISD::FLOG2: ExpandFloatRes_FLOG2(N, Lo, Hi); break;
1334 case ISD::STRICT_FLOG10:
1335 case ISD::FLOG10: ExpandFloatRes_FLOG10(N, Lo, Hi); break;
1336 case ISD::STRICT_FMA:
1337 case ISD::FMA: ExpandFloatRes_FMA(N, Lo, Hi); break;
1338 case ISD::STRICT_FMUL:
1339 case ISD::FMUL: ExpandFloatRes_FMUL(N, Lo, Hi); break;
1340 case ISD::STRICT_FNEARBYINT:
1341 case ISD::FNEARBYINT: ExpandFloatRes_FNEARBYINT(N, Lo, Hi); break;
1342 case ISD::FNEG: ExpandFloatRes_FNEG(N, Lo, Hi); break;
1343 case ISD::STRICT_FP_EXTEND:
1344 case ISD::FP_EXTEND: ExpandFloatRes_FP_EXTEND(N, Lo, Hi); break;
1345 case ISD::STRICT_FPOW:
1346 case ISD::FPOW: ExpandFloatRes_FPOW(N, Lo, Hi); break;
1347 case ISD::STRICT_FPOWI:
1348 case ISD::FPOWI: ExpandFloatRes_FPOWI(N, Lo, Hi); break;
1349 case ISD::FLDEXP:
1350 case ISD::STRICT_FLDEXP: ExpandFloatRes_FLDEXP(N, Lo, Hi); break;
1351 case ISD::FREEZE: ExpandFloatRes_FREEZE(N, Lo, Hi); break;
1352 case ISD::STRICT_FRINT:
1353 case ISD::FRINT: ExpandFloatRes_FRINT(N, Lo, Hi); break;
1354 case ISD::STRICT_FROUND:
1355 case ISD::FROUND: ExpandFloatRes_FROUND(N, Lo, Hi); break;
1356 case ISD::STRICT_FROUNDEVEN:
1357 case ISD::FROUNDEVEN: ExpandFloatRes_FROUNDEVEN(N, Lo, Hi); break;
1358 case ISD::STRICT_FSIN:
1359 case ISD::FSIN: ExpandFloatRes_FSIN(N, Lo, Hi); break;
1360 case ISD::STRICT_FSQRT:
1361 case ISD::FSQRT: ExpandFloatRes_FSQRT(N, Lo, Hi); break;
1362 case ISD::STRICT_FSUB:
1363 case ISD::FSUB: ExpandFloatRes_FSUB(N, Lo, Hi); break;
1364 case ISD::STRICT_FTRUNC:
1365 case ISD::FTRUNC: ExpandFloatRes_FTRUNC(N, Lo, Hi); break;
1366 case ISD::LOAD: ExpandFloatRes_LOAD(N, Lo, Hi); break;
1367 case ISD::STRICT_SINT_TO_FP:
1368 case ISD::STRICT_UINT_TO_FP:
1369 case ISD::SINT_TO_FP:
1370 case ISD::UINT_TO_FP: ExpandFloatRes_XINT_TO_FP(N, Lo, Hi); break;
1371 case ISD::STRICT_FREM:
1372 case ISD::FREM: ExpandFloatRes_FREM(N, Lo, Hi); break;
1373 }
1374
1375 // If Lo/Hi is null, the sub-method took care of registering results etc.
1376 if (Lo.getNode())
1377 SetExpandedFloat(Op: SDValue(N, ResNo), Lo, Hi);
1378}
1379
1380void DAGTypeLegalizer::ExpandFloatRes_ConstantFP(SDNode *N, SDValue &Lo,
1381 SDValue &Hi) {
1382 EVT NVT = TLI.getTypeToTransformTo(Context&: *DAG.getContext(), VT: N->getValueType(ResNo: 0));
1383 assert(NVT.getSizeInBits() == 64 &&
1384 "Do not know how to expand this float constant!");
1385 APInt C = cast<ConstantFPSDNode>(Val: N)->getValueAPF().bitcastToAPInt();
1386 SDLoc dl(N);
1387 Lo = DAG.getConstantFP(Val: APFloat(DAG.EVTToAPFloatSemantics(VT: NVT),
1388 APInt(64, C.getRawData()[1])),
1389 DL: dl, VT: NVT);
1390 Hi = DAG.getConstantFP(Val: APFloat(DAG.EVTToAPFloatSemantics(VT: NVT),
1391 APInt(64, C.getRawData()[0])),
1392 DL: dl, VT: NVT);
1393}
1394
1395void DAGTypeLegalizer::ExpandFloatRes_Unary(SDNode *N, RTLIB::Libcall LC,
1396 SDValue &Lo, SDValue &Hi) {
1397 bool IsStrict = N->isStrictFPOpcode();
1398 unsigned Offset = IsStrict ? 1 : 0;
1399 SDValue Op = N->getOperand(Num: 0 + Offset);
1400 SDValue Chain = IsStrict ? N->getOperand(Num: 0) : SDValue();
1401 TargetLowering::MakeLibCallOptions CallOptions;
1402 std::pair<SDValue, SDValue> Tmp = TLI.makeLibCall(DAG, LC, RetVT: N->getValueType(ResNo: 0),
1403 Ops: Op, CallOptions, dl: SDLoc(N),
1404 Chain);
1405 if (IsStrict)
1406 ReplaceValueWith(From: SDValue(N, 1), To: Tmp.second);
1407 GetPairElements(Pair: Tmp.first, Lo, Hi);
1408}
1409
1410void DAGTypeLegalizer::ExpandFloatRes_Binary(SDNode *N, RTLIB::Libcall LC,
1411 SDValue &Lo, SDValue &Hi) {
1412 bool IsStrict = N->isStrictFPOpcode();
1413 unsigned Offset = IsStrict ? 1 : 0;
1414 SDValue Ops[] = { N->getOperand(Num: 0 + Offset), N->getOperand(Num: 1 + Offset) };
1415 SDValue Chain = IsStrict ? N->getOperand(Num: 0) : SDValue();
1416 TargetLowering::MakeLibCallOptions CallOptions;
1417 std::pair<SDValue, SDValue> Tmp = TLI.makeLibCall(DAG, LC, RetVT: N->getValueType(ResNo: 0),
1418 Ops, CallOptions, dl: SDLoc(N),
1419 Chain);
1420 if (IsStrict)
1421 ReplaceValueWith(From: SDValue(N, 1), To: Tmp.second);
1422 GetPairElements(Pair: Tmp.first, Lo, Hi);
1423}
1424
1425void DAGTypeLegalizer::ExpandFloatRes_FABS(SDNode *N, SDValue &Lo,
1426 SDValue &Hi) {
1427 assert(N->getValueType(0) == MVT::ppcf128 &&
1428 "Logic only correct for ppcf128!");
1429 SDLoc dl(N);
1430 SDValue Tmp;
1431 GetExpandedFloat(Op: N->getOperand(Num: 0), Lo, Hi&: Tmp);
1432 Hi = DAG.getNode(Opcode: ISD::FABS, DL: dl, VT: Tmp.getValueType(), Operand: Tmp);
1433 // Lo = Hi==fabs(Hi) ? Lo : -Lo;
1434 Lo = DAG.getSelectCC(DL: dl, LHS: Tmp, RHS: Hi, True: Lo,
1435 False: DAG.getNode(Opcode: ISD::FNEG, DL: dl, VT: Lo.getValueType(), Operand: Lo),
1436 Cond: ISD::SETEQ);
1437}
1438
1439void DAGTypeLegalizer::ExpandFloatRes_FMINNUM(SDNode *N, SDValue &Lo,
1440 SDValue &Hi) {
1441 ExpandFloatRes_Binary(N, LC: GetFPLibCall(VT: N->getValueType(ResNo: 0),
1442 Call_F32: RTLIB::FMIN_F32, Call_F64: RTLIB::FMIN_F64,
1443 Call_F80: RTLIB::FMIN_F80, Call_F128: RTLIB::FMIN_F128,
1444 Call_PPCF128: RTLIB::FMIN_PPCF128), Lo, Hi);
1445}
1446
1447void DAGTypeLegalizer::ExpandFloatRes_FMAXNUM(SDNode *N, SDValue &Lo,
1448 SDValue &Hi) {
1449 ExpandFloatRes_Binary(N, LC: GetFPLibCall(VT: N->getValueType(ResNo: 0),
1450 Call_F32: RTLIB::FMAX_F32, Call_F64: RTLIB::FMAX_F64,
1451 Call_F80: RTLIB::FMAX_F80, Call_F128: RTLIB::FMAX_F128,
1452 Call_PPCF128: RTLIB::FMAX_PPCF128), Lo, Hi);
1453}
1454
1455void DAGTypeLegalizer::ExpandFloatRes_FADD(SDNode *N, SDValue &Lo,
1456 SDValue &Hi) {
1457 ExpandFloatRes_Binary(N, LC: GetFPLibCall(VT: N->getValueType(ResNo: 0),
1458 Call_F32: RTLIB::ADD_F32, Call_F64: RTLIB::ADD_F64,
1459 Call_F80: RTLIB::ADD_F80, Call_F128: RTLIB::ADD_F128,
1460 Call_PPCF128: RTLIB::ADD_PPCF128), Lo, Hi);
1461}
1462
1463void DAGTypeLegalizer::ExpandFloatRes_FCBRT(SDNode *N, SDValue &Lo,
1464 SDValue &Hi) {
1465 ExpandFloatRes_Unary(N, LC: GetFPLibCall(VT: N->getValueType(ResNo: 0), Call_F32: RTLIB::CBRT_F32,
1466 Call_F64: RTLIB::CBRT_F64, Call_F80: RTLIB::CBRT_F80,
1467 Call_F128: RTLIB::CBRT_F128,
1468 Call_PPCF128: RTLIB::CBRT_PPCF128), Lo, Hi);
1469}
1470
1471void DAGTypeLegalizer::ExpandFloatRes_FCEIL(SDNode *N,
1472 SDValue &Lo, SDValue &Hi) {
1473 ExpandFloatRes_Unary(N, LC: GetFPLibCall(VT: N->getValueType(ResNo: 0),
1474 Call_F32: RTLIB::CEIL_F32, Call_F64: RTLIB::CEIL_F64,
1475 Call_F80: RTLIB::CEIL_F80, Call_F128: RTLIB::CEIL_F128,
1476 Call_PPCF128: RTLIB::CEIL_PPCF128), Lo, Hi);
1477}
1478
1479void DAGTypeLegalizer::ExpandFloatRes_FCOPYSIGN(SDNode *N,
1480 SDValue &Lo, SDValue &Hi) {
1481 ExpandFloatRes_Binary(N, LC: GetFPLibCall(VT: N->getValueType(ResNo: 0),
1482 Call_F32: RTLIB::COPYSIGN_F32,
1483 Call_F64: RTLIB::COPYSIGN_F64,
1484 Call_F80: RTLIB::COPYSIGN_F80,
1485 Call_F128: RTLIB::COPYSIGN_F128,
1486 Call_PPCF128: RTLIB::COPYSIGN_PPCF128), Lo, Hi);
1487}
1488
1489void DAGTypeLegalizer::ExpandFloatRes_FCOS(SDNode *N,
1490 SDValue &Lo, SDValue &Hi) {
1491 ExpandFloatRes_Unary(N, LC: GetFPLibCall(VT: N->getValueType(ResNo: 0),
1492 Call_F32: RTLIB::COS_F32, Call_F64: RTLIB::COS_F64,
1493 Call_F80: RTLIB::COS_F80, Call_F128: RTLIB::COS_F128,
1494 Call_PPCF128: RTLIB::COS_PPCF128), Lo, Hi);
1495}
1496
1497void DAGTypeLegalizer::ExpandFloatRes_FDIV(SDNode *N, SDValue &Lo,
1498 SDValue &Hi) {
1499 ExpandFloatRes_Binary(N, LC: GetFPLibCall(VT: N->getValueType(ResNo: 0),
1500 Call_F32: RTLIB::DIV_F32,
1501 Call_F64: RTLIB::DIV_F64,
1502 Call_F80: RTLIB::DIV_F80,
1503 Call_F128: RTLIB::DIV_F128,
1504 Call_PPCF128: RTLIB::DIV_PPCF128), Lo, Hi);
1505}
1506
1507void DAGTypeLegalizer::ExpandFloatRes_FEXP(SDNode *N,
1508 SDValue &Lo, SDValue &Hi) {
1509 ExpandFloatRes_Unary(N, LC: GetFPLibCall(VT: N->getValueType(ResNo: 0),
1510 Call_F32: RTLIB::EXP_F32, Call_F64: RTLIB::EXP_F64,
1511 Call_F80: RTLIB::EXP_F80, Call_F128: RTLIB::EXP_F128,
1512 Call_PPCF128: RTLIB::EXP_PPCF128), Lo, Hi);
1513}
1514
1515void DAGTypeLegalizer::ExpandFloatRes_FEXP2(SDNode *N,
1516 SDValue &Lo, SDValue &Hi) {
1517 ExpandFloatRes_Unary(N, LC: GetFPLibCall(VT: N->getValueType(ResNo: 0),
1518 Call_F32: RTLIB::EXP2_F32, Call_F64: RTLIB::EXP2_F64,
1519 Call_F80: RTLIB::EXP2_F80, Call_F128: RTLIB::EXP2_F128,
1520 Call_PPCF128: RTLIB::EXP2_PPCF128), Lo, Hi);
1521}
1522
1523void DAGTypeLegalizer::ExpandFloatRes_FEXP10(SDNode *N, SDValue &Lo,
1524 SDValue &Hi) {
1525 ExpandFloatRes_Unary(N,
1526 LC: GetFPLibCall(VT: N->getValueType(ResNo: 0), Call_F32: RTLIB::EXP10_F32,
1527 Call_F64: RTLIB::EXP10_F64, Call_F80: RTLIB::EXP10_F80,
1528 Call_F128: RTLIB::EXP10_F128, Call_PPCF128: RTLIB::EXP10_PPCF128),
1529 Lo, Hi);
1530}
1531
1532void DAGTypeLegalizer::ExpandFloatRes_FFLOOR(SDNode *N,
1533 SDValue &Lo, SDValue &Hi) {
1534 ExpandFloatRes_Unary(N, LC: GetFPLibCall(VT: N->getValueType(ResNo: 0),
1535 Call_F32: RTLIB::FLOOR_F32, Call_F64: RTLIB::FLOOR_F64,
1536 Call_F80: RTLIB::FLOOR_F80, Call_F128: RTLIB::FLOOR_F128,
1537 Call_PPCF128: RTLIB::FLOOR_PPCF128), Lo, Hi);
1538}
1539
1540void DAGTypeLegalizer::ExpandFloatRes_FLOG(SDNode *N,
1541 SDValue &Lo, SDValue &Hi) {
1542 ExpandFloatRes_Unary(N, LC: GetFPLibCall(VT: N->getValueType(ResNo: 0),
1543 Call_F32: RTLIB::LOG_F32, Call_F64: RTLIB::LOG_F64,
1544 Call_F80: RTLIB::LOG_F80, Call_F128: RTLIB::LOG_F128,
1545 Call_PPCF128: RTLIB::LOG_PPCF128), Lo, Hi);
1546}
1547
1548void DAGTypeLegalizer::ExpandFloatRes_FLOG2(SDNode *N,
1549 SDValue &Lo, SDValue &Hi) {
1550 ExpandFloatRes_Unary(N, LC: GetFPLibCall(VT: N->getValueType(ResNo: 0),
1551 Call_F32: RTLIB::LOG2_F32, Call_F64: RTLIB::LOG2_F64,
1552 Call_F80: RTLIB::LOG2_F80, Call_F128: RTLIB::LOG2_F128,
1553 Call_PPCF128: RTLIB::LOG2_PPCF128), Lo, Hi);
1554}
1555
1556void DAGTypeLegalizer::ExpandFloatRes_FLOG10(SDNode *N,
1557 SDValue &Lo, SDValue &Hi) {
1558 ExpandFloatRes_Unary(N, LC: GetFPLibCall(VT: N->getValueType(ResNo: 0),
1559 Call_F32: RTLIB::LOG10_F32, Call_F64: RTLIB::LOG10_F64,
1560 Call_F80: RTLIB::LOG10_F80, Call_F128: RTLIB::LOG10_F128,
1561 Call_PPCF128: RTLIB::LOG10_PPCF128), Lo, Hi);
1562}
1563
1564void DAGTypeLegalizer::ExpandFloatRes_FMA(SDNode *N, SDValue &Lo,
1565 SDValue &Hi) {
1566 bool IsStrict = N->isStrictFPOpcode();
1567 unsigned Offset = IsStrict ? 1 : 0;
1568 SDValue Ops[3] = { N->getOperand(Num: 0 + Offset), N->getOperand(Num: 1 + Offset),
1569 N->getOperand(Num: 2 + Offset) };
1570 SDValue Chain = IsStrict ? N->getOperand(Num: 0) : SDValue();
1571 TargetLowering::MakeLibCallOptions CallOptions;
1572 std::pair<SDValue, SDValue> Tmp = TLI.makeLibCall(DAG, LC: GetFPLibCall(VT: N->getValueType(ResNo: 0),
1573 Call_F32: RTLIB::FMA_F32,
1574 Call_F64: RTLIB::FMA_F64,
1575 Call_F80: RTLIB::FMA_F80,
1576 Call_F128: RTLIB::FMA_F128,
1577 Call_PPCF128: RTLIB::FMA_PPCF128),
1578 RetVT: N->getValueType(ResNo: 0), Ops, CallOptions,
1579 dl: SDLoc(N), Chain);
1580 if (IsStrict)
1581 ReplaceValueWith(From: SDValue(N, 1), To: Tmp.second);
1582 GetPairElements(Pair: Tmp.first, Lo, Hi);
1583}
1584
1585void DAGTypeLegalizer::ExpandFloatRes_FMUL(SDNode *N, SDValue &Lo,
1586 SDValue &Hi) {
1587 ExpandFloatRes_Binary(N, LC: GetFPLibCall(VT: N->getValueType(ResNo: 0),
1588 Call_F32: RTLIB::MUL_F32,
1589 Call_F64: RTLIB::MUL_F64,
1590 Call_F80: RTLIB::MUL_F80,
1591 Call_F128: RTLIB::MUL_F128,
1592 Call_PPCF128: RTLIB::MUL_PPCF128), Lo, Hi);
1593}
1594
1595void DAGTypeLegalizer::ExpandFloatRes_FNEARBYINT(SDNode *N,
1596 SDValue &Lo, SDValue &Hi) {
1597 ExpandFloatRes_Unary(N, LC: GetFPLibCall(VT: N->getValueType(ResNo: 0),
1598 Call_F32: RTLIB::NEARBYINT_F32,
1599 Call_F64: RTLIB::NEARBYINT_F64,
1600 Call_F80: RTLIB::NEARBYINT_F80,
1601 Call_F128: RTLIB::NEARBYINT_F128,
1602 Call_PPCF128: RTLIB::NEARBYINT_PPCF128), Lo, Hi);
1603}
1604
1605void DAGTypeLegalizer::ExpandFloatRes_FNEG(SDNode *N, SDValue &Lo,
1606 SDValue &Hi) {
1607 SDLoc dl(N);
1608 GetExpandedFloat(Op: N->getOperand(Num: 0), Lo, Hi);
1609 Lo = DAG.getNode(Opcode: ISD::FNEG, DL: dl, VT: Lo.getValueType(), Operand: Lo);
1610 Hi = DAG.getNode(Opcode: ISD::FNEG, DL: dl, VT: Hi.getValueType(), Operand: Hi);
1611}
1612
1613void DAGTypeLegalizer::ExpandFloatRes_FP_EXTEND(SDNode *N, SDValue &Lo,
1614 SDValue &Hi) {
1615 EVT NVT = TLI.getTypeToTransformTo(Context&: *DAG.getContext(), VT: N->getValueType(ResNo: 0));
1616 SDLoc dl(N);
1617 bool IsStrict = N->isStrictFPOpcode();
1618
1619 SDValue Chain;
1620 if (IsStrict) {
1621 // If the expanded type is the same as the input type, just bypass the node.
1622 if (NVT == N->getOperand(Num: 1).getValueType()) {
1623 Hi = N->getOperand(Num: 1);
1624 Chain = N->getOperand(Num: 0);
1625 } else {
1626 // Other we need to extend.
1627 Hi = DAG.getNode(ISD::STRICT_FP_EXTEND, dl, { NVT, MVT::Other },
1628 { N->getOperand(0), N->getOperand(1) });
1629 Chain = Hi.getValue(R: 1);
1630 }
1631 } else {
1632 Hi = DAG.getNode(Opcode: ISD::FP_EXTEND, DL: dl, VT: NVT, Operand: N->getOperand(Num: 0));
1633 }
1634
1635 Lo = DAG.getConstantFP(Val: APFloat(DAG.EVTToAPFloatSemantics(VT: NVT),
1636 APInt(NVT.getSizeInBits(), 0)), DL: dl, VT: NVT);
1637
1638 if (IsStrict)
1639 ReplaceValueWith(From: SDValue(N, 1), To: Chain);
1640}
1641
1642void DAGTypeLegalizer::ExpandFloatRes_FPOW(SDNode *N,
1643 SDValue &Lo, SDValue &Hi) {
1644 ExpandFloatRes_Binary(N, LC: GetFPLibCall(VT: N->getValueType(ResNo: 0),
1645 Call_F32: RTLIB::POW_F32, Call_F64: RTLIB::POW_F64,
1646 Call_F80: RTLIB::POW_F80, Call_F128: RTLIB::POW_F128,
1647 Call_PPCF128: RTLIB::POW_PPCF128), Lo, Hi);
1648}
1649
1650void DAGTypeLegalizer::ExpandFloatRes_FPOWI(SDNode *N,
1651 SDValue &Lo, SDValue &Hi) {
1652 ExpandFloatRes_Binary(N, LC: RTLIB::getPOWI(RetVT: N->getValueType(ResNo: 0)), Lo, Hi);
1653}
1654
1655void DAGTypeLegalizer::ExpandFloatRes_FLDEXP(SDNode *N, SDValue &Lo,
1656 SDValue &Hi) {
1657 ExpandFloatRes_Binary(N, LC: RTLIB::getLDEXP(RetVT: N->getValueType(ResNo: 0)), Lo, Hi);
1658}
1659
1660void DAGTypeLegalizer::ExpandFloatRes_FREEZE(SDNode *N,
1661 SDValue &Lo, SDValue &Hi) {
1662 assert(N->getValueType(0) == MVT::ppcf128 &&
1663 "Logic only correct for ppcf128!");
1664
1665 SDLoc dl(N);
1666 GetExpandedFloat(Op: N->getOperand(Num: 0), Lo, Hi);
1667 Lo = DAG.getNode(Opcode: ISD::FREEZE, DL: dl, VT: Lo.getValueType(), Operand: Lo);
1668 Hi = DAG.getNode(Opcode: ISD::FREEZE, DL: dl, VT: Hi.getValueType(), Operand: Hi);
1669}
1670
1671void DAGTypeLegalizer::ExpandFloatRes_FREM(SDNode *N,
1672 SDValue &Lo, SDValue &Hi) {
1673 ExpandFloatRes_Binary(N, LC: GetFPLibCall(VT: N->getValueType(ResNo: 0),
1674 Call_F32: RTLIB::REM_F32, Call_F64: RTLIB::REM_F64,
1675 Call_F80: RTLIB::REM_F80, Call_F128: RTLIB::REM_F128,
1676 Call_PPCF128: RTLIB::REM_PPCF128), Lo, Hi);
1677}
1678
1679void DAGTypeLegalizer::ExpandFloatRes_FRINT(SDNode *N,
1680 SDValue &Lo, SDValue &Hi) {
1681 ExpandFloatRes_Unary(N, LC: GetFPLibCall(VT: N->getValueType(ResNo: 0),
1682 Call_F32: RTLIB::RINT_F32, Call_F64: RTLIB::RINT_F64,
1683 Call_F80: RTLIB::RINT_F80, Call_F128: RTLIB::RINT_F128,
1684 Call_PPCF128: RTLIB::RINT_PPCF128), Lo, Hi);
1685}
1686
1687void DAGTypeLegalizer::ExpandFloatRes_FROUND(SDNode *N,
1688 SDValue &Lo, SDValue &Hi) {
1689 ExpandFloatRes_Unary(N, LC: GetFPLibCall(VT: N->getValueType(ResNo: 0),
1690 Call_F32: RTLIB::ROUND_F32,
1691 Call_F64: RTLIB::ROUND_F64,
1692 Call_F80: RTLIB::ROUND_F80,
1693 Call_F128: RTLIB::ROUND_F128,
1694 Call_PPCF128: RTLIB::ROUND_PPCF128), Lo, Hi);
1695}
1696
1697void DAGTypeLegalizer::ExpandFloatRes_FROUNDEVEN(SDNode *N,
1698 SDValue &Lo, SDValue &Hi) {
1699 ExpandFloatRes_Unary(N, LC: GetFPLibCall(VT: N->getValueType(ResNo: 0),
1700 Call_F32: RTLIB::ROUNDEVEN_F32,
1701 Call_F64: RTLIB::ROUNDEVEN_F64,
1702 Call_F80: RTLIB::ROUNDEVEN_F80,
1703 Call_F128: RTLIB::ROUNDEVEN_F128,
1704 Call_PPCF128: RTLIB::ROUNDEVEN_PPCF128), Lo, Hi);
1705}
1706
1707void DAGTypeLegalizer::ExpandFloatRes_FSIN(SDNode *N,
1708 SDValue &Lo, SDValue &Hi) {
1709 ExpandFloatRes_Unary(N, LC: GetFPLibCall(VT: N->getValueType(ResNo: 0),
1710 Call_F32: RTLIB::SIN_F32, Call_F64: RTLIB::SIN_F64,
1711 Call_F80: RTLIB::SIN_F80, Call_F128: RTLIB::SIN_F128,
1712 Call_PPCF128: RTLIB::SIN_PPCF128), Lo, Hi);
1713}
1714
1715void DAGTypeLegalizer::ExpandFloatRes_FSQRT(SDNode *N,
1716 SDValue &Lo, SDValue &Hi) {
1717 ExpandFloatRes_Unary(N, LC: GetFPLibCall(VT: N->getValueType(ResNo: 0),
1718 Call_F32: RTLIB::SQRT_F32, Call_F64: RTLIB::SQRT_F64,
1719 Call_F80: RTLIB::SQRT_F80, Call_F128: RTLIB::SQRT_F128,
1720 Call_PPCF128: RTLIB::SQRT_PPCF128), Lo, Hi);
1721}
1722
1723void DAGTypeLegalizer::ExpandFloatRes_FSUB(SDNode *N, SDValue &Lo,
1724 SDValue &Hi) {
1725 ExpandFloatRes_Binary(N, LC: GetFPLibCall(VT: N->getValueType(ResNo: 0),
1726 Call_F32: RTLIB::SUB_F32,
1727 Call_F64: RTLIB::SUB_F64,
1728 Call_F80: RTLIB::SUB_F80,
1729 Call_F128: RTLIB::SUB_F128,
1730 Call_PPCF128: RTLIB::SUB_PPCF128), Lo, Hi);
1731}
1732
1733void DAGTypeLegalizer::ExpandFloatRes_FTRUNC(SDNode *N,
1734 SDValue &Lo, SDValue &Hi) {
1735 ExpandFloatRes_Unary(N, LC: GetFPLibCall(VT: N->getValueType(ResNo: 0),
1736 Call_F32: RTLIB::TRUNC_F32, Call_F64: RTLIB::TRUNC_F64,
1737 Call_F80: RTLIB::TRUNC_F80, Call_F128: RTLIB::TRUNC_F128,
1738 Call_PPCF128: RTLIB::TRUNC_PPCF128), Lo, Hi);
1739}
1740
1741void DAGTypeLegalizer::ExpandFloatRes_LOAD(SDNode *N, SDValue &Lo,
1742 SDValue &Hi) {
1743 if (ISD::isNormalLoad(N)) {
1744 ExpandRes_NormalLoad(N, Lo, Hi);
1745 return;
1746 }
1747
1748 assert(ISD::isUNINDEXEDLoad(N) && "Indexed load during type legalization!");
1749 LoadSDNode *LD = cast<LoadSDNode>(Val: N);
1750 SDValue Chain = LD->getChain();
1751 SDValue Ptr = LD->getBasePtr();
1752 SDLoc dl(N);
1753
1754 EVT NVT = TLI.getTypeToTransformTo(Context&: *DAG.getContext(), VT: LD->getValueType(ResNo: 0));
1755 assert(NVT.isByteSized() && "Expanded type not byte sized!");
1756 assert(LD->getMemoryVT().bitsLE(NVT) && "Float type not round?");
1757
1758 Hi = DAG.getExtLoad(ExtType: LD->getExtensionType(), dl, VT: NVT, Chain, Ptr,
1759 MemVT: LD->getMemoryVT(), MMO: LD->getMemOperand());
1760
1761 // Remember the chain.
1762 Chain = Hi.getValue(R: 1);
1763
1764 // The low part is zero.
1765 Lo = DAG.getConstantFP(Val: APFloat(DAG.EVTToAPFloatSemantics(VT: NVT),
1766 APInt(NVT.getSizeInBits(), 0)), DL: dl, VT: NVT);
1767
1768 // Modified the chain - switch anything that used the old chain to use the
1769 // new one.
1770 ReplaceValueWith(From: SDValue(LD, 1), To: Chain);
1771}
1772
1773void DAGTypeLegalizer::ExpandFloatRes_XINT_TO_FP(SDNode *N, SDValue &Lo,
1774 SDValue &Hi) {
1775 assert(N->getValueType(0) == MVT::ppcf128 && "Unsupported XINT_TO_FP!");
1776 EVT VT = N->getValueType(ResNo: 0);
1777 EVT NVT = TLI.getTypeToTransformTo(Context&: *DAG.getContext(), VT);
1778 bool Strict = N->isStrictFPOpcode();
1779 SDValue Src = N->getOperand(Num: Strict ? 1 : 0);
1780 EVT SrcVT = Src.getValueType();
1781 bool isSigned = N->getOpcode() == ISD::SINT_TO_FP ||
1782 N->getOpcode() == ISD::STRICT_SINT_TO_FP;
1783 SDLoc dl(N);
1784 SDValue Chain = Strict ? N->getOperand(Num: 0) : DAG.getEntryNode();
1785
1786 // TODO: Any other flags to propagate?
1787 SDNodeFlags Flags;
1788 Flags.setNoFPExcept(N->getFlags().hasNoFPExcept());
1789
1790 // First do an SINT_TO_FP, whether the original was signed or unsigned.
1791 // When promoting partial word types to i32 we must honor the signedness,
1792 // though.
1793 if (SrcVT.bitsLE(MVT::VT: i32)) {
1794 // The integer can be represented exactly in an f64.
1795 Lo = DAG.getConstantFP(Val: APFloat(DAG.EVTToAPFloatSemantics(VT: NVT),
1796 APInt(NVT.getSizeInBits(), 0)), DL: dl, VT: NVT);
1797 if (Strict) {
1798 Hi = DAG.getNode(N->getOpcode(), dl, DAG.getVTList(NVT, MVT::Other),
1799 {Chain, Src}, Flags);
1800 Chain = Hi.getValue(R: 1);
1801 } else
1802 Hi = DAG.getNode(Opcode: N->getOpcode(), DL: dl, VT: NVT, Operand: Src);
1803 } else {
1804 RTLIB::Libcall LC = RTLIB::UNKNOWN_LIBCALL;
1805 if (SrcVT.bitsLE(MVT::VT: i64)) {
1806 Src = DAG.getNode(isSigned ? ISD::SIGN_EXTEND : ISD::ZERO_EXTEND, dl,
1807 MVT::i64, Src);
1808 LC = RTLIB::SINTTOFP_I64_PPCF128;
1809 } else if (SrcVT.bitsLE(MVT::VT: i128)) {
1810 Src = DAG.getNode(ISD::SIGN_EXTEND, dl, MVT::i128, Src);
1811 LC = RTLIB::SINTTOFP_I128_PPCF128;
1812 }
1813 assert(LC != RTLIB::UNKNOWN_LIBCALL && "Unsupported XINT_TO_FP!");
1814
1815 TargetLowering::MakeLibCallOptions CallOptions;
1816 CallOptions.setSExt(true);
1817 std::pair<SDValue, SDValue> Tmp =
1818 TLI.makeLibCall(DAG, LC, RetVT: VT, Ops: Src, CallOptions, dl, Chain);
1819 if (Strict)
1820 Chain = Tmp.second;
1821 GetPairElements(Pair: Tmp.first, Lo, Hi);
1822 }
1823
1824 // No need to complement for unsigned 32-bit integers
1825 if (isSigned || SrcVT.bitsLE(MVT::VT: i32)) {
1826 if (Strict)
1827 ReplaceValueWith(From: SDValue(N, 1), To: Chain);
1828
1829 return;
1830 }
1831
1832 // Unsigned - fix up the SINT_TO_FP value just calculated.
1833 // FIXME: For unsigned i128 to ppc_fp128 conversion, we need to carefully
1834 // keep semantics correctness if the integer is not exactly representable
1835 // here. See ExpandLegalINT_TO_FP.
1836 Hi = DAG.getNode(Opcode: ISD::BUILD_PAIR, DL: dl, VT, N1: Lo, N2: Hi);
1837 SrcVT = Src.getValueType();
1838
1839 // x>=0 ? (ppcf128)(iN)x : (ppcf128)(iN)x + 2^N; N=32,64,128.
1840 static const uint64_t TwoE32[] = { 0x41f0000000000000LL, 0 };
1841 static const uint64_t TwoE64[] = { 0x43f0000000000000LL, 0 };
1842 static const uint64_t TwoE128[] = { 0x47f0000000000000LL, 0 };
1843 ArrayRef<uint64_t> Parts;
1844
1845 switch (SrcVT.getSimpleVT().SimpleTy) {
1846 default:
1847 llvm_unreachable("Unsupported UINT_TO_FP!");
1848 case MVT::i32:
1849 Parts = TwoE32;
1850 break;
1851 case MVT::i64:
1852 Parts = TwoE64;
1853 break;
1854 case MVT::i128:
1855 Parts = TwoE128;
1856 break;
1857 }
1858
1859 // TODO: Are there other fast-math-flags to propagate to this FADD?
1860 SDValue NewLo = DAG.getConstantFP(
1861 APFloat(APFloat::PPCDoubleDouble(), APInt(128, Parts)), dl, MVT::ppcf128);
1862 if (Strict) {
1863 Lo = DAG.getNode(ISD::STRICT_FADD, dl, DAG.getVTList(VT, MVT::Other),
1864 {Chain, Hi, NewLo}, Flags);
1865 Chain = Lo.getValue(R: 1);
1866 ReplaceValueWith(From: SDValue(N, 1), To: Chain);
1867 } else
1868 Lo = DAG.getNode(Opcode: ISD::FADD, DL: dl, VT, N1: Hi, N2: NewLo);
1869 Lo = DAG.getSelectCC(DL: dl, LHS: Src, RHS: DAG.getConstant(Val: 0, DL: dl, VT: SrcVT),
1870 True: Lo, False: Hi, Cond: ISD::SETLT);
1871 GetPairElements(Pair: Lo, Lo, Hi);
1872}
1873
1874
1875//===----------------------------------------------------------------------===//
1876// Float Operand Expansion
1877//===----------------------------------------------------------------------===//
1878
1879/// ExpandFloatOperand - This method is called when the specified operand of the
1880/// specified node is found to need expansion. At this point, all of the result
1881/// types of the node are known to be legal, but other operands of the node may
1882/// need promotion or expansion as well as the specified one.
1883bool DAGTypeLegalizer::ExpandFloatOperand(SDNode *N, unsigned OpNo) {
1884 LLVM_DEBUG(dbgs() << "Expand float operand: "; N->dump(&DAG));
1885 SDValue Res = SDValue();
1886
1887 // See if the target wants to custom expand this node.
1888 if (CustomLowerNode(N, VT: N->getOperand(Num: OpNo).getValueType(), LegalizeResult: false))
1889 return false;
1890
1891 switch (N->getOpcode()) {
1892 default:
1893#ifndef NDEBUG
1894 dbgs() << "ExpandFloatOperand Op #" << OpNo << ": ";
1895 N->dump(G: &DAG); dbgs() << "\n";
1896#endif
1897 report_fatal_error(reason: "Do not know how to expand this operator's operand!");
1898
1899 case ISD::BITCAST: Res = ExpandOp_BITCAST(N); break;
1900 case ISD::BUILD_VECTOR: Res = ExpandOp_BUILD_VECTOR(N); break;
1901 case ISD::EXTRACT_ELEMENT: Res = ExpandOp_EXTRACT_ELEMENT(N); break;
1902
1903 case ISD::BR_CC: Res = ExpandFloatOp_BR_CC(N); break;
1904 case ISD::FCOPYSIGN: Res = ExpandFloatOp_FCOPYSIGN(N); break;
1905 case ISD::STRICT_FP_ROUND:
1906 case ISD::FP_ROUND: Res = ExpandFloatOp_FP_ROUND(N); break;
1907 case ISD::STRICT_FP_TO_SINT:
1908 case ISD::STRICT_FP_TO_UINT:
1909 case ISD::FP_TO_SINT:
1910 case ISD::FP_TO_UINT: Res = ExpandFloatOp_FP_TO_XINT(N); break;
1911 case ISD::LROUND: Res = ExpandFloatOp_LROUND(N); break;
1912 case ISD::LLROUND: Res = ExpandFloatOp_LLROUND(N); break;
1913 case ISD::LRINT: Res = ExpandFloatOp_LRINT(N); break;
1914 case ISD::LLRINT: Res = ExpandFloatOp_LLRINT(N); break;
1915 case ISD::SELECT_CC: Res = ExpandFloatOp_SELECT_CC(N); break;
1916 case ISD::STRICT_FSETCC:
1917 case ISD::STRICT_FSETCCS:
1918 case ISD::SETCC: Res = ExpandFloatOp_SETCC(N); break;
1919 case ISD::STORE: Res = ExpandFloatOp_STORE(N: cast<StoreSDNode>(Val: N),
1920 OpNo); break;
1921 }
1922
1923 // If the result is null, the sub-method took care of registering results etc.
1924 if (!Res.getNode()) return false;
1925
1926 // If the result is N, the sub-method updated N in place. Tell the legalizer
1927 // core about this.
1928 if (Res.getNode() == N)
1929 return true;
1930
1931 assert(Res.getValueType() == N->getValueType(0) && N->getNumValues() == 1 &&
1932 "Invalid operand expansion");
1933
1934 ReplaceValueWith(From: SDValue(N, 0), To: Res);
1935 return false;
1936}
1937
1938/// FloatExpandSetCCOperands - Expand the operands of a comparison. This code
1939/// is shared among BR_CC, SELECT_CC, and SETCC handlers.
1940void DAGTypeLegalizer::FloatExpandSetCCOperands(SDValue &NewLHS,
1941 SDValue &NewRHS,
1942 ISD::CondCode &CCCode,
1943 const SDLoc &dl, SDValue &Chain,
1944 bool IsSignaling) {
1945 SDValue LHSLo, LHSHi, RHSLo, RHSHi;
1946 GetExpandedFloat(Op: NewLHS, Lo&: LHSLo, Hi&: LHSHi);
1947 GetExpandedFloat(Op: NewRHS, Lo&: RHSLo, Hi&: RHSHi);
1948
1949 assert(NewLHS.getValueType() == MVT::ppcf128 && "Unsupported setcc type!");
1950
1951 // FIXME: This generated code sucks. We want to generate
1952 // FCMPU crN, hi1, hi2
1953 // BNE crN, L:
1954 // FCMPU crN, lo1, lo2
1955 // The following can be improved, but not that much.
1956 SDValue Tmp1, Tmp2, Tmp3, OutputChain;
1957 Tmp1 = DAG.getSetCC(DL: dl, VT: getSetCCResultType(VT: LHSHi.getValueType()), LHS: LHSHi,
1958 RHS: RHSHi, Cond: ISD::SETOEQ, Chain, IsSignaling);
1959 OutputChain = Tmp1->getNumValues() > 1 ? Tmp1.getValue(R: 1) : SDValue();
1960 Tmp2 = DAG.getSetCC(DL: dl, VT: getSetCCResultType(VT: LHSLo.getValueType()), LHS: LHSLo,
1961 RHS: RHSLo, Cond: CCCode, Chain: OutputChain, IsSignaling);
1962 OutputChain = Tmp2->getNumValues() > 1 ? Tmp2.getValue(R: 1) : SDValue();
1963 Tmp3 = DAG.getNode(Opcode: ISD::AND, DL: dl, VT: Tmp1.getValueType(), N1: Tmp1, N2: Tmp2);
1964 Tmp1 =
1965 DAG.getSetCC(DL: dl, VT: getSetCCResultType(VT: LHSHi.getValueType()), LHS: LHSHi, RHS: RHSHi,
1966 Cond: ISD::SETUNE, Chain: OutputChain, IsSignaling);
1967 OutputChain = Tmp1->getNumValues() > 1 ? Tmp1.getValue(R: 1) : SDValue();
1968 Tmp2 = DAG.getSetCC(DL: dl, VT: getSetCCResultType(VT: LHSHi.getValueType()), LHS: LHSHi,
1969 RHS: RHSHi, Cond: CCCode, Chain: OutputChain, IsSignaling);
1970 OutputChain = Tmp2->getNumValues() > 1 ? Tmp2.getValue(R: 1) : SDValue();
1971 Tmp1 = DAG.getNode(Opcode: ISD::AND, DL: dl, VT: Tmp1.getValueType(), N1: Tmp1, N2: Tmp2);
1972 NewLHS = DAG.getNode(Opcode: ISD::OR, DL: dl, VT: Tmp1.getValueType(), N1: Tmp1, N2: Tmp3);
1973 NewRHS = SDValue(); // LHS is the result, not a compare.
1974 Chain = OutputChain;
1975}
1976
1977SDValue DAGTypeLegalizer::ExpandFloatOp_BR_CC(SDNode *N) {
1978 SDValue NewLHS = N->getOperand(Num: 2), NewRHS = N->getOperand(Num: 3);
1979 ISD::CondCode CCCode = cast<CondCodeSDNode>(Val: N->getOperand(Num: 1))->get();
1980 SDValue Chain;
1981 FloatExpandSetCCOperands(NewLHS, NewRHS, CCCode, dl: SDLoc(N), Chain);
1982
1983 // If ExpandSetCCOperands returned a scalar, we need to compare the result
1984 // against zero to select between true and false values.
1985 if (!NewRHS.getNode()) {
1986 NewRHS = DAG.getConstant(Val: 0, DL: SDLoc(N), VT: NewLHS.getValueType());
1987 CCCode = ISD::SETNE;
1988 }
1989
1990 // Update N to have the operands specified.
1991 return SDValue(DAG.UpdateNodeOperands(N, Op1: N->getOperand(Num: 0),
1992 Op2: DAG.getCondCode(Cond: CCCode), Op3: NewLHS, Op4: NewRHS,
1993 Op5: N->getOperand(Num: 4)), 0);
1994}
1995
1996SDValue DAGTypeLegalizer::ExpandFloatOp_FCOPYSIGN(SDNode *N) {
1997 assert(N->getOperand(1).getValueType() == MVT::ppcf128 &&
1998 "Logic only correct for ppcf128!");
1999 SDValue Lo, Hi;
2000 GetExpandedFloat(Op: N->getOperand(Num: 1), Lo, Hi);
2001 // The ppcf128 value is providing only the sign; take it from the
2002 // higher-order double (which must have the larger magnitude).
2003 return DAG.getNode(Opcode: ISD::FCOPYSIGN, DL: SDLoc(N),
2004 VT: N->getValueType(ResNo: 0), N1: N->getOperand(Num: 0), N2: Hi);
2005}
2006
2007SDValue DAGTypeLegalizer::ExpandFloatOp_FP_ROUND(SDNode *N) {
2008 bool IsStrict = N->isStrictFPOpcode();
2009 assert(N->getOperand(IsStrict ? 1 : 0).getValueType() == MVT::ppcf128 &&
2010 "Logic only correct for ppcf128!");
2011 SDValue Lo, Hi;
2012 GetExpandedFloat(Op: N->getOperand(Num: IsStrict ? 1 : 0), Lo, Hi);
2013
2014 if (!IsStrict)
2015 // Round it the rest of the way (e.g. to f32) if needed.
2016 return DAG.getNode(Opcode: ISD::FP_ROUND, DL: SDLoc(N),
2017 VT: N->getValueType(ResNo: 0), N1: Hi, N2: N->getOperand(Num: 1));
2018
2019 // Eliminate the node if the input float type is the same as the output float
2020 // type.
2021 if (Hi.getValueType() == N->getValueType(ResNo: 0)) {
2022 // Connect the output chain to the input chain, unlinking the node.
2023 ReplaceValueWith(From: SDValue(N, 1), To: N->getOperand(Num: 0));
2024 ReplaceValueWith(From: SDValue(N, 0), To: Hi);
2025 return SDValue();
2026 }
2027
2028 SDValue Expansion = DAG.getNode(ISD::STRICT_FP_ROUND, SDLoc(N),
2029 {N->getValueType(0), MVT::Other},
2030 {N->getOperand(0), Hi, N->getOperand(2)});
2031 ReplaceValueWith(From: SDValue(N, 1), To: Expansion.getValue(R: 1));
2032 ReplaceValueWith(From: SDValue(N, 0), To: Expansion);
2033 return SDValue();
2034}
2035
2036SDValue DAGTypeLegalizer::ExpandFloatOp_FP_TO_XINT(SDNode *N) {
2037 EVT RVT = N->getValueType(ResNo: 0);
2038 SDLoc dl(N);
2039
2040 bool IsStrict = N->isStrictFPOpcode();
2041 bool Signed = N->getOpcode() == ISD::FP_TO_SINT ||
2042 N->getOpcode() == ISD::STRICT_FP_TO_SINT;
2043 SDValue Op = N->getOperand(Num: IsStrict ? 1 : 0);
2044 SDValue Chain = IsStrict ? N->getOperand(Num: 0) : SDValue();
2045
2046 EVT NVT;
2047 RTLIB::Libcall LC = findFPToIntLibcall(SrcVT: Op.getValueType(), RetVT: RVT, Promoted&: NVT, Signed);
2048 assert(LC != RTLIB::UNKNOWN_LIBCALL && NVT.isSimple() &&
2049 "Unsupported FP_TO_XINT!");
2050 TargetLowering::MakeLibCallOptions CallOptions;
2051 std::pair<SDValue, SDValue> Tmp =
2052 TLI.makeLibCall(DAG, LC, RetVT: NVT, Ops: Op, CallOptions, dl, Chain);
2053 if (!IsStrict)
2054 return Tmp.first;
2055
2056 ReplaceValueWith(From: SDValue(N, 1), To: Tmp.second);
2057 ReplaceValueWith(From: SDValue(N, 0), To: Tmp.first);
2058 return SDValue();
2059}
2060
2061SDValue DAGTypeLegalizer::ExpandFloatOp_SELECT_CC(SDNode *N) {
2062 SDValue NewLHS = N->getOperand(Num: 0), NewRHS = N->getOperand(Num: 1);
2063 ISD::CondCode CCCode = cast<CondCodeSDNode>(Val: N->getOperand(Num: 4))->get();
2064 SDValue Chain;
2065 FloatExpandSetCCOperands(NewLHS, NewRHS, CCCode, dl: SDLoc(N), Chain);
2066
2067 // If ExpandSetCCOperands returned a scalar, we need to compare the result
2068 // against zero to select between true and false values.
2069 if (!NewRHS.getNode()) {
2070 NewRHS = DAG.getConstant(Val: 0, DL: SDLoc(N), VT: NewLHS.getValueType());
2071 CCCode = ISD::SETNE;
2072 }
2073
2074 // Update N to have the operands specified.
2075 return SDValue(DAG.UpdateNodeOperands(N, Op1: NewLHS, Op2: NewRHS,
2076 Op3: N->getOperand(Num: 2), Op4: N->getOperand(Num: 3),
2077 Op5: DAG.getCondCode(Cond: CCCode)), 0);
2078}
2079
2080SDValue DAGTypeLegalizer::ExpandFloatOp_SETCC(SDNode *N) {
2081 bool IsStrict = N->isStrictFPOpcode();
2082 SDValue NewLHS = N->getOperand(Num: IsStrict ? 1 : 0);
2083 SDValue NewRHS = N->getOperand(Num: IsStrict ? 2 : 1);
2084 SDValue Chain = IsStrict ? N->getOperand(Num: 0) : SDValue();
2085 ISD::CondCode CCCode =
2086 cast<CondCodeSDNode>(Val: N->getOperand(Num: IsStrict ? 3 : 2))->get();
2087 FloatExpandSetCCOperands(NewLHS, NewRHS, CCCode, dl: SDLoc(N), Chain,
2088 IsSignaling: N->getOpcode() == ISD::STRICT_FSETCCS);
2089
2090 // FloatExpandSetCCOperands always returned a scalar.
2091 assert(!NewRHS.getNode() && "Expect to return scalar");
2092 assert(NewLHS.getValueType() == N->getValueType(0) &&
2093 "Unexpected setcc expansion!");
2094 if (Chain) {
2095 ReplaceValueWith(From: SDValue(N, 0), To: NewLHS);
2096 ReplaceValueWith(From: SDValue(N, 1), To: Chain);
2097 return SDValue();
2098 }
2099 return NewLHS;
2100}
2101
2102SDValue DAGTypeLegalizer::ExpandFloatOp_STORE(SDNode *N, unsigned OpNo) {
2103 if (ISD::isNormalStore(N))
2104 return ExpandOp_NormalStore(N, OpNo);
2105
2106 assert(ISD::isUNINDEXEDStore(N) && "Indexed store during type legalization!");
2107 assert(OpNo == 1 && "Can only expand the stored value so far");
2108 StoreSDNode *ST = cast<StoreSDNode>(Val: N);
2109
2110 SDValue Chain = ST->getChain();
2111 SDValue Ptr = ST->getBasePtr();
2112
2113 EVT NVT = TLI.getTypeToTransformTo(Context&: *DAG.getContext(),
2114 VT: ST->getValue().getValueType());
2115 assert(NVT.isByteSized() && "Expanded type not byte sized!");
2116 assert(ST->getMemoryVT().bitsLE(NVT) && "Float type not round?");
2117 (void)NVT;
2118
2119 SDValue Lo, Hi;
2120 GetExpandedOp(Op: ST->getValue(), Lo, Hi);
2121
2122 return DAG.getTruncStore(Chain, dl: SDLoc(N), Val: Hi, Ptr,
2123 SVT: ST->getMemoryVT(), MMO: ST->getMemOperand());
2124}
2125
2126SDValue DAGTypeLegalizer::ExpandFloatOp_LROUND(SDNode *N) {
2127 EVT RVT = N->getValueType(ResNo: 0);
2128 EVT RetVT = N->getOperand(Num: 0).getValueType();
2129 TargetLowering::MakeLibCallOptions CallOptions;
2130 return TLI.makeLibCall(DAG, LC: GetFPLibCall(VT: RetVT,
2131 Call_F32: RTLIB::LROUND_F32,
2132 Call_F64: RTLIB::LROUND_F64,
2133 Call_F80: RTLIB::LROUND_F80,
2134 Call_F128: RTLIB::LROUND_F128,
2135 Call_PPCF128: RTLIB::LROUND_PPCF128),
2136 RetVT: RVT, Ops: N->getOperand(Num: 0), CallOptions, dl: SDLoc(N)).first;
2137}
2138
2139SDValue DAGTypeLegalizer::ExpandFloatOp_LLROUND(SDNode *N) {
2140 EVT RVT = N->getValueType(ResNo: 0);
2141 EVT RetVT = N->getOperand(Num: 0).getValueType();
2142 TargetLowering::MakeLibCallOptions CallOptions;
2143 return TLI.makeLibCall(DAG, LC: GetFPLibCall(VT: RetVT,
2144 Call_F32: RTLIB::LLROUND_F32,
2145 Call_F64: RTLIB::LLROUND_F64,
2146 Call_F80: RTLIB::LLROUND_F80,
2147 Call_F128: RTLIB::LLROUND_F128,
2148 Call_PPCF128: RTLIB::LLROUND_PPCF128),
2149 RetVT: RVT, Ops: N->getOperand(Num: 0), CallOptions, dl: SDLoc(N)).first;
2150}
2151
2152SDValue DAGTypeLegalizer::ExpandFloatOp_LRINT(SDNode *N) {
2153 EVT RVT = N->getValueType(ResNo: 0);
2154 EVT RetVT = N->getOperand(Num: 0).getValueType();
2155 TargetLowering::MakeLibCallOptions CallOptions;
2156 return TLI.makeLibCall(DAG, LC: GetFPLibCall(VT: RetVT,
2157 Call_F32: RTLIB::LRINT_F32,
2158 Call_F64: RTLIB::LRINT_F64,
2159 Call_F80: RTLIB::LRINT_F80,
2160 Call_F128: RTLIB::LRINT_F128,
2161 Call_PPCF128: RTLIB::LRINT_PPCF128),
2162 RetVT: RVT, Ops: N->getOperand(Num: 0), CallOptions, dl: SDLoc(N)).first;
2163}
2164
2165SDValue DAGTypeLegalizer::ExpandFloatOp_LLRINT(SDNode *N) {
2166 EVT RVT = N->getValueType(ResNo: 0);
2167 EVT RetVT = N->getOperand(Num: 0).getValueType();
2168 TargetLowering::MakeLibCallOptions CallOptions;
2169 return TLI.makeLibCall(DAG, LC: GetFPLibCall(VT: RetVT,
2170 Call_F32: RTLIB::LLRINT_F32,
2171 Call_F64: RTLIB::LLRINT_F64,
2172 Call_F80: RTLIB::LLRINT_F80,
2173 Call_F128: RTLIB::LLRINT_F128,
2174 Call_PPCF128: RTLIB::LLRINT_PPCF128),
2175 RetVT: RVT, Ops: N->getOperand(Num: 0), CallOptions, dl: SDLoc(N)).first;
2176}
2177
2178//===----------------------------------------------------------------------===//
2179// Float Operand Promotion
2180//===----------------------------------------------------------------------===//
2181//
2182
2183static ISD::NodeType GetPromotionOpcode(EVT OpVT, EVT RetVT) {
2184 if (OpVT == MVT::f16) {
2185 return ISD::FP16_TO_FP;
2186 } else if (RetVT == MVT::f16) {
2187 return ISD::FP_TO_FP16;
2188 } else if (OpVT == MVT::bf16) {
2189 return ISD::BF16_TO_FP;
2190 } else if (RetVT == MVT::bf16) {
2191 return ISD::FP_TO_BF16;
2192 }
2193
2194 report_fatal_error(reason: "Attempt at an invalid promotion-related conversion");
2195}
2196
2197static ISD::NodeType GetPromotionOpcodeStrict(EVT OpVT, EVT RetVT) {
2198 if (OpVT == MVT::f16)
2199 return ISD::STRICT_FP16_TO_FP;
2200
2201 if (RetVT == MVT::f16)
2202 return ISD::STRICT_FP_TO_FP16;
2203
2204 if (OpVT == MVT::bf16)
2205 return ISD::STRICT_BF16_TO_FP;
2206
2207 if (RetVT == MVT::bf16)
2208 return ISD::STRICT_FP_TO_BF16;
2209
2210 report_fatal_error(reason: "Attempt at an invalid promotion-related conversion");
2211}
2212
2213bool DAGTypeLegalizer::PromoteFloatOperand(SDNode *N, unsigned OpNo) {
2214 LLVM_DEBUG(dbgs() << "Promote float operand " << OpNo << ": "; N->dump(&DAG));
2215 SDValue R = SDValue();
2216
2217 if (CustomLowerNode(N, VT: N->getOperand(Num: OpNo).getValueType(), LegalizeResult: false)) {
2218 LLVM_DEBUG(dbgs() << "Node has been custom lowered, done\n");
2219 return false;
2220 }
2221
2222 // Nodes that use a promotion-requiring floating point operand, but doesn't
2223 // produce a promotion-requiring floating point result, need to be legalized
2224 // to use the promoted float operand. Nodes that produce at least one
2225 // promotion-requiring floating point result have their operands legalized as
2226 // a part of PromoteFloatResult.
2227 // clang-format off
2228 switch (N->getOpcode()) {
2229 default:
2230 #ifndef NDEBUG
2231 dbgs() << "PromoteFloatOperand Op #" << OpNo << ": ";
2232 N->dump(G: &DAG); dbgs() << "\n";
2233 #endif
2234 report_fatal_error(reason: "Do not know how to promote this operator's operand!");
2235
2236 case ISD::BITCAST: R = PromoteFloatOp_BITCAST(N, OpNo); break;
2237 case ISD::FCOPYSIGN: R = PromoteFloatOp_FCOPYSIGN(N, OpNo); break;
2238 case ISD::FP_TO_SINT:
2239 case ISD::FP_TO_UINT:
2240 case ISD::LRINT:
2241 case ISD::LLRINT: R = PromoteFloatOp_UnaryOp(N, OpNo); break;
2242 case ISD::FP_TO_SINT_SAT:
2243 case ISD::FP_TO_UINT_SAT:
2244 R = PromoteFloatOp_FP_TO_XINT_SAT(N, OpNo); break;
2245 case ISD::FP_EXTEND: R = PromoteFloatOp_FP_EXTEND(N, OpNo); break;
2246 case ISD::STRICT_FP_EXTEND:
2247 R = PromoteFloatOp_STRICT_FP_EXTEND(N, OpNo);
2248 break;
2249 case ISD::SELECT_CC: R = PromoteFloatOp_SELECT_CC(N, OpNo); break;
2250 case ISD::SETCC: R = PromoteFloatOp_SETCC(N, OpNo); break;
2251 case ISD::STORE: R = PromoteFloatOp_STORE(N, OpNo); break;
2252 }
2253 // clang-format on
2254
2255 if (R.getNode())
2256 ReplaceValueWith(From: SDValue(N, 0), To: R);
2257 return false;
2258}
2259
2260SDValue DAGTypeLegalizer::PromoteFloatOp_BITCAST(SDNode *N, unsigned OpNo) {
2261 SDValue Op = N->getOperand(Num: 0);
2262 EVT OpVT = Op->getValueType(ResNo: 0);
2263
2264 SDValue Promoted = GetPromotedFloat(Op: N->getOperand(Num: 0));
2265 EVT PromotedVT = Promoted->getValueType(ResNo: 0);
2266
2267 // Convert the promoted float value to the desired IVT.
2268 EVT IVT = EVT::getIntegerVT(Context&: *DAG.getContext(), BitWidth: OpVT.getSizeInBits());
2269 SDValue Convert = DAG.getNode(Opcode: GetPromotionOpcode(OpVT: PromotedVT, RetVT: OpVT), DL: SDLoc(N),
2270 VT: IVT, Operand: Promoted);
2271 // The final result type might not be an scalar so we need a bitcast. The
2272 // bitcast will be further legalized if needed.
2273 return DAG.getBitcast(VT: N->getValueType(ResNo: 0), V: Convert);
2274}
2275
2276// Promote Operand 1 of FCOPYSIGN. Operand 0 ought to be handled by
2277// PromoteFloatRes_FCOPYSIGN.
2278SDValue DAGTypeLegalizer::PromoteFloatOp_FCOPYSIGN(SDNode *N, unsigned OpNo) {
2279 assert (OpNo == 1 && "Only Operand 1 must need promotion here");
2280 SDValue Op1 = GetPromotedFloat(Op: N->getOperand(Num: 1));
2281
2282 return DAG.getNode(Opcode: N->getOpcode(), DL: SDLoc(N), VT: N->getValueType(ResNo: 0),
2283 N1: N->getOperand(Num: 0), N2: Op1);
2284}
2285
2286// Convert the promoted float value to the desired integer type
2287SDValue DAGTypeLegalizer::PromoteFloatOp_UnaryOp(SDNode *N, unsigned OpNo) {
2288 SDValue Op = GetPromotedFloat(Op: N->getOperand(Num: 0));
2289 return DAG.getNode(Opcode: N->getOpcode(), DL: SDLoc(N), VT: N->getValueType(ResNo: 0), Operand: Op);
2290}
2291
2292SDValue DAGTypeLegalizer::PromoteFloatOp_FP_TO_XINT_SAT(SDNode *N,
2293 unsigned OpNo) {
2294 SDValue Op = GetPromotedFloat(Op: N->getOperand(Num: 0));
2295 return DAG.getNode(Opcode: N->getOpcode(), DL: SDLoc(N), VT: N->getValueType(ResNo: 0), N1: Op,
2296 N2: N->getOperand(Num: 1));
2297}
2298
2299SDValue DAGTypeLegalizer::PromoteFloatOp_FP_EXTEND(SDNode *N, unsigned OpNo) {
2300 SDValue Op = GetPromotedFloat(Op: N->getOperand(Num: 0));
2301 EVT VT = N->getValueType(ResNo: 0);
2302
2303 // Desired VT is same as promoted type. Use promoted float directly.
2304 if (VT == Op->getValueType(ResNo: 0))
2305 return Op;
2306
2307 // Else, extend the promoted float value to the desired VT.
2308 return DAG.getNode(Opcode: ISD::FP_EXTEND, DL: SDLoc(N), VT, Operand: Op);
2309}
2310
2311SDValue DAGTypeLegalizer::PromoteFloatOp_STRICT_FP_EXTEND(SDNode *N,
2312 unsigned OpNo) {
2313 assert(OpNo == 1 && "Promoting unpromotable operand");
2314
2315 SDValue Op = GetPromotedFloat(Op: N->getOperand(Num: 1));
2316 EVT VT = N->getValueType(ResNo: 0);
2317
2318 // Desired VT is same as promoted type. Use promoted float directly.
2319 if (VT == Op->getValueType(ResNo: 0)) {
2320 ReplaceValueWith(From: SDValue(N, 1), To: N->getOperand(Num: 0));
2321 return Op;
2322 }
2323
2324 // Else, extend the promoted float value to the desired VT.
2325 SDValue Res = DAG.getNode(Opcode: ISD::STRICT_FP_EXTEND, DL: SDLoc(N), VTList: N->getVTList(),
2326 N1: N->getOperand(Num: 0), N2: Op);
2327 ReplaceValueWith(From: SDValue(N, 1), To: Res.getValue(R: 1));
2328 return Res;
2329}
2330
2331// Promote the float operands used for comparison. The true- and false-
2332// operands have the same type as the result and are promoted, if needed, by
2333// PromoteFloatRes_SELECT_CC
2334SDValue DAGTypeLegalizer::PromoteFloatOp_SELECT_CC(SDNode *N, unsigned OpNo) {
2335 SDValue LHS = GetPromotedFloat(Op: N->getOperand(Num: 0));
2336 SDValue RHS = GetPromotedFloat(Op: N->getOperand(Num: 1));
2337
2338 return DAG.getNode(Opcode: ISD::SELECT_CC, DL: SDLoc(N), VT: N->getValueType(ResNo: 0),
2339 N1: LHS, N2: RHS, N3: N->getOperand(Num: 2), N4: N->getOperand(Num: 3),
2340 N5: N->getOperand(Num: 4));
2341}
2342
2343// Construct a SETCC that compares the promoted values and sets the conditional
2344// code.
2345SDValue DAGTypeLegalizer::PromoteFloatOp_SETCC(SDNode *N, unsigned OpNo) {
2346 EVT VT = N->getValueType(ResNo: 0);
2347 SDValue Op0 = GetPromotedFloat(Op: N->getOperand(Num: 0));
2348 SDValue Op1 = GetPromotedFloat(Op: N->getOperand(Num: 1));
2349 ISD::CondCode CCCode = cast<CondCodeSDNode>(Val: N->getOperand(Num: 2))->get();
2350
2351 return DAG.getSetCC(DL: SDLoc(N), VT, LHS: Op0, RHS: Op1, Cond: CCCode);
2352
2353}
2354
2355// Lower the promoted Float down to the integer value of same size and construct
2356// a STORE of the integer value.
2357SDValue DAGTypeLegalizer::PromoteFloatOp_STORE(SDNode *N, unsigned OpNo) {
2358 StoreSDNode *ST = cast<StoreSDNode>(Val: N);
2359 SDValue Val = ST->getValue();
2360 SDLoc DL(N);
2361
2362 SDValue Promoted = GetPromotedFloat(Op: Val);
2363 EVT VT = ST->getOperand(Num: 1).getValueType();
2364 EVT IVT = EVT::getIntegerVT(Context&: *DAG.getContext(), BitWidth: VT.getSizeInBits());
2365
2366 SDValue NewVal;
2367 NewVal = DAG.getNode(Opcode: GetPromotionOpcode(OpVT: Promoted.getValueType(), RetVT: VT), DL,
2368 VT: IVT, Operand: Promoted);
2369
2370 return DAG.getStore(Chain: ST->getChain(), dl: DL, Val: NewVal, Ptr: ST->getBasePtr(),
2371 MMO: ST->getMemOperand());
2372}
2373
2374//===----------------------------------------------------------------------===//
2375// Float Result Promotion
2376//===----------------------------------------------------------------------===//
2377
2378void DAGTypeLegalizer::PromoteFloatResult(SDNode *N, unsigned ResNo) {
2379 LLVM_DEBUG(dbgs() << "Promote float result " << ResNo << ": "; N->dump(&DAG));
2380 SDValue R = SDValue();
2381
2382 // See if the target wants to custom expand this node.
2383 if (CustomLowerNode(N, VT: N->getValueType(ResNo), LegalizeResult: true)) {
2384 LLVM_DEBUG(dbgs() << "Node has been custom expanded, done\n");
2385 return;
2386 }
2387
2388 switch (N->getOpcode()) {
2389 // These opcodes cannot appear if promotion of FP16 is done in the backend
2390 // instead of Clang
2391 case ISD::FP16_TO_FP:
2392 case ISD::FP_TO_FP16:
2393 default:
2394#ifndef NDEBUG
2395 dbgs() << "PromoteFloatResult #" << ResNo << ": ";
2396 N->dump(G: &DAG); dbgs() << "\n";
2397#endif
2398 report_fatal_error(reason: "Do not know how to promote this operator's result!");
2399
2400 case ISD::BITCAST: R = PromoteFloatRes_BITCAST(N); break;
2401 case ISD::ConstantFP: R = PromoteFloatRes_ConstantFP(N); break;
2402 case ISD::EXTRACT_VECTOR_ELT:
2403 R = PromoteFloatRes_EXTRACT_VECTOR_ELT(N); break;
2404 case ISD::FCOPYSIGN: R = PromoteFloatRes_FCOPYSIGN(N); break;
2405
2406 // Unary FP Operations
2407 case ISD::FABS:
2408 case ISD::FCBRT:
2409 case ISD::FCEIL:
2410 case ISD::FCOS:
2411 case ISD::FEXP:
2412 case ISD::FEXP2:
2413 case ISD::FEXP10:
2414 case ISD::FFLOOR:
2415 case ISD::FLOG:
2416 case ISD::FLOG2:
2417 case ISD::FLOG10:
2418 case ISD::FNEARBYINT:
2419 case ISD::FNEG:
2420 case ISD::FRINT:
2421 case ISD::FROUND:
2422 case ISD::FROUNDEVEN:
2423 case ISD::FSIN:
2424 case ISD::FSQRT:
2425 case ISD::FTRUNC:
2426 case ISD::FCANONICALIZE: R = PromoteFloatRes_UnaryOp(N); break;
2427
2428 // Binary FP Operations
2429 case ISD::FADD:
2430 case ISD::FDIV:
2431 case ISD::FMAXIMUM:
2432 case ISD::FMINIMUM:
2433 case ISD::FMAXNUM:
2434 case ISD::FMINNUM:
2435 case ISD::FMUL:
2436 case ISD::FPOW:
2437 case ISD::FREM:
2438 case ISD::FSUB: R = PromoteFloatRes_BinOp(N); break;
2439
2440 case ISD::FMA: // FMA is same as FMAD
2441 case ISD::FMAD: R = PromoteFloatRes_FMAD(N); break;
2442
2443 case ISD::FPOWI:
2444 case ISD::FLDEXP: R = PromoteFloatRes_ExpOp(N); break;
2445 case ISD::FFREXP: R = PromoteFloatRes_FFREXP(N); break;
2446
2447 case ISD::FP_ROUND: R = PromoteFloatRes_FP_ROUND(N); break;
2448 case ISD::STRICT_FP_ROUND:
2449 R = PromoteFloatRes_STRICT_FP_ROUND(N);
2450 break;
2451 case ISD::LOAD: R = PromoteFloatRes_LOAD(N); break;
2452 case ISD::SELECT: R = PromoteFloatRes_SELECT(N); break;
2453 case ISD::SELECT_CC: R = PromoteFloatRes_SELECT_CC(N); break;
2454
2455 case ISD::SINT_TO_FP:
2456 case ISD::UINT_TO_FP: R = PromoteFloatRes_XINT_TO_FP(N); break;
2457 case ISD::UNDEF: R = PromoteFloatRes_UNDEF(N); break;
2458 case ISD::ATOMIC_SWAP: R = BitcastToInt_ATOMIC_SWAP(N); break;
2459 case ISD::VECREDUCE_FADD:
2460 case ISD::VECREDUCE_FMUL:
2461 case ISD::VECREDUCE_FMIN:
2462 case ISD::VECREDUCE_FMAX:
2463 case ISD::VECREDUCE_FMAXIMUM:
2464 case ISD::VECREDUCE_FMINIMUM:
2465 R = PromoteFloatRes_VECREDUCE(N);
2466 break;
2467 case ISD::VECREDUCE_SEQ_FADD:
2468 case ISD::VECREDUCE_SEQ_FMUL:
2469 R = PromoteFloatRes_VECREDUCE_SEQ(N);
2470 break;
2471 }
2472
2473 if (R.getNode())
2474 SetPromotedFloat(Op: SDValue(N, ResNo), Result: R);
2475}
2476
2477// Bitcast from i16 to f16: convert the i16 to a f32 value instead.
2478// At this point, it is not possible to determine if the bitcast value is
2479// eventually stored to memory or promoted to f32 or promoted to a floating
2480// point at a higher precision. Some of these cases are handled by FP_EXTEND,
2481// STORE promotion handlers.
2482SDValue DAGTypeLegalizer::PromoteFloatRes_BITCAST(SDNode *N) {
2483 EVT VT = N->getValueType(ResNo: 0);
2484 EVT NVT = TLI.getTypeToTransformTo(Context&: *DAG.getContext(), VT);
2485 // Input type isn't guaranteed to be a scalar int so bitcast if not. The
2486 // bitcast will be legalized further if necessary.
2487 EVT IVT = EVT::getIntegerVT(Context&: *DAG.getContext(),
2488 BitWidth: N->getOperand(Num: 0).getValueType().getSizeInBits());
2489 SDValue Cast = DAG.getBitcast(VT: IVT, V: N->getOperand(Num: 0));
2490 return DAG.getNode(Opcode: GetPromotionOpcode(OpVT: VT, RetVT: NVT), DL: SDLoc(N), VT: NVT, Operand: Cast);
2491}
2492
2493SDValue DAGTypeLegalizer::PromoteFloatRes_ConstantFP(SDNode *N) {
2494 ConstantFPSDNode *CFPNode = cast<ConstantFPSDNode>(Val: N);
2495 EVT VT = N->getValueType(ResNo: 0);
2496 SDLoc DL(N);
2497
2498 // Get the (bit-cast) APInt of the APFloat and build an integer constant
2499 EVT IVT = EVT::getIntegerVT(Context&: *DAG.getContext(), BitWidth: VT.getSizeInBits());
2500 SDValue C = DAG.getConstant(Val: CFPNode->getValueAPF().bitcastToAPInt(), DL,
2501 VT: IVT);
2502
2503 // Convert the Constant to the desired FP type
2504 // FIXME We might be able to do the conversion during compilation and get rid
2505 // of it from the object code
2506 EVT NVT = TLI.getTypeToTransformTo(Context&: *DAG.getContext(), VT);
2507 return DAG.getNode(Opcode: GetPromotionOpcode(OpVT: VT, RetVT: NVT), DL, VT: NVT, Operand: C);
2508}
2509
2510// If the Index operand is a constant, try to redirect the extract operation to
2511// the correct legalized vector. If not, bit-convert the input vector to
2512// equivalent integer vector. Extract the element as an (bit-cast) integer
2513// value and convert it to the promoted type.
2514SDValue DAGTypeLegalizer::PromoteFloatRes_EXTRACT_VECTOR_ELT(SDNode *N) {
2515 SDLoc DL(N);
2516
2517 // If the index is constant, try to extract the value from the legalized
2518 // vector type.
2519 if (isa<ConstantSDNode>(Val: N->getOperand(Num: 1))) {
2520 SDValue Vec = N->getOperand(Num: 0);
2521 SDValue Idx = N->getOperand(Num: 1);
2522 EVT VecVT = Vec->getValueType(ResNo: 0);
2523 EVT EltVT = VecVT.getVectorElementType();
2524
2525 uint64_t IdxVal = Idx->getAsZExtVal();
2526
2527 switch (getTypeAction(VT: VecVT)) {
2528 default: break;
2529 case TargetLowering::TypeScalarizeVector: {
2530 SDValue Res = GetScalarizedVector(Op: N->getOperand(Num: 0));
2531 ReplaceValueWith(From: SDValue(N, 0), To: Res);
2532 return SDValue();
2533 }
2534 case TargetLowering::TypeWidenVector: {
2535 Vec = GetWidenedVector(Op: Vec);
2536 SDValue Res = DAG.getNode(Opcode: N->getOpcode(), DL, VT: EltVT, N1: Vec, N2: Idx);
2537 ReplaceValueWith(From: SDValue(N, 0), To: Res);
2538 return SDValue();
2539 }
2540 case TargetLowering::TypeSplitVector: {
2541 SDValue Lo, Hi;
2542 GetSplitVector(Op: Vec, Lo, Hi);
2543
2544 uint64_t LoElts = Lo.getValueType().getVectorNumElements();
2545 SDValue Res;
2546 if (IdxVal < LoElts)
2547 Res = DAG.getNode(Opcode: N->getOpcode(), DL, VT: EltVT, N1: Lo, N2: Idx);
2548 else
2549 Res = DAG.getNode(Opcode: N->getOpcode(), DL, VT: EltVT, N1: Hi,
2550 N2: DAG.getConstant(Val: IdxVal - LoElts, DL,
2551 VT: Idx.getValueType()));
2552 ReplaceValueWith(From: SDValue(N, 0), To: Res);
2553 return SDValue();
2554 }
2555
2556 }
2557 }
2558
2559 // Bit-convert the input vector to the equivalent integer vector
2560 SDValue NewOp = BitConvertVectorToIntegerVector(Op: N->getOperand(Num: 0));
2561 EVT IVT = NewOp.getValueType().getVectorElementType();
2562
2563 // Extract the element as an (bit-cast) integer value
2564 SDValue NewVal = DAG.getNode(Opcode: ISD::EXTRACT_VECTOR_ELT, DL, VT: IVT,
2565 N1: NewOp, N2: N->getOperand(Num: 1));
2566
2567 // Convert the element to the desired FP type
2568 EVT VT = N->getValueType(ResNo: 0);
2569 EVT NVT = TLI.getTypeToTransformTo(Context&: *DAG.getContext(), VT);
2570 return DAG.getNode(Opcode: GetPromotionOpcode(OpVT: VT, RetVT: NVT), DL: SDLoc(N), VT: NVT, Operand: NewVal);
2571}
2572
2573// FCOPYSIGN(X, Y) returns the value of X with the sign of Y. If the result
2574// needs promotion, so does the argument X. Note that Y, if needed, will be
2575// handled during operand promotion.
2576SDValue DAGTypeLegalizer::PromoteFloatRes_FCOPYSIGN(SDNode *N) {
2577 EVT VT = N->getValueType(ResNo: 0);
2578 EVT NVT = TLI.getTypeToTransformTo(Context&: *DAG.getContext(), VT);
2579 SDValue Op0 = GetPromotedFloat(Op: N->getOperand(Num: 0));
2580
2581 SDValue Op1 = N->getOperand(Num: 1);
2582
2583 return DAG.getNode(Opcode: N->getOpcode(), DL: SDLoc(N), VT: NVT, N1: Op0, N2: Op1);
2584}
2585
2586// Unary operation where the result and the operand have PromoteFloat type
2587// action. Construct a new SDNode with the promoted float value of the old
2588// operand.
2589SDValue DAGTypeLegalizer::PromoteFloatRes_UnaryOp(SDNode *N) {
2590 EVT VT = N->getValueType(ResNo: 0);
2591 EVT NVT = TLI.getTypeToTransformTo(Context&: *DAG.getContext(), VT);
2592 SDValue Op = GetPromotedFloat(Op: N->getOperand(Num: 0));
2593
2594 return DAG.getNode(Opcode: N->getOpcode(), DL: SDLoc(N), VT: NVT, Operand: Op);
2595}
2596
2597// Binary operations where the result and both operands have PromoteFloat type
2598// action. Construct a new SDNode with the promoted float values of the old
2599// operands.
2600SDValue DAGTypeLegalizer::PromoteFloatRes_BinOp(SDNode *N) {
2601 EVT VT = N->getValueType(ResNo: 0);
2602 EVT NVT = TLI.getTypeToTransformTo(Context&: *DAG.getContext(), VT);
2603 SDValue Op0 = GetPromotedFloat(Op: N->getOperand(Num: 0));
2604 SDValue Op1 = GetPromotedFloat(Op: N->getOperand(Num: 1));
2605 return DAG.getNode(Opcode: N->getOpcode(), DL: SDLoc(N), VT: NVT, N1: Op0, N2: Op1, Flags: N->getFlags());
2606}
2607
2608SDValue DAGTypeLegalizer::PromoteFloatRes_FMAD(SDNode *N) {
2609 EVT VT = N->getValueType(ResNo: 0);
2610 EVT NVT = TLI.getTypeToTransformTo(Context&: *DAG.getContext(), VT);
2611 SDValue Op0 = GetPromotedFloat(Op: N->getOperand(Num: 0));
2612 SDValue Op1 = GetPromotedFloat(Op: N->getOperand(Num: 1));
2613 SDValue Op2 = GetPromotedFloat(Op: N->getOperand(Num: 2));
2614
2615 return DAG.getNode(Opcode: N->getOpcode(), DL: SDLoc(N), VT: NVT, N1: Op0, N2: Op1, N3: Op2);
2616}
2617
2618// Promote the Float (first) operand and retain the Integer (second) operand
2619SDValue DAGTypeLegalizer::PromoteFloatRes_ExpOp(SDNode *N) {
2620 EVT VT = N->getValueType(ResNo: 0);
2621 EVT NVT = TLI.getTypeToTransformTo(Context&: *DAG.getContext(), VT);
2622 SDValue Op0 = GetPromotedFloat(Op: N->getOperand(Num: 0));
2623 SDValue Op1 = N->getOperand(Num: 1);
2624
2625 return DAG.getNode(Opcode: N->getOpcode(), DL: SDLoc(N), VT: NVT, N1: Op0, N2: Op1);
2626}
2627
2628SDValue DAGTypeLegalizer::PromoteFloatRes_FFREXP(SDNode *N) {
2629 EVT VT = N->getValueType(ResNo: 0);
2630 EVT NVT = TLI.getTypeToTransformTo(Context&: *DAG.getContext(), VT);
2631 SDValue Op = GetPromotedFloat(Op: N->getOperand(Num: 0));
2632 SDValue Res =
2633 DAG.getNode(Opcode: N->getOpcode(), DL: SDLoc(N), ResultTys: {NVT, N->getValueType(ResNo: 1)}, Ops: Op);
2634
2635 ReplaceValueWith(From: SDValue(N, 1), To: Res.getValue(R: 1));
2636 return Res;
2637}
2638
2639// Explicit operation to reduce precision. Reduce the value to half precision
2640// and promote it back to the legal type.
2641SDValue DAGTypeLegalizer::PromoteFloatRes_FP_ROUND(SDNode *N) {
2642 SDLoc DL(N);
2643
2644 SDValue Op = N->getOperand(Num: 0);
2645 EVT VT = N->getValueType(ResNo: 0);
2646 EVT OpVT = Op->getValueType(ResNo: 0);
2647 EVT NVT = TLI.getTypeToTransformTo(Context&: *DAG.getContext(), VT: N->getValueType(ResNo: 0));
2648 EVT IVT = EVT::getIntegerVT(Context&: *DAG.getContext(), BitWidth: VT.getSizeInBits());
2649
2650 // Round promoted float to desired precision
2651 SDValue Round = DAG.getNode(Opcode: GetPromotionOpcode(OpVT, RetVT: VT), DL, VT: IVT, Operand: Op);
2652 // Promote it back to the legal output type
2653 return DAG.getNode(Opcode: GetPromotionOpcode(OpVT: VT, RetVT: NVT), DL, VT: NVT, Operand: Round);
2654}
2655
2656// Explicit operation to reduce precision. Reduce the value to half precision
2657// and promote it back to the legal type.
2658SDValue DAGTypeLegalizer::PromoteFloatRes_STRICT_FP_ROUND(SDNode *N) {
2659 SDLoc DL(N);
2660
2661 SDValue Chain = N->getOperand(Num: 0);
2662 SDValue Op = N->getOperand(Num: 1);
2663 EVT VT = N->getValueType(ResNo: 0);
2664 EVT OpVT = Op->getValueType(ResNo: 0);
2665 EVT NVT = TLI.getTypeToTransformTo(Context&: *DAG.getContext(), VT: N->getValueType(ResNo: 0));
2666 EVT IVT = EVT::getIntegerVT(Context&: *DAG.getContext(), BitWidth: VT.getSizeInBits());
2667
2668 // Round promoted float to desired precision
2669 SDValue Round = DAG.getNode(GetPromotionOpcodeStrict(OpVT, VT), DL,
2670 DAG.getVTList(IVT, MVT::Other), Chain, Op);
2671 // Promote it back to the legal output type
2672 SDValue Res =
2673 DAG.getNode(GetPromotionOpcodeStrict(VT, NVT), DL,
2674 DAG.getVTList(NVT, MVT::Other), Round.getValue(1), Round);
2675 ReplaceValueWith(From: SDValue(N, 1), To: Res.getValue(R: 1));
2676 return Res;
2677}
2678
2679SDValue DAGTypeLegalizer::PromoteFloatRes_LOAD(SDNode *N) {
2680 LoadSDNode *L = cast<LoadSDNode>(Val: N);
2681 EVT VT = N->getValueType(ResNo: 0);
2682
2683 // Load the value as an integer value with the same number of bits.
2684 EVT IVT = EVT::getIntegerVT(Context&: *DAG.getContext(), BitWidth: VT.getSizeInBits());
2685 SDValue newL = DAG.getLoad(
2686 AM: L->getAddressingMode(), ExtType: L->getExtensionType(), VT: IVT, dl: SDLoc(N),
2687 Chain: L->getChain(), Ptr: L->getBasePtr(), Offset: L->getOffset(), PtrInfo: L->getPointerInfo(), MemVT: IVT,
2688 Alignment: L->getOriginalAlign(), MMOFlags: L->getMemOperand()->getFlags(), AAInfo: L->getAAInfo());
2689 // Legalize the chain result by replacing uses of the old value chain with the
2690 // new one
2691 ReplaceValueWith(From: SDValue(N, 1), To: newL.getValue(R: 1));
2692
2693 // Convert the integer value to the desired FP type
2694 EVT NVT = TLI.getTypeToTransformTo(Context&: *DAG.getContext(), VT);
2695 return DAG.getNode(Opcode: GetPromotionOpcode(OpVT: VT, RetVT: NVT), DL: SDLoc(N), VT: NVT, Operand: newL);
2696}
2697
2698// Construct a new SELECT node with the promoted true- and false- values.
2699SDValue DAGTypeLegalizer::PromoteFloatRes_SELECT(SDNode *N) {
2700 SDValue TrueVal = GetPromotedFloat(Op: N->getOperand(Num: 1));
2701 SDValue FalseVal = GetPromotedFloat(Op: N->getOperand(Num: 2));
2702
2703 return DAG.getNode(Opcode: ISD::SELECT, DL: SDLoc(N), VT: TrueVal->getValueType(ResNo: 0),
2704 N1: N->getOperand(Num: 0), N2: TrueVal, N3: FalseVal);
2705}
2706
2707// Construct a new SELECT_CC node with the promoted true- and false- values.
2708// The operands used for comparison are promoted by PromoteFloatOp_SELECT_CC.
2709SDValue DAGTypeLegalizer::PromoteFloatRes_SELECT_CC(SDNode *N) {
2710 SDValue TrueVal = GetPromotedFloat(Op: N->getOperand(Num: 2));
2711 SDValue FalseVal = GetPromotedFloat(Op: N->getOperand(Num: 3));
2712
2713 return DAG.getNode(Opcode: ISD::SELECT_CC, DL: SDLoc(N),
2714 VT: TrueVal.getNode()->getValueType(ResNo: 0), N1: N->getOperand(Num: 0),
2715 N2: N->getOperand(Num: 1), N3: TrueVal, N4: FalseVal, N5: N->getOperand(Num: 4));
2716}
2717
2718// Construct a SDNode that transforms the SINT or UINT operand to the promoted
2719// float type.
2720SDValue DAGTypeLegalizer::PromoteFloatRes_XINT_TO_FP(SDNode *N) {
2721 SDLoc DL(N);
2722 EVT VT = N->getValueType(ResNo: 0);
2723 EVT NVT = TLI.getTypeToTransformTo(Context&: *DAG.getContext(), VT);
2724 SDValue NV = DAG.getNode(Opcode: N->getOpcode(), DL, VT: NVT, Operand: N->getOperand(Num: 0));
2725 // Round the value to the desired precision (that of the source type).
2726 return DAG.getNode(
2727 Opcode: ISD::FP_EXTEND, DL, VT: NVT,
2728 Operand: DAG.getNode(Opcode: ISD::FP_ROUND, DL, VT, N1: NV,
2729 N2: DAG.getIntPtrConstant(Val: 0, DL, /*isTarget=*/true)));
2730}
2731
2732SDValue DAGTypeLegalizer::PromoteFloatRes_UNDEF(SDNode *N) {
2733 return DAG.getUNDEF(VT: TLI.getTypeToTransformTo(Context&: *DAG.getContext(),
2734 VT: N->getValueType(ResNo: 0)));
2735}
2736
2737SDValue DAGTypeLegalizer::PromoteFloatRes_VECREDUCE(SDNode *N) {
2738 // Expand and promote recursively.
2739 // TODO: This is non-optimal, but dealing with the concurrently happening
2740 // vector-legalization is non-trivial. We could do something similar to
2741 // PromoteFloatRes_EXTRACT_VECTOR_ELT here.
2742 ReplaceValueWith(From: SDValue(N, 0), To: TLI.expandVecReduce(Node: N, DAG));
2743 return SDValue();
2744}
2745
2746SDValue DAGTypeLegalizer::PromoteFloatRes_VECREDUCE_SEQ(SDNode *N) {
2747 ReplaceValueWith(From: SDValue(N, 0), To: TLI.expandVecReduceSeq(Node: N, DAG));
2748 return SDValue();
2749}
2750
2751SDValue DAGTypeLegalizer::BitcastToInt_ATOMIC_SWAP(SDNode *N) {
2752 EVT VT = N->getValueType(ResNo: 0);
2753
2754 AtomicSDNode *AM = cast<AtomicSDNode>(Val: N);
2755 SDLoc SL(N);
2756
2757 SDValue CastVal = BitConvertToInteger(Op: AM->getVal());
2758 EVT CastVT = CastVal.getValueType();
2759
2760 SDValue NewAtomic
2761 = DAG.getAtomic(ISD::ATOMIC_SWAP, SL, CastVT,
2762 DAG.getVTList(CastVT, MVT::Other),
2763 { AM->getChain(), AM->getBasePtr(), CastVal },
2764 AM->getMemOperand());
2765
2766 SDValue Result = NewAtomic;
2767
2768 if (getTypeAction(VT) == TargetLowering::TypePromoteFloat) {
2769 EVT NFPVT = TLI.getTypeToTransformTo(Context&: *DAG.getContext(), VT);
2770 Result = DAG.getNode(Opcode: GetPromotionOpcode(OpVT: VT, RetVT: NFPVT), DL: SL, VT: NFPVT,
2771 Operand: NewAtomic);
2772 }
2773
2774 // Legalize the chain result by replacing uses of the old value chain with the
2775 // new one
2776 ReplaceValueWith(From: SDValue(N, 1), To: NewAtomic.getValue(R: 1));
2777
2778 return Result;
2779
2780}
2781
2782//===----------------------------------------------------------------------===//
2783// Half Result Soft Promotion
2784//===----------------------------------------------------------------------===//
2785
2786void DAGTypeLegalizer::SoftPromoteHalfResult(SDNode *N, unsigned ResNo) {
2787 LLVM_DEBUG(dbgs() << "Soft promote half result " << ResNo << ": ";
2788 N->dump(&DAG));
2789 SDValue R = SDValue();
2790
2791 // See if the target wants to custom expand this node.
2792 if (CustomLowerNode(N, VT: N->getValueType(ResNo), LegalizeResult: true)) {
2793 LLVM_DEBUG(dbgs() << "Node has been custom expanded, done\n");
2794 return;
2795 }
2796
2797 switch (N->getOpcode()) {
2798 default:
2799#ifndef NDEBUG
2800 dbgs() << "SoftPromoteHalfResult #" << ResNo << ": ";
2801 N->dump(G: &DAG); dbgs() << "\n";
2802#endif
2803 report_fatal_error(reason: "Do not know how to soft promote this operator's "
2804 "result!");
2805
2806 case ISD::BITCAST: R = SoftPromoteHalfRes_BITCAST(N); break;
2807 case ISD::ConstantFP: R = SoftPromoteHalfRes_ConstantFP(N); break;
2808 case ISD::EXTRACT_VECTOR_ELT:
2809 R = SoftPromoteHalfRes_EXTRACT_VECTOR_ELT(N); break;
2810 case ISD::FCOPYSIGN: R = SoftPromoteHalfRes_FCOPYSIGN(N); break;
2811 case ISD::STRICT_FP_ROUND:
2812 case ISD::FP_ROUND: R = SoftPromoteHalfRes_FP_ROUND(N); break;
2813
2814 // Unary FP Operations
2815 case ISD::FABS:
2816 case ISD::FCBRT:
2817 case ISD::FCEIL:
2818 case ISD::FCOS:
2819 case ISD::FEXP:
2820 case ISD::FEXP2:
2821 case ISD::FEXP10:
2822 case ISD::FFLOOR:
2823 case ISD::FLOG:
2824 case ISD::FLOG2:
2825 case ISD::FLOG10:
2826 case ISD::FNEARBYINT:
2827 case ISD::FNEG:
2828 case ISD::FREEZE:
2829 case ISD::FRINT:
2830 case ISD::FROUND:
2831 case ISD::FROUNDEVEN:
2832 case ISD::FSIN:
2833 case ISD::FSQRT:
2834 case ISD::FTRUNC:
2835 case ISD::FCANONICALIZE: R = SoftPromoteHalfRes_UnaryOp(N); break;
2836
2837 // Binary FP Operations
2838 case ISD::FADD:
2839 case ISD::FDIV:
2840 case ISD::FMAXIMUM:
2841 case ISD::FMINIMUM:
2842 case ISD::FMAXNUM:
2843 case ISD::FMINNUM:
2844 case ISD::FMUL:
2845 case ISD::FPOW:
2846 case ISD::FREM:
2847 case ISD::FSUB: R = SoftPromoteHalfRes_BinOp(N); break;
2848
2849 case ISD::FMA: // FMA is same as FMAD
2850 case ISD::FMAD: R = SoftPromoteHalfRes_FMAD(N); break;
2851
2852 case ISD::FPOWI:
2853 case ISD::FLDEXP: R = SoftPromoteHalfRes_ExpOp(N); break;
2854
2855 case ISD::FFREXP: R = SoftPromoteHalfRes_FFREXP(N); break;
2856
2857 case ISD::LOAD: R = SoftPromoteHalfRes_LOAD(N); break;
2858 case ISD::SELECT: R = SoftPromoteHalfRes_SELECT(N); break;
2859 case ISD::SELECT_CC: R = SoftPromoteHalfRes_SELECT_CC(N); break;
2860 case ISD::SINT_TO_FP:
2861 case ISD::UINT_TO_FP: R = SoftPromoteHalfRes_XINT_TO_FP(N); break;
2862 case ISD::UNDEF: R = SoftPromoteHalfRes_UNDEF(N); break;
2863 case ISD::ATOMIC_SWAP: R = BitcastToInt_ATOMIC_SWAP(N); break;
2864 case ISD::VECREDUCE_FADD:
2865 case ISD::VECREDUCE_FMUL:
2866 case ISD::VECREDUCE_FMIN:
2867 case ISD::VECREDUCE_FMAX:
2868 case ISD::VECREDUCE_FMAXIMUM:
2869 case ISD::VECREDUCE_FMINIMUM:
2870 R = SoftPromoteHalfRes_VECREDUCE(N);
2871 break;
2872 case ISD::VECREDUCE_SEQ_FADD:
2873 case ISD::VECREDUCE_SEQ_FMUL:
2874 R = SoftPromoteHalfRes_VECREDUCE_SEQ(N);
2875 break;
2876 }
2877
2878 if (R.getNode())
2879 SetSoftPromotedHalf(Op: SDValue(N, ResNo), Result: R);
2880}
2881
2882SDValue DAGTypeLegalizer::SoftPromoteHalfRes_BITCAST(SDNode *N) {
2883 return BitConvertToInteger(Op: N->getOperand(Num: 0));
2884}
2885
2886SDValue DAGTypeLegalizer::SoftPromoteHalfRes_ConstantFP(SDNode *N) {
2887 ConstantFPSDNode *CN = cast<ConstantFPSDNode>(Val: N);
2888
2889 // Get the (bit-cast) APInt of the APFloat and build an integer constant
2890 return DAG.getConstant(CN->getValueAPF().bitcastToAPInt(), SDLoc(CN),
2891 MVT::i16);
2892}
2893
2894SDValue DAGTypeLegalizer::SoftPromoteHalfRes_EXTRACT_VECTOR_ELT(SDNode *N) {
2895 SDValue NewOp = BitConvertVectorToIntegerVector(Op: N->getOperand(Num: 0));
2896 return DAG.getNode(Opcode: ISD::EXTRACT_VECTOR_ELT, DL: SDLoc(N),
2897 VT: NewOp.getValueType().getVectorElementType(), N1: NewOp,
2898 N2: N->getOperand(Num: 1));
2899}
2900
2901SDValue DAGTypeLegalizer::SoftPromoteHalfRes_FCOPYSIGN(SDNode *N) {
2902 SDValue LHS = GetSoftPromotedHalf(Op: N->getOperand(Num: 0));
2903 SDValue RHS = BitConvertToInteger(Op: N->getOperand(Num: 1));
2904 SDLoc dl(N);
2905
2906 EVT LVT = LHS.getValueType();
2907 EVT RVT = RHS.getValueType();
2908
2909 unsigned LSize = LVT.getSizeInBits();
2910 unsigned RSize = RVT.getSizeInBits();
2911
2912 // First get the sign bit of second operand.
2913 SDValue SignBit = DAG.getNode(
2914 Opcode: ISD::SHL, DL: dl, VT: RVT, N1: DAG.getConstant(Val: 1, DL: dl, VT: RVT),
2915 N2: DAG.getConstant(Val: RSize - 1, DL: dl,
2916 VT: TLI.getShiftAmountTy(LHSTy: RVT, DL: DAG.getDataLayout())));
2917 SignBit = DAG.getNode(Opcode: ISD::AND, DL: dl, VT: RVT, N1: RHS, N2: SignBit);
2918
2919 // Shift right or sign-extend it if the two operands have different types.
2920 int SizeDiff = RVT.getSizeInBits() - LVT.getSizeInBits();
2921 if (SizeDiff > 0) {
2922 SignBit =
2923 DAG.getNode(Opcode: ISD::SRL, DL: dl, VT: RVT, N1: SignBit,
2924 N2: DAG.getConstant(Val: SizeDiff, DL: dl,
2925 VT: TLI.getShiftAmountTy(LHSTy: SignBit.getValueType(),
2926 DL: DAG.getDataLayout())));
2927 SignBit = DAG.getNode(Opcode: ISD::TRUNCATE, DL: dl, VT: LVT, Operand: SignBit);
2928 } else if (SizeDiff < 0) {
2929 SignBit = DAG.getNode(Opcode: ISD::ANY_EXTEND, DL: dl, VT: LVT, Operand: SignBit);
2930 SignBit =
2931 DAG.getNode(Opcode: ISD::SHL, DL: dl, VT: LVT, N1: SignBit,
2932 N2: DAG.getConstant(Val: -SizeDiff, DL: dl,
2933 VT: TLI.getShiftAmountTy(LHSTy: SignBit.getValueType(),
2934 DL: DAG.getDataLayout())));
2935 }
2936
2937 // Clear the sign bit of the first operand.
2938 SDValue Mask = DAG.getNode(
2939 Opcode: ISD::SHL, DL: dl, VT: LVT, N1: DAG.getConstant(Val: 1, DL: dl, VT: LVT),
2940 N2: DAG.getConstant(Val: LSize - 1, DL: dl,
2941 VT: TLI.getShiftAmountTy(LHSTy: LVT, DL: DAG.getDataLayout())));
2942 Mask = DAG.getNode(Opcode: ISD::SUB, DL: dl, VT: LVT, N1: Mask, N2: DAG.getConstant(Val: 1, DL: dl, VT: LVT));
2943 LHS = DAG.getNode(Opcode: ISD::AND, DL: dl, VT: LVT, N1: LHS, N2: Mask);
2944
2945 // Or the value with the sign bit.
2946 return DAG.getNode(Opcode: ISD::OR, DL: dl, VT: LVT, N1: LHS, N2: SignBit);
2947}
2948
2949SDValue DAGTypeLegalizer::SoftPromoteHalfRes_FMAD(SDNode *N) {
2950 EVT OVT = N->getValueType(ResNo: 0);
2951 EVT NVT = TLI.getTypeToTransformTo(Context&: *DAG.getContext(), VT: OVT);
2952 SDValue Op0 = GetSoftPromotedHalf(Op: N->getOperand(Num: 0));
2953 SDValue Op1 = GetSoftPromotedHalf(Op: N->getOperand(Num: 1));
2954 SDValue Op2 = GetSoftPromotedHalf(Op: N->getOperand(Num: 2));
2955 SDLoc dl(N);
2956
2957 // Promote to the larger FP type.
2958 auto PromotionOpcode = GetPromotionOpcode(OpVT: OVT, RetVT: NVT);
2959 Op0 = DAG.getNode(Opcode: PromotionOpcode, DL: dl, VT: NVT, Operand: Op0);
2960 Op1 = DAG.getNode(Opcode: PromotionOpcode, DL: dl, VT: NVT, Operand: Op1);
2961 Op2 = DAG.getNode(Opcode: PromotionOpcode, DL: dl, VT: NVT, Operand: Op2);
2962
2963 SDValue Res = DAG.getNode(Opcode: N->getOpcode(), DL: dl, VT: NVT, N1: Op0, N2: Op1, N3: Op2);
2964
2965 // Convert back to FP16 as an integer.
2966 return DAG.getNode(GetPromotionOpcode(NVT, OVT), dl, MVT::i16, Res);
2967}
2968
2969SDValue DAGTypeLegalizer::SoftPromoteHalfRes_ExpOp(SDNode *N) {
2970 EVT OVT = N->getValueType(ResNo: 0);
2971 EVT NVT = TLI.getTypeToTransformTo(Context&: *DAG.getContext(), VT: OVT);
2972 SDValue Op0 = GetSoftPromotedHalf(Op: N->getOperand(Num: 0));
2973 SDValue Op1 = N->getOperand(Num: 1);
2974 SDLoc dl(N);
2975
2976 // Promote to the larger FP type.
2977 Op0 = DAG.getNode(Opcode: GetPromotionOpcode(OpVT: OVT, RetVT: NVT), DL: dl, VT: NVT, Operand: Op0);
2978
2979 SDValue Res = DAG.getNode(Opcode: N->getOpcode(), DL: dl, VT: NVT, N1: Op0, N2: Op1);
2980
2981 // Convert back to FP16 as an integer.
2982 return DAG.getNode(GetPromotionOpcode(NVT, OVT), dl, MVT::i16, Res);
2983}
2984
2985SDValue DAGTypeLegalizer::SoftPromoteHalfRes_FFREXP(SDNode *N) {
2986 EVT OVT = N->getValueType(ResNo: 0);
2987 EVT NVT = TLI.getTypeToTransformTo(Context&: *DAG.getContext(), VT: OVT);
2988 SDValue Op = GetSoftPromotedHalf(Op: N->getOperand(Num: 0));
2989 SDLoc dl(N);
2990
2991 // Promote to the larger FP type.
2992 Op = DAG.getNode(Opcode: GetPromotionOpcode(OpVT: OVT, RetVT: NVT), DL: dl, VT: NVT, Operand: Op);
2993
2994 SDValue Res = DAG.getNode(Opcode: N->getOpcode(), DL: dl,
2995 VTList: DAG.getVTList(VT1: NVT, VT2: N->getValueType(ResNo: 1)), N: Op);
2996
2997 ReplaceValueWith(From: SDValue(N, 1), To: Res.getValue(R: 1));
2998
2999 // Convert back to FP16 as an integer.
3000 return DAG.getNode(GetPromotionOpcode(NVT, OVT), dl, MVT::i16, Res);
3001}
3002
3003SDValue DAGTypeLegalizer::SoftPromoteHalfRes_FP_ROUND(SDNode *N) {
3004 EVT RVT = N->getValueType(ResNo: 0);
3005 EVT SVT = N->getOperand(Num: 0).getValueType();
3006
3007 if (N->isStrictFPOpcode()) {
3008 // FIXME: assume we only have two f16 variants for now.
3009 unsigned Opcode;
3010 if (RVT == MVT::f16)
3011 Opcode = ISD::STRICT_FP_TO_FP16;
3012 else if (RVT == MVT::bf16)
3013 Opcode = ISD::STRICT_FP_TO_BF16;
3014 else
3015 llvm_unreachable("unknown half type");
3016 SDValue Res = DAG.getNode(Opcode, SDLoc(N), {MVT::i16, MVT::Other},
3017 {N->getOperand(0), N->getOperand(1)});
3018 ReplaceValueWith(From: SDValue(N, 1), To: Res.getValue(R: 1));
3019 return Res;
3020 }
3021
3022 return DAG.getNode(GetPromotionOpcode(SVT, RVT), SDLoc(N), MVT::i16,
3023 N->getOperand(0));
3024}
3025
3026SDValue DAGTypeLegalizer::SoftPromoteHalfRes_LOAD(SDNode *N) {
3027 LoadSDNode *L = cast<LoadSDNode>(Val: N);
3028
3029 // Load the value as an integer value with the same number of bits.
3030 assert(L->getExtensionType() == ISD::NON_EXTLOAD && "Unexpected extension!");
3031 SDValue NewL =
3032 DAG.getLoad(L->getAddressingMode(), L->getExtensionType(), MVT::i16,
3033 SDLoc(N), L->getChain(), L->getBasePtr(), L->getOffset(),
3034 L->getPointerInfo(), MVT::i16, L->getOriginalAlign(),
3035 L->getMemOperand()->getFlags(), L->getAAInfo());
3036 // Legalize the chain result by replacing uses of the old value chain with the
3037 // new one
3038 ReplaceValueWith(From: SDValue(N, 1), To: NewL.getValue(R: 1));
3039 return NewL;
3040}
3041
3042SDValue DAGTypeLegalizer::SoftPromoteHalfRes_SELECT(SDNode *N) {
3043 SDValue Op1 = GetSoftPromotedHalf(Op: N->getOperand(Num: 1));
3044 SDValue Op2 = GetSoftPromotedHalf(Op: N->getOperand(Num: 2));
3045 return DAG.getSelect(DL: SDLoc(N), VT: Op1.getValueType(), Cond: N->getOperand(Num: 0), LHS: Op1,
3046 RHS: Op2);
3047}
3048
3049SDValue DAGTypeLegalizer::SoftPromoteHalfRes_SELECT_CC(SDNode *N) {
3050 SDValue Op2 = GetSoftPromotedHalf(Op: N->getOperand(Num: 2));
3051 SDValue Op3 = GetSoftPromotedHalf(Op: N->getOperand(Num: 3));
3052 return DAG.getNode(Opcode: ISD::SELECT_CC, DL: SDLoc(N), VT: Op2.getValueType(),
3053 N1: N->getOperand(Num: 0), N2: N->getOperand(Num: 1), N3: Op2, N4: Op3,
3054 N5: N->getOperand(Num: 4));
3055}
3056
3057SDValue DAGTypeLegalizer::SoftPromoteHalfRes_XINT_TO_FP(SDNode *N) {
3058 EVT OVT = N->getValueType(ResNo: 0);
3059 EVT NVT = TLI.getTypeToTransformTo(Context&: *DAG.getContext(), VT: OVT);
3060 SDLoc dl(N);
3061
3062 SDValue Res = DAG.getNode(Opcode: N->getOpcode(), DL: dl, VT: NVT, Operand: N->getOperand(Num: 0));
3063
3064 // Round the value to the softened type.
3065 return DAG.getNode(GetPromotionOpcode(NVT, OVT), dl, MVT::i16, Res);
3066}
3067
3068SDValue DAGTypeLegalizer::SoftPromoteHalfRes_UNDEF(SDNode *N) {
3069 return DAG.getUNDEF(MVT::i16);
3070}
3071
3072SDValue DAGTypeLegalizer::SoftPromoteHalfRes_UnaryOp(SDNode *N) {
3073 EVT OVT = N->getValueType(ResNo: 0);
3074 EVT NVT = TLI.getTypeToTransformTo(Context&: *DAG.getContext(), VT: OVT);
3075 SDValue Op = GetSoftPromotedHalf(Op: N->getOperand(Num: 0));
3076 SDLoc dl(N);
3077
3078 // Promote to the larger FP type.
3079 Op = DAG.getNode(Opcode: GetPromotionOpcode(OpVT: OVT, RetVT: NVT), DL: dl, VT: NVT, Operand: Op);
3080
3081 SDValue Res = DAG.getNode(Opcode: N->getOpcode(), DL: dl, VT: NVT, Operand: Op);
3082
3083 // Convert back to FP16 as an integer.
3084 return DAG.getNode(GetPromotionOpcode(NVT, OVT), dl, MVT::i16, Res);
3085}
3086
3087SDValue DAGTypeLegalizer::SoftPromoteHalfRes_BinOp(SDNode *N) {
3088 EVT OVT = N->getValueType(ResNo: 0);
3089 EVT NVT = TLI.getTypeToTransformTo(Context&: *DAG.getContext(), VT: OVT);
3090 SDValue Op0 = GetSoftPromotedHalf(Op: N->getOperand(Num: 0));
3091 SDValue Op1 = GetSoftPromotedHalf(Op: N->getOperand(Num: 1));
3092 SDLoc dl(N);
3093
3094 // Promote to the larger FP type.
3095 auto PromotionOpcode = GetPromotionOpcode(OpVT: OVT, RetVT: NVT);
3096 Op0 = DAG.getNode(Opcode: PromotionOpcode, DL: dl, VT: NVT, Operand: Op0);
3097 Op1 = DAG.getNode(Opcode: PromotionOpcode, DL: dl, VT: NVT, Operand: Op1);
3098
3099 SDValue Res = DAG.getNode(Opcode: N->getOpcode(), DL: dl, VT: NVT, N1: Op0, N2: Op1);
3100
3101 // Convert back to FP16 as an integer.
3102 return DAG.getNode(GetPromotionOpcode(NVT, OVT), dl, MVT::i16, Res);
3103}
3104
3105SDValue DAGTypeLegalizer::SoftPromoteHalfRes_VECREDUCE(SDNode *N) {
3106 // Expand and soften recursively.
3107 ReplaceValueWith(From: SDValue(N, 0), To: TLI.expandVecReduce(Node: N, DAG));
3108 return SDValue();
3109}
3110
3111SDValue DAGTypeLegalizer::SoftPromoteHalfRes_VECREDUCE_SEQ(SDNode *N) {
3112 // Expand and soften.
3113 ReplaceValueWith(From: SDValue(N, 0), To: TLI.expandVecReduceSeq(Node: N, DAG));
3114 return SDValue();
3115}
3116
3117//===----------------------------------------------------------------------===//
3118// Half Operand Soft Promotion
3119//===----------------------------------------------------------------------===//
3120
3121bool DAGTypeLegalizer::SoftPromoteHalfOperand(SDNode *N, unsigned OpNo) {
3122 LLVM_DEBUG(dbgs() << "Soft promote half operand " << OpNo << ": ";
3123 N->dump(&DAG));
3124 SDValue Res = SDValue();
3125
3126 if (CustomLowerNode(N, VT: N->getOperand(Num: OpNo).getValueType(), LegalizeResult: false)) {
3127 LLVM_DEBUG(dbgs() << "Node has been custom lowered, done\n");
3128 return false;
3129 }
3130
3131 // Nodes that use a promotion-requiring floating point operand, but doesn't
3132 // produce a soft promotion-requiring floating point result, need to be
3133 // legalized to use the soft promoted float operand. Nodes that produce at
3134 // least one soft promotion-requiring floating point result have their
3135 // operands legalized as a part of PromoteFloatResult.
3136 switch (N->getOpcode()) {
3137 default:
3138 #ifndef NDEBUG
3139 dbgs() << "SoftPromoteHalfOperand Op #" << OpNo << ": ";
3140 N->dump(G: &DAG); dbgs() << "\n";
3141 #endif
3142 report_fatal_error(reason: "Do not know how to soft promote this operator's "
3143 "operand!");
3144
3145 case ISD::BITCAST: Res = SoftPromoteHalfOp_BITCAST(N); break;
3146 case ISD::FCOPYSIGN: Res = SoftPromoteHalfOp_FCOPYSIGN(N, OpNo); break;
3147 case ISD::FP_TO_SINT:
3148 case ISD::FP_TO_UINT: Res = SoftPromoteHalfOp_FP_TO_XINT(N); break;
3149 case ISD::FP_TO_SINT_SAT:
3150 case ISD::FP_TO_UINT_SAT:
3151 Res = SoftPromoteHalfOp_FP_TO_XINT_SAT(N); break;
3152 case ISD::STRICT_FP_EXTEND:
3153 case ISD::FP_EXTEND: Res = SoftPromoteHalfOp_FP_EXTEND(N); break;
3154 case ISD::SELECT_CC: Res = SoftPromoteHalfOp_SELECT_CC(N, OpNo); break;
3155 case ISD::SETCC: Res = SoftPromoteHalfOp_SETCC(N); break;
3156 case ISD::STORE: Res = SoftPromoteHalfOp_STORE(N, OpNo); break;
3157 case ISD::STACKMAP:
3158 Res = SoftPromoteHalfOp_STACKMAP(N, OpNo);
3159 break;
3160 case ISD::PATCHPOINT:
3161 Res = SoftPromoteHalfOp_PATCHPOINT(N, OpNo);
3162 break;
3163 }
3164
3165 if (!Res.getNode())
3166 return false;
3167
3168 assert(Res.getNode() != N && "Expected a new node!");
3169
3170 assert(Res.getValueType() == N->getValueType(0) && N->getNumValues() == 1 &&
3171 "Invalid operand expansion");
3172
3173 ReplaceValueWith(From: SDValue(N, 0), To: Res);
3174 return false;
3175}
3176
3177SDValue DAGTypeLegalizer::SoftPromoteHalfOp_BITCAST(SDNode *N) {
3178 SDValue Op0 = GetSoftPromotedHalf(Op: N->getOperand(Num: 0));
3179
3180 return DAG.getNode(Opcode: ISD::BITCAST, DL: SDLoc(N), VT: N->getValueType(ResNo: 0), Operand: Op0);
3181}
3182
3183SDValue DAGTypeLegalizer::SoftPromoteHalfOp_FCOPYSIGN(SDNode *N,
3184 unsigned OpNo) {
3185 assert(OpNo == 1 && "Only Operand 1 must need promotion here");
3186 SDValue Op1 = N->getOperand(Num: 1);
3187 EVT RVT = Op1.getValueType();
3188 SDLoc dl(N);
3189
3190 EVT NVT = TLI.getTypeToTransformTo(Context&: *DAG.getContext(), VT: Op1.getValueType());
3191
3192 Op1 = GetSoftPromotedHalf(Op: Op1);
3193 Op1 = DAG.getNode(Opcode: GetPromotionOpcode(OpVT: RVT, RetVT: NVT), DL: dl, VT: NVT, Operand: Op1);
3194
3195 return DAG.getNode(Opcode: N->getOpcode(), DL: dl, VT: N->getValueType(ResNo: 0), N1: N->getOperand(Num: 0),
3196 N2: Op1);
3197}
3198
3199SDValue DAGTypeLegalizer::SoftPromoteHalfOp_FP_EXTEND(SDNode *N) {
3200 EVT RVT = N->getValueType(ResNo: 0);
3201 bool IsStrict = N->isStrictFPOpcode();
3202 SDValue Op = N->getOperand(Num: IsStrict ? 1 : 0);
3203 EVT SVT = Op.getValueType();
3204 Op = GetSoftPromotedHalf(Op: N->getOperand(Num: IsStrict ? 1 : 0));
3205
3206 if (IsStrict) {
3207 unsigned Opcode;
3208 if (SVT == MVT::f16)
3209 Opcode = ISD::STRICT_FP16_TO_FP;
3210 else if (SVT == MVT::bf16)
3211 Opcode = ISD::STRICT_BF16_TO_FP;
3212 else
3213 llvm_unreachable("unknown half type");
3214 SDValue Res =
3215 DAG.getNode(Opcode, SDLoc(N), {N->getValueType(0), MVT::Other},
3216 {N->getOperand(0), Op});
3217 ReplaceValueWith(From: SDValue(N, 1), To: Res.getValue(R: 1));
3218 ReplaceValueWith(From: SDValue(N, 0), To: Res);
3219 return SDValue();
3220 }
3221
3222 return DAG.getNode(Opcode: GetPromotionOpcode(OpVT: SVT, RetVT: RVT), DL: SDLoc(N), VT: RVT, Operand: Op);
3223}
3224
3225SDValue DAGTypeLegalizer::SoftPromoteHalfOp_FP_TO_XINT(SDNode *N) {
3226 EVT RVT = N->getValueType(ResNo: 0);
3227 SDValue Op = N->getOperand(Num: 0);
3228 EVT SVT = Op.getValueType();
3229 SDLoc dl(N);
3230
3231 EVT NVT = TLI.getTypeToTransformTo(Context&: *DAG.getContext(), VT: Op.getValueType());
3232
3233 Op = GetSoftPromotedHalf(Op);
3234
3235 SDValue Res = DAG.getNode(Opcode: GetPromotionOpcode(OpVT: SVT, RetVT: RVT), DL: dl, VT: NVT, Operand: Op);
3236
3237 return DAG.getNode(Opcode: N->getOpcode(), DL: dl, VT: N->getValueType(ResNo: 0), Operand: Res);
3238}
3239
3240SDValue DAGTypeLegalizer::SoftPromoteHalfOp_FP_TO_XINT_SAT(SDNode *N) {
3241 EVT RVT = N->getValueType(ResNo: 0);
3242 SDValue Op = N->getOperand(Num: 0);
3243 EVT SVT = Op.getValueType();
3244 SDLoc dl(N);
3245
3246 EVT NVT = TLI.getTypeToTransformTo(Context&: *DAG.getContext(), VT: Op.getValueType());
3247
3248 Op = GetSoftPromotedHalf(Op);
3249
3250 SDValue Res = DAG.getNode(Opcode: GetPromotionOpcode(OpVT: SVT, RetVT: RVT), DL: dl, VT: NVT, Operand: Op);
3251
3252 return DAG.getNode(Opcode: N->getOpcode(), DL: dl, VT: N->getValueType(ResNo: 0), N1: Res,
3253 N2: N->getOperand(Num: 1));
3254}
3255
3256SDValue DAGTypeLegalizer::SoftPromoteHalfOp_SELECT_CC(SDNode *N,
3257 unsigned OpNo) {
3258 assert(OpNo == 0 && "Can only soften the comparison values");
3259 SDValue Op0 = N->getOperand(Num: 0);
3260 SDValue Op1 = N->getOperand(Num: 1);
3261 SDLoc dl(N);
3262
3263 EVT SVT = Op0.getValueType();
3264 EVT NVT = TLI.getTypeToTransformTo(Context&: *DAG.getContext(), VT: SVT);
3265
3266 Op0 = GetSoftPromotedHalf(Op: Op0);
3267 Op1 = GetSoftPromotedHalf(Op: Op1);
3268
3269 // Promote to the larger FP type.
3270 auto PromotionOpcode = GetPromotionOpcode(OpVT: SVT, RetVT: NVT);
3271 Op0 = DAG.getNode(Opcode: PromotionOpcode, DL: dl, VT: NVT, Operand: Op0);
3272 Op1 = DAG.getNode(Opcode: PromotionOpcode, DL: dl, VT: NVT, Operand: Op1);
3273
3274 return DAG.getNode(Opcode: ISD::SELECT_CC, DL: SDLoc(N), VT: N->getValueType(ResNo: 0), N1: Op0, N2: Op1,
3275 N3: N->getOperand(Num: 2), N4: N->getOperand(Num: 3), N5: N->getOperand(Num: 4));
3276}
3277
3278SDValue DAGTypeLegalizer::SoftPromoteHalfOp_SETCC(SDNode *N) {
3279 SDValue Op0 = N->getOperand(Num: 0);
3280 SDValue Op1 = N->getOperand(Num: 1);
3281 ISD::CondCode CCCode = cast<CondCodeSDNode>(Val: N->getOperand(Num: 2))->get();
3282 SDLoc dl(N);
3283
3284 EVT SVT = Op0.getValueType();
3285 EVT NVT = TLI.getTypeToTransformTo(Context&: *DAG.getContext(), VT: Op0.getValueType());
3286
3287 Op0 = GetSoftPromotedHalf(Op: Op0);
3288 Op1 = GetSoftPromotedHalf(Op: Op1);
3289
3290 // Promote to the larger FP type.
3291 auto PromotionOpcode = GetPromotionOpcode(OpVT: SVT, RetVT: NVT);
3292 Op0 = DAG.getNode(Opcode: PromotionOpcode, DL: dl, VT: NVT, Operand: Op0);
3293 Op1 = DAG.getNode(Opcode: PromotionOpcode, DL: dl, VT: NVT, Operand: Op1);
3294
3295 return DAG.getSetCC(DL: SDLoc(N), VT: N->getValueType(ResNo: 0), LHS: Op0, RHS: Op1, Cond: CCCode);
3296}
3297
3298SDValue DAGTypeLegalizer::SoftPromoteHalfOp_STORE(SDNode *N, unsigned OpNo) {
3299 assert(OpNo == 1 && "Can only soften the stored value!");
3300 StoreSDNode *ST = cast<StoreSDNode>(Val: N);
3301 SDValue Val = ST->getValue();
3302 SDLoc dl(N);
3303
3304 assert(!ST->isTruncatingStore() && "Unexpected truncating store.");
3305 SDValue Promoted = GetSoftPromotedHalf(Op: Val);
3306 return DAG.getStore(Chain: ST->getChain(), dl, Val: Promoted, Ptr: ST->getBasePtr(),
3307 MMO: ST->getMemOperand());
3308}
3309
3310SDValue DAGTypeLegalizer::SoftPromoteHalfOp_STACKMAP(SDNode *N, unsigned OpNo) {
3311 assert(OpNo > 1); // Because the first two arguments are guaranteed legal.
3312 SmallVector<SDValue> NewOps(N->ops().begin(), N->ops().end());
3313 SDValue Op = N->getOperand(Num: OpNo);
3314 NewOps[OpNo] = GetSoftPromotedHalf(Op);
3315 SDValue NewNode =
3316 DAG.getNode(Opcode: N->getOpcode(), DL: SDLoc(N), VTList: N->getVTList(), Ops: NewOps);
3317
3318 for (unsigned ResNum = 0; ResNum < N->getNumValues(); ResNum++)
3319 ReplaceValueWith(From: SDValue(N, ResNum), To: NewNode.getValue(R: ResNum));
3320
3321 return SDValue(); // Signal that we replaced the node ourselves.
3322}
3323
3324SDValue DAGTypeLegalizer::SoftPromoteHalfOp_PATCHPOINT(SDNode *N,
3325 unsigned OpNo) {
3326 assert(OpNo >= 7);
3327 SmallVector<SDValue> NewOps(N->ops().begin(), N->ops().end());
3328 SDValue Op = N->getOperand(Num: OpNo);
3329 NewOps[OpNo] = GetSoftPromotedHalf(Op);
3330 SDValue NewNode =
3331 DAG.getNode(Opcode: N->getOpcode(), DL: SDLoc(N), VTList: N->getVTList(), Ops: NewOps);
3332
3333 for (unsigned ResNum = 0; ResNum < N->getNumValues(); ResNum++)
3334 ReplaceValueWith(From: SDValue(N, ResNum), To: NewNode.getValue(R: ResNum));
3335
3336 return SDValue(); // Signal that we replaced the node ourselves.
3337}
3338

source code of llvm/lib/CodeGen/SelectionDAG/LegalizeFloatTypes.cpp