1//===---- CGBuiltin.cpp - Emit LLVM Code for builtins ---------------------===//
2//
3// Part of the LLVM Project, under the Apache License v2.0 with LLVM Exceptions.
4// See https://llvm.org/LICENSE.txt for license information.
5// SPDX-License-Identifier: Apache-2.0 WITH LLVM-exception
6//
7//===----------------------------------------------------------------------===//
8//
9// This contains code to emit Builtin calls as LLVM code.
10//
11//===----------------------------------------------------------------------===//
12
13#include "CGBuiltin.h"
14#include "ABIInfo.h"
15#include "CGCUDARuntime.h"
16#include "CGCXXABI.h"
17#include "CGDebugInfo.h"
18#include "CGObjCRuntime.h"
19#include "CGOpenCLRuntime.h"
20#include "CGPointerAuthInfo.h"
21#include "CGRecordLayout.h"
22#include "CGValue.h"
23#include "CodeGenFunction.h"
24#include "CodeGenModule.h"
25#include "ConstantEmitter.h"
26#include "PatternInit.h"
27#include "TargetInfo.h"
28#include "clang/AST/OSLog.h"
29#include "clang/AST/StmtVisitor.h"
30#include "clang/Basic/TargetInfo.h"
31#include "clang/Frontend/FrontendDiagnostic.h"
32#include "llvm/IR/InlineAsm.h"
33#include "llvm/IR/Instruction.h"
34#include "llvm/IR/Intrinsics.h"
35#include "llvm/IR/IntrinsicsX86.h"
36#include "llvm/IR/MatrixBuilder.h"
37#include "llvm/Support/ConvertUTF.h"
38#include "llvm/Support/ScopedPrinter.h"
39#include <optional>
40#include <utility>
41
42using namespace clang;
43using namespace CodeGen;
44using namespace llvm;
45
46/// Some builtins do not have library implementation on some targets and
47/// are instead emitted as LLVM IRs by some target builtin emitters.
48/// FIXME: Remove this when library support is added
49static bool shouldEmitBuiltinAsIR(unsigned BuiltinID,
50 const Builtin::Context &BI,
51 const CodeGenFunction &CGF) {
52 if (!CGF.CGM.getLangOpts().MathErrno &&
53 CGF.CurFPFeatures.getExceptionMode() ==
54 LangOptions::FPExceptionModeKind::FPE_Ignore &&
55 !CGF.CGM.getTargetCodeGenInfo().supportsLibCall()) {
56 switch (BuiltinID) {
57 default:
58 return false;
59 case Builtin::BIlogbf:
60 case Builtin::BI__builtin_logbf:
61 case Builtin::BIlogb:
62 case Builtin::BI__builtin_logb:
63 case Builtin::BIscalbnf:
64 case Builtin::BI__builtin_scalbnf:
65 case Builtin::BIscalbn:
66 case Builtin::BI__builtin_scalbn:
67 return true;
68 }
69 }
70 return false;
71}
72
73static Value *EmitTargetArchBuiltinExpr(CodeGenFunction *CGF,
74 unsigned BuiltinID, const CallExpr *E,
75 ReturnValueSlot ReturnValue,
76 llvm::Triple::ArchType Arch) {
77 // When compiling in HipStdPar mode we have to be conservative in rejecting
78 // target specific features in the FE, and defer the possible error to the
79 // AcceleratorCodeSelection pass, wherein iff an unsupported target builtin is
80 // referenced by an accelerator executable function, we emit an error.
81 // Returning nullptr here leads to the builtin being handled in
82 // EmitStdParUnsupportedBuiltin.
83 if (CGF->getLangOpts().HIPStdPar && CGF->getLangOpts().CUDAIsDevice &&
84 Arch != CGF->getTarget().getTriple().getArch())
85 return nullptr;
86
87 switch (Arch) {
88 case llvm::Triple::arm:
89 case llvm::Triple::armeb:
90 case llvm::Triple::thumb:
91 case llvm::Triple::thumbeb:
92 return CGF->EmitARMBuiltinExpr(BuiltinID, E, ReturnValue, Arch);
93 case llvm::Triple::aarch64:
94 case llvm::Triple::aarch64_32:
95 case llvm::Triple::aarch64_be:
96 return CGF->EmitAArch64BuiltinExpr(BuiltinID, E, Arch);
97 case llvm::Triple::bpfeb:
98 case llvm::Triple::bpfel:
99 return CGF->EmitBPFBuiltinExpr(BuiltinID, E);
100 case llvm::Triple::dxil:
101 return CGF->EmitDirectXBuiltinExpr(BuiltinID, E);
102 case llvm::Triple::x86:
103 case llvm::Triple::x86_64:
104 return CGF->EmitX86BuiltinExpr(BuiltinID, E);
105 case llvm::Triple::ppc:
106 case llvm::Triple::ppcle:
107 case llvm::Triple::ppc64:
108 case llvm::Triple::ppc64le:
109 return CGF->EmitPPCBuiltinExpr(BuiltinID, E);
110 case llvm::Triple::r600:
111 case llvm::Triple::amdgcn:
112 return CGF->EmitAMDGPUBuiltinExpr(BuiltinID, E);
113 case llvm::Triple::systemz:
114 return CGF->EmitSystemZBuiltinExpr(BuiltinID, E);
115 case llvm::Triple::nvptx:
116 case llvm::Triple::nvptx64:
117 return CGF->EmitNVPTXBuiltinExpr(BuiltinID, E);
118 case llvm::Triple::wasm32:
119 case llvm::Triple::wasm64:
120 return CGF->EmitWebAssemblyBuiltinExpr(BuiltinID, E);
121 case llvm::Triple::hexagon:
122 return CGF->EmitHexagonBuiltinExpr(BuiltinID, E);
123 case llvm::Triple::riscv32:
124 case llvm::Triple::riscv64:
125 return CGF->EmitRISCVBuiltinExpr(BuiltinID, E, ReturnValue);
126 case llvm::Triple::spirv32:
127 case llvm::Triple::spirv64:
128 if (CGF->getTarget().getTriple().getOS() == llvm::Triple::OSType::AMDHSA)
129 return CGF->EmitAMDGPUBuiltinExpr(BuiltinID, E);
130 [[fallthrough]];
131 case llvm::Triple::spirv:
132 return CGF->EmitSPIRVBuiltinExpr(BuiltinID, E);
133 default:
134 return nullptr;
135 }
136}
137
138Value *CodeGenFunction::EmitTargetBuiltinExpr(unsigned BuiltinID,
139 const CallExpr *E,
140 ReturnValueSlot ReturnValue) {
141 if (getContext().BuiltinInfo.isAuxBuiltinID(ID: BuiltinID)) {
142 assert(getContext().getAuxTargetInfo() && "Missing aux target info");
143 return EmitTargetArchBuiltinExpr(
144 CGF: this, BuiltinID: getContext().BuiltinInfo.getAuxBuiltinID(ID: BuiltinID), E,
145 ReturnValue, Arch: getContext().getAuxTargetInfo()->getTriple().getArch());
146 }
147
148 return EmitTargetArchBuiltinExpr(CGF: this, BuiltinID, E, ReturnValue,
149 Arch: getTarget().getTriple().getArch());
150}
151
152static void initializeAlloca(CodeGenFunction &CGF, AllocaInst *AI, Value *Size,
153 Align AlignmentInBytes) {
154 ConstantInt *Byte;
155 switch (CGF.getLangOpts().getTrivialAutoVarInit()) {
156 case LangOptions::TrivialAutoVarInitKind::Uninitialized:
157 // Nothing to initialize.
158 return;
159 case LangOptions::TrivialAutoVarInitKind::Zero:
160 Byte = CGF.Builder.getInt8(C: 0x00);
161 break;
162 case LangOptions::TrivialAutoVarInitKind::Pattern: {
163 llvm::Type *Int8 = llvm::IntegerType::getInt8Ty(C&: CGF.CGM.getLLVMContext());
164 Byte = llvm::dyn_cast<llvm::ConstantInt>(
165 Val: initializationPatternFor(CGF.CGM, Int8));
166 break;
167 }
168 }
169 if (CGF.CGM.stopAutoInit())
170 return;
171 auto *I = CGF.Builder.CreateMemSet(Ptr: AI, Val: Byte, Size, Align: AlignmentInBytes);
172 I->addAnnotationMetadata(Annotation: "auto-init");
173}
174
175/// getBuiltinLibFunction - Given a builtin id for a function like
176/// "__builtin_fabsf", return a Function* for "fabsf".
177llvm::Constant *CodeGenModule::getBuiltinLibFunction(const FunctionDecl *FD,
178 unsigned BuiltinID) {
179 assert(Context.BuiltinInfo.isLibFunction(BuiltinID));
180
181 // Get the name, skip over the __builtin_ prefix (if necessary). We may have
182 // to build this up so provide a small stack buffer to handle the vast
183 // majority of names.
184 llvm::SmallString<64> Name;
185 GlobalDecl D(FD);
186
187 // TODO: This list should be expanded or refactored after all GCC-compatible
188 // std libcall builtins are implemented.
189 static SmallDenseMap<unsigned, StringRef, 64> F128Builtins{
190 {Builtin::BI__builtin___fprintf_chk, "__fprintf_chkieee128"},
191 {Builtin::BI__builtin___printf_chk, "__printf_chkieee128"},
192 {Builtin::BI__builtin___snprintf_chk, "__snprintf_chkieee128"},
193 {Builtin::BI__builtin___sprintf_chk, "__sprintf_chkieee128"},
194 {Builtin::BI__builtin___vfprintf_chk, "__vfprintf_chkieee128"},
195 {Builtin::BI__builtin___vprintf_chk, "__vprintf_chkieee128"},
196 {Builtin::BI__builtin___vsnprintf_chk, "__vsnprintf_chkieee128"},
197 {Builtin::BI__builtin___vsprintf_chk, "__vsprintf_chkieee128"},
198 {Builtin::BI__builtin_fprintf, "__fprintfieee128"},
199 {Builtin::BI__builtin_printf, "__printfieee128"},
200 {Builtin::BI__builtin_snprintf, "__snprintfieee128"},
201 {Builtin::BI__builtin_sprintf, "__sprintfieee128"},
202 {Builtin::BI__builtin_vfprintf, "__vfprintfieee128"},
203 {Builtin::BI__builtin_vprintf, "__vprintfieee128"},
204 {Builtin::BI__builtin_vsnprintf, "__vsnprintfieee128"},
205 {Builtin::BI__builtin_vsprintf, "__vsprintfieee128"},
206 {Builtin::BI__builtin_fscanf, "__fscanfieee128"},
207 {Builtin::BI__builtin_scanf, "__scanfieee128"},
208 {Builtin::BI__builtin_sscanf, "__sscanfieee128"},
209 {Builtin::BI__builtin_vfscanf, "__vfscanfieee128"},
210 {Builtin::BI__builtin_vscanf, "__vscanfieee128"},
211 {Builtin::BI__builtin_vsscanf, "__vsscanfieee128"},
212 {Builtin::BI__builtin_nexttowardf128, "__nexttowardieee128"},
213 };
214
215 // The AIX library functions frexpl, ldexpl, and modfl are for 128-bit
216 // IBM 'long double' (i.e. __ibm128). Map to the 'double' versions
217 // if it is 64-bit 'long double' mode.
218 static SmallDenseMap<unsigned, StringRef, 4> AIXLongDouble64Builtins{
219 {Builtin::BI__builtin_frexpl, "frexp"},
220 {Builtin::BI__builtin_ldexpl, "ldexp"},
221 {Builtin::BI__builtin_modfl, "modf"},
222 };
223
224 // If the builtin has been declared explicitly with an assembler label,
225 // use the mangled name. This differs from the plain label on platforms
226 // that prefix labels.
227 if (FD->hasAttr<AsmLabelAttr>())
228 Name = getMangledName(GD: D);
229 else {
230 // TODO: This mutation should also be applied to other targets other than
231 // PPC, after backend supports IEEE 128-bit style libcalls.
232 if (getTriple().isPPC64() &&
233 &getTarget().getLongDoubleFormat() == &llvm::APFloat::IEEEquad() &&
234 F128Builtins.contains(Val: BuiltinID))
235 Name = F128Builtins[BuiltinID];
236 else if (getTriple().isOSAIX() &&
237 &getTarget().getLongDoubleFormat() ==
238 &llvm::APFloat::IEEEdouble() &&
239 AIXLongDouble64Builtins.contains(Val: BuiltinID))
240 Name = AIXLongDouble64Builtins[BuiltinID];
241 else
242 Name = Context.BuiltinInfo.getName(ID: BuiltinID).substr(pos: 10);
243 }
244
245 llvm::FunctionType *Ty =
246 cast<llvm::FunctionType>(getTypes().ConvertType(T: FD->getType()));
247
248 return GetOrCreateLLVMFunction(MangledName: Name, Ty, D, /*ForVTable=*/false);
249}
250
251/// Emit the conversions required to turn the given value into an
252/// integer of the given size.
253Value *EmitToInt(CodeGenFunction &CGF, llvm::Value *V,
254 QualType T, llvm::IntegerType *IntType) {
255 V = CGF.EmitToMemory(Value: V, Ty: T);
256
257 if (V->getType()->isPointerTy())
258 return CGF.Builder.CreatePtrToInt(V, DestTy: IntType);
259
260 assert(V->getType() == IntType);
261 return V;
262}
263
264Value *EmitFromInt(CodeGenFunction &CGF, llvm::Value *V,
265 QualType T, llvm::Type *ResultType) {
266 V = CGF.EmitFromMemory(Value: V, Ty: T);
267
268 if (ResultType->isPointerTy())
269 return CGF.Builder.CreateIntToPtr(V, DestTy: ResultType);
270
271 assert(V->getType() == ResultType);
272 return V;
273}
274
275Address CheckAtomicAlignment(CodeGenFunction &CGF, const CallExpr *E) {
276 ASTContext &Ctx = CGF.getContext();
277 Address Ptr = CGF.EmitPointerWithAlignment(Addr: E->getArg(Arg: 0));
278 const llvm::DataLayout &DL = CGF.CGM.getDataLayout();
279 unsigned Bytes = Ptr.getElementType()->isPointerTy()
280 ? Ctx.getTypeSizeInChars(Ctx.VoidPtrTy).getQuantity()
281 : DL.getTypeStoreSize(Ty: Ptr.getElementType());
282 unsigned Align = Ptr.getAlignment().getQuantity();
283 if (Align % Bytes != 0) {
284 DiagnosticsEngine &Diags = CGF.CGM.getDiags();
285 Diags.Report(E->getBeginLoc(), diag::warn_sync_op_misaligned);
286 // Force address to be at least naturally-aligned.
287 return Ptr.withAlignment(NewAlignment: CharUnits::fromQuantity(Quantity: Bytes));
288 }
289 return Ptr;
290}
291
292/// Utility to insert an atomic instruction based on Intrinsic::ID
293/// and the expression node.
294Value *MakeBinaryAtomicValue(
295 CodeGenFunction &CGF, llvm::AtomicRMWInst::BinOp Kind, const CallExpr *E,
296 AtomicOrdering Ordering) {
297
298 QualType T = E->getType();
299 assert(E->getArg(0)->getType()->isPointerType());
300 assert(CGF.getContext().hasSameUnqualifiedType(T,
301 E->getArg(0)->getType()->getPointeeType()));
302 assert(CGF.getContext().hasSameUnqualifiedType(T, E->getArg(1)->getType()));
303
304 Address DestAddr = CheckAtomicAlignment(CGF, E);
305
306 llvm::IntegerType *IntType = llvm::IntegerType::get(
307 C&: CGF.getLLVMContext(), NumBits: CGF.getContext().getTypeSize(T));
308
309 llvm::Value *Val = CGF.EmitScalarExpr(E: E->getArg(Arg: 1));
310 llvm::Type *ValueType = Val->getType();
311 Val = EmitToInt(CGF, V: Val, T, IntType);
312
313 llvm::Value *Result =
314 CGF.Builder.CreateAtomicRMW(Op: Kind, Addr: DestAddr, Val, Ordering);
315 return EmitFromInt(CGF, V: Result, T, ResultType: ValueType);
316}
317
318static Value *EmitNontemporalStore(CodeGenFunction &CGF, const CallExpr *E) {
319 Value *Val = CGF.EmitScalarExpr(E: E->getArg(Arg: 0));
320 Address Addr = CGF.EmitPointerWithAlignment(Addr: E->getArg(Arg: 1));
321
322 Val = CGF.EmitToMemory(Value: Val, Ty: E->getArg(Arg: 0)->getType());
323 LValue LV = CGF.MakeAddrLValue(Addr, T: E->getArg(Arg: 0)->getType());
324 LV.setNontemporal(true);
325 CGF.EmitStoreOfScalar(value: Val, lvalue: LV, isInit: false);
326 return nullptr;
327}
328
329static Value *EmitNontemporalLoad(CodeGenFunction &CGF, const CallExpr *E) {
330 Address Addr = CGF.EmitPointerWithAlignment(Addr: E->getArg(Arg: 0));
331
332 LValue LV = CGF.MakeAddrLValue(Addr, E->getType());
333 LV.setNontemporal(true);
334 return CGF.EmitLoadOfScalar(LV, E->getExprLoc());
335}
336
337static RValue EmitBinaryAtomic(CodeGenFunction &CGF,
338 llvm::AtomicRMWInst::BinOp Kind,
339 const CallExpr *E) {
340 return RValue::get(V: MakeBinaryAtomicValue(CGF, Kind, E));
341}
342
343/// Utility to insert an atomic instruction based Intrinsic::ID and
344/// the expression node, where the return value is the result of the
345/// operation.
346static RValue EmitBinaryAtomicPost(CodeGenFunction &CGF,
347 llvm::AtomicRMWInst::BinOp Kind,
348 const CallExpr *E,
349 Instruction::BinaryOps Op,
350 bool Invert = false) {
351 QualType T = E->getType();
352 assert(E->getArg(0)->getType()->isPointerType());
353 assert(CGF.getContext().hasSameUnqualifiedType(T,
354 E->getArg(0)->getType()->getPointeeType()));
355 assert(CGF.getContext().hasSameUnqualifiedType(T, E->getArg(1)->getType()));
356
357 Address DestAddr = CheckAtomicAlignment(CGF, E);
358
359 llvm::IntegerType *IntType = llvm::IntegerType::get(
360 C&: CGF.getLLVMContext(), NumBits: CGF.getContext().getTypeSize(T));
361
362 llvm::Value *Val = CGF.EmitScalarExpr(E: E->getArg(Arg: 1));
363 llvm::Type *ValueType = Val->getType();
364 Val = EmitToInt(CGF, V: Val, T, IntType);
365
366 llvm::Value *Result = CGF.Builder.CreateAtomicRMW(
367 Op: Kind, Addr: DestAddr, Val, Ordering: llvm::AtomicOrdering::SequentiallyConsistent);
368 Result = CGF.Builder.CreateBinOp(Opc: Op, LHS: Result, RHS: Val);
369 if (Invert)
370 Result =
371 CGF.Builder.CreateBinOp(Opc: llvm::Instruction::Xor, LHS: Result,
372 RHS: llvm::ConstantInt::getAllOnesValue(Ty: IntType));
373 Result = EmitFromInt(CGF, V: Result, T, ResultType: ValueType);
374 return RValue::get(V: Result);
375}
376
377/// Utility to insert an atomic cmpxchg instruction.
378///
379/// @param CGF The current codegen function.
380/// @param E Builtin call expression to convert to cmpxchg.
381/// arg0 - address to operate on
382/// arg1 - value to compare with
383/// arg2 - new value
384/// @param ReturnBool Specifies whether to return success flag of
385/// cmpxchg result or the old value.
386///
387/// @returns result of cmpxchg, according to ReturnBool
388///
389/// Note: In order to lower Microsoft's _InterlockedCompareExchange* intrinsics
390/// invoke the function EmitAtomicCmpXchgForMSIntrin.
391Value *MakeAtomicCmpXchgValue(CodeGenFunction &CGF, const CallExpr *E,
392 bool ReturnBool) {
393 QualType T = ReturnBool ? E->getArg(Arg: 1)->getType() : E->getType();
394 Address DestAddr = CheckAtomicAlignment(CGF, E);
395
396 llvm::IntegerType *IntType = llvm::IntegerType::get(
397 C&: CGF.getLLVMContext(), NumBits: CGF.getContext().getTypeSize(T));
398
399 Value *Cmp = CGF.EmitScalarExpr(E: E->getArg(Arg: 1));
400 llvm::Type *ValueType = Cmp->getType();
401 Cmp = EmitToInt(CGF, V: Cmp, T, IntType);
402 Value *New = EmitToInt(CGF, V: CGF.EmitScalarExpr(E: E->getArg(Arg: 2)), T, IntType);
403
404 Value *Pair = CGF.Builder.CreateAtomicCmpXchg(
405 Addr: DestAddr, Cmp, New, SuccessOrdering: llvm::AtomicOrdering::SequentiallyConsistent,
406 FailureOrdering: llvm::AtomicOrdering::SequentiallyConsistent);
407 if (ReturnBool)
408 // Extract boolean success flag and zext it to int.
409 return CGF.Builder.CreateZExt(V: CGF.Builder.CreateExtractValue(Agg: Pair, Idxs: 1),
410 DestTy: CGF.ConvertType(E->getType()));
411 else
412 // Extract old value and emit it using the same type as compare value.
413 return EmitFromInt(CGF, V: CGF.Builder.CreateExtractValue(Agg: Pair, Idxs: 0), T,
414 ResultType: ValueType);
415}
416
417/// This function should be invoked to emit atomic cmpxchg for Microsoft's
418/// _InterlockedCompareExchange* intrinsics which have the following signature:
419/// T _InterlockedCompareExchange(T volatile *Destination,
420/// T Exchange,
421/// T Comparand);
422///
423/// Whereas the llvm 'cmpxchg' instruction has the following syntax:
424/// cmpxchg *Destination, Comparand, Exchange.
425/// So we need to swap Comparand and Exchange when invoking
426/// CreateAtomicCmpXchg. That is the reason we could not use the above utility
427/// function MakeAtomicCmpXchgValue since it expects the arguments to be
428/// already swapped.
429
430static
431Value *EmitAtomicCmpXchgForMSIntrin(CodeGenFunction &CGF, const CallExpr *E,
432 AtomicOrdering SuccessOrdering = AtomicOrdering::SequentiallyConsistent) {
433 assert(E->getArg(0)->getType()->isPointerType());
434 assert(CGF.getContext().hasSameUnqualifiedType(
435 E->getType(), E->getArg(0)->getType()->getPointeeType()));
436 assert(CGF.getContext().hasSameUnqualifiedType(E->getType(),
437 E->getArg(1)->getType()));
438 assert(CGF.getContext().hasSameUnqualifiedType(E->getType(),
439 E->getArg(2)->getType()));
440
441 Address DestAddr = CheckAtomicAlignment(CGF, E);
442
443 auto *Exchange = CGF.EmitScalarExpr(E: E->getArg(Arg: 1));
444 auto *RTy = Exchange->getType();
445
446 auto *Comparand = CGF.EmitScalarExpr(E: E->getArg(Arg: 2));
447
448 if (RTy->isPointerTy()) {
449 Exchange = CGF.Builder.CreatePtrToInt(V: Exchange, DestTy: CGF.IntPtrTy);
450 Comparand = CGF.Builder.CreatePtrToInt(V: Comparand, DestTy: CGF.IntPtrTy);
451 }
452
453 // For Release ordering, the failure ordering should be Monotonic.
454 auto FailureOrdering = SuccessOrdering == AtomicOrdering::Release ?
455 AtomicOrdering::Monotonic :
456 SuccessOrdering;
457
458 // The atomic instruction is marked volatile for consistency with MSVC. This
459 // blocks the few atomics optimizations that LLVM has. If we want to optimize
460 // _Interlocked* operations in the future, we will have to remove the volatile
461 // marker.
462 auto *CmpXchg = CGF.Builder.CreateAtomicCmpXchg(
463 Addr: DestAddr, Cmp: Comparand, New: Exchange, SuccessOrdering, FailureOrdering);
464 CmpXchg->setVolatile(true);
465
466 auto *Result = CGF.Builder.CreateExtractValue(Agg: CmpXchg, Idxs: 0);
467 if (RTy->isPointerTy()) {
468 Result = CGF.Builder.CreateIntToPtr(V: Result, DestTy: RTy);
469 }
470
471 return Result;
472}
473
474// 64-bit Microsoft platforms support 128 bit cmpxchg operations. They are
475// prototyped like this:
476//
477// unsigned char _InterlockedCompareExchange128...(
478// __int64 volatile * _Destination,
479// __int64 _ExchangeHigh,
480// __int64 _ExchangeLow,
481// __int64 * _ComparandResult);
482//
483// Note that Destination is assumed to be at least 16-byte aligned, despite
484// being typed int64.
485
486static Value *EmitAtomicCmpXchg128ForMSIntrin(CodeGenFunction &CGF,
487 const CallExpr *E,
488 AtomicOrdering SuccessOrdering) {
489 assert(E->getNumArgs() == 4);
490 llvm::Value *DestPtr = CGF.EmitScalarExpr(E: E->getArg(Arg: 0));
491 llvm::Value *ExchangeHigh = CGF.EmitScalarExpr(E: E->getArg(Arg: 1));
492 llvm::Value *ExchangeLow = CGF.EmitScalarExpr(E: E->getArg(Arg: 2));
493 Address ComparandAddr = CGF.EmitPointerWithAlignment(Addr: E->getArg(Arg: 3));
494
495 assert(DestPtr->getType()->isPointerTy());
496 assert(!ExchangeHigh->getType()->isPointerTy());
497 assert(!ExchangeLow->getType()->isPointerTy());
498
499 // For Release ordering, the failure ordering should be Monotonic.
500 auto FailureOrdering = SuccessOrdering == AtomicOrdering::Release
501 ? AtomicOrdering::Monotonic
502 : SuccessOrdering;
503
504 // Convert to i128 pointers and values. Alignment is also overridden for
505 // destination pointer.
506 llvm::Type *Int128Ty = llvm::IntegerType::get(C&: CGF.getLLVMContext(), NumBits: 128);
507 Address DestAddr(DestPtr, Int128Ty,
508 CGF.getContext().toCharUnitsFromBits(BitSize: 128));
509 ComparandAddr = ComparandAddr.withElementType(ElemTy: Int128Ty);
510
511 // (((i128)hi) << 64) | ((i128)lo)
512 ExchangeHigh = CGF.Builder.CreateZExt(V: ExchangeHigh, DestTy: Int128Ty);
513 ExchangeLow = CGF.Builder.CreateZExt(V: ExchangeLow, DestTy: Int128Ty);
514 ExchangeHigh =
515 CGF.Builder.CreateShl(LHS: ExchangeHigh, RHS: llvm::ConstantInt::get(Ty: Int128Ty, V: 64));
516 llvm::Value *Exchange = CGF.Builder.CreateOr(LHS: ExchangeHigh, RHS: ExchangeLow);
517
518 // Load the comparand for the instruction.
519 llvm::Value *Comparand = CGF.Builder.CreateLoad(Addr: ComparandAddr);
520
521 auto *CXI = CGF.Builder.CreateAtomicCmpXchg(Addr: DestAddr, Cmp: Comparand, New: Exchange,
522 SuccessOrdering, FailureOrdering);
523
524 // The atomic instruction is marked volatile for consistency with MSVC. This
525 // blocks the few atomics optimizations that LLVM has. If we want to optimize
526 // _Interlocked* operations in the future, we will have to remove the volatile
527 // marker.
528 CXI->setVolatile(true);
529
530 // Store the result as an outparameter.
531 CGF.Builder.CreateStore(Val: CGF.Builder.CreateExtractValue(Agg: CXI, Idxs: 0),
532 Addr: ComparandAddr);
533
534 // Get the success boolean and zero extend it to i8.
535 Value *Success = CGF.Builder.CreateExtractValue(Agg: CXI, Idxs: 1);
536 return CGF.Builder.CreateZExt(V: Success, DestTy: CGF.Int8Ty);
537}
538
539static Value *EmitAtomicIncrementValue(CodeGenFunction &CGF, const CallExpr *E,
540 AtomicOrdering Ordering = AtomicOrdering::SequentiallyConsistent) {
541 assert(E->getArg(0)->getType()->isPointerType());
542
543 auto *IntTy = CGF.ConvertType(E->getType());
544 Address DestAddr = CheckAtomicAlignment(CGF, E);
545 auto *Result = CGF.Builder.CreateAtomicRMW(
546 Op: AtomicRMWInst::Add, Addr: DestAddr, Val: ConstantInt::get(IntTy, 1), Ordering);
547 return CGF.Builder.CreateAdd(LHS: Result, RHS: ConstantInt::get(IntTy, 1));
548}
549
550static Value *EmitAtomicDecrementValue(
551 CodeGenFunction &CGF, const CallExpr *E,
552 AtomicOrdering Ordering = AtomicOrdering::SequentiallyConsistent) {
553 assert(E->getArg(0)->getType()->isPointerType());
554
555 auto *IntTy = CGF.ConvertType(E->getType());
556 Address DestAddr = CheckAtomicAlignment(CGF, E);
557 auto *Result = CGF.Builder.CreateAtomicRMW(
558 Op: AtomicRMWInst::Sub, Addr: DestAddr, Val: ConstantInt::get(IntTy, 1), Ordering);
559 return CGF.Builder.CreateSub(LHS: Result, RHS: ConstantInt::get(IntTy, 1));
560}
561
562// Build a plain volatile load.
563static Value *EmitISOVolatileLoad(CodeGenFunction &CGF, const CallExpr *E) {
564 Value *Ptr = CGF.EmitScalarExpr(E: E->getArg(Arg: 0));
565 QualType ElTy = E->getArg(Arg: 0)->getType()->getPointeeType();
566 CharUnits LoadSize = CGF.getContext().getTypeSizeInChars(T: ElTy);
567 llvm::Type *ITy =
568 llvm::IntegerType::get(C&: CGF.getLLVMContext(), NumBits: LoadSize.getQuantity() * 8);
569 llvm::LoadInst *Load = CGF.Builder.CreateAlignedLoad(Ty: ITy, Addr: Ptr, Align: LoadSize);
570 Load->setVolatile(true);
571 return Load;
572}
573
574// Build a plain volatile store.
575static Value *EmitISOVolatileStore(CodeGenFunction &CGF, const CallExpr *E) {
576 Value *Ptr = CGF.EmitScalarExpr(E: E->getArg(Arg: 0));
577 Value *Value = CGF.EmitScalarExpr(E: E->getArg(Arg: 1));
578 QualType ElTy = E->getArg(Arg: 0)->getType()->getPointeeType();
579 CharUnits StoreSize = CGF.getContext().getTypeSizeInChars(T: ElTy);
580 llvm::StoreInst *Store =
581 CGF.Builder.CreateAlignedStore(Val: Value, Addr: Ptr, Align: StoreSize);
582 Store->setVolatile(true);
583 return Store;
584}
585
586// Emit a simple mangled intrinsic that has 1 argument and a return type
587// matching the argument type. Depending on mode, this may be a constrained
588// floating-point intrinsic.
589Value *emitUnaryMaybeConstrainedFPBuiltin(CodeGenFunction &CGF,
590 const CallExpr *E, unsigned IntrinsicID,
591 unsigned ConstrainedIntrinsicID) {
592 llvm::Value *Src0 = CGF.EmitScalarExpr(E: E->getArg(Arg: 0));
593
594 CodeGenFunction::CGFPOptionsRAII FPOptsRAII(CGF, E);
595 if (CGF.Builder.getIsFPConstrained()) {
596 Function *F = CGF.CGM.getIntrinsic(IID: ConstrainedIntrinsicID, Tys: Src0->getType());
597 return CGF.Builder.CreateConstrainedFPCall(Callee: F, Args: { Src0 });
598 } else {
599 Function *F = CGF.CGM.getIntrinsic(IID: IntrinsicID, Tys: Src0->getType());
600 return CGF.Builder.CreateCall(Callee: F, Args: Src0);
601 }
602}
603
604// Emit an intrinsic that has 2 operands of the same type as its result.
605// Depending on mode, this may be a constrained floating-point intrinsic.
606static Value *emitBinaryMaybeConstrainedFPBuiltin(CodeGenFunction &CGF,
607 const CallExpr *E, unsigned IntrinsicID,
608 unsigned ConstrainedIntrinsicID) {
609 llvm::Value *Src0 = CGF.EmitScalarExpr(E: E->getArg(Arg: 0));
610 llvm::Value *Src1 = CGF.EmitScalarExpr(E: E->getArg(Arg: 1));
611
612 CodeGenFunction::CGFPOptionsRAII FPOptsRAII(CGF, E);
613 if (CGF.Builder.getIsFPConstrained()) {
614 Function *F = CGF.CGM.getIntrinsic(IID: ConstrainedIntrinsicID, Tys: Src0->getType());
615 return CGF.Builder.CreateConstrainedFPCall(Callee: F, Args: { Src0, Src1 });
616 } else {
617 Function *F = CGF.CGM.getIntrinsic(IID: IntrinsicID, Tys: Src0->getType());
618 return CGF.Builder.CreateCall(Callee: F, Args: { Src0, Src1 });
619 }
620}
621
622// Has second type mangled argument.
623static Value *
624emitBinaryExpMaybeConstrainedFPBuiltin(CodeGenFunction &CGF, const CallExpr *E,
625 Intrinsic::ID IntrinsicID,
626 Intrinsic::ID ConstrainedIntrinsicID) {
627 llvm::Value *Src0 = CGF.EmitScalarExpr(E: E->getArg(Arg: 0));
628 llvm::Value *Src1 = CGF.EmitScalarExpr(E: E->getArg(Arg: 1));
629
630 CodeGenFunction::CGFPOptionsRAII FPOptsRAII(CGF, E);
631 if (CGF.Builder.getIsFPConstrained()) {
632 Function *F = CGF.CGM.getIntrinsic(IID: ConstrainedIntrinsicID,
633 Tys: {Src0->getType(), Src1->getType()});
634 return CGF.Builder.CreateConstrainedFPCall(Callee: F, Args: {Src0, Src1});
635 }
636
637 Function *F =
638 CGF.CGM.getIntrinsic(IID: IntrinsicID, Tys: {Src0->getType(), Src1->getType()});
639 return CGF.Builder.CreateCall(Callee: F, Args: {Src0, Src1});
640}
641
642// Emit an intrinsic that has 3 operands of the same type as its result.
643// Depending on mode, this may be a constrained floating-point intrinsic.
644static Value *emitTernaryMaybeConstrainedFPBuiltin(CodeGenFunction &CGF,
645 const CallExpr *E, unsigned IntrinsicID,
646 unsigned ConstrainedIntrinsicID) {
647 llvm::Value *Src0 = CGF.EmitScalarExpr(E: E->getArg(Arg: 0));
648 llvm::Value *Src1 = CGF.EmitScalarExpr(E: E->getArg(Arg: 1));
649 llvm::Value *Src2 = CGF.EmitScalarExpr(E: E->getArg(Arg: 2));
650
651 CodeGenFunction::CGFPOptionsRAII FPOptsRAII(CGF, E);
652 if (CGF.Builder.getIsFPConstrained()) {
653 Function *F = CGF.CGM.getIntrinsic(IID: ConstrainedIntrinsicID, Tys: Src0->getType());
654 return CGF.Builder.CreateConstrainedFPCall(Callee: F, Args: { Src0, Src1, Src2 });
655 } else {
656 Function *F = CGF.CGM.getIntrinsic(IID: IntrinsicID, Tys: Src0->getType());
657 return CGF.Builder.CreateCall(Callee: F, Args: { Src0, Src1, Src2 });
658 }
659}
660
661// Emit an intrinsic that has overloaded integer result and fp operand.
662static Value *
663emitMaybeConstrainedFPToIntRoundBuiltin(CodeGenFunction &CGF, const CallExpr *E,
664 unsigned IntrinsicID,
665 unsigned ConstrainedIntrinsicID) {
666 llvm::Type *ResultType = CGF.ConvertType(E->getType());
667 llvm::Value *Src0 = CGF.EmitScalarExpr(E: E->getArg(Arg: 0));
668
669 if (CGF.Builder.getIsFPConstrained()) {
670 CodeGenFunction::CGFPOptionsRAII FPOptsRAII(CGF, E);
671 Function *F = CGF.CGM.getIntrinsic(IID: ConstrainedIntrinsicID,
672 Tys: {ResultType, Src0->getType()});
673 return CGF.Builder.CreateConstrainedFPCall(Callee: F, Args: {Src0});
674 } else {
675 Function *F =
676 CGF.CGM.getIntrinsic(IID: IntrinsicID, Tys: {ResultType, Src0->getType()});
677 return CGF.Builder.CreateCall(Callee: F, Args: Src0);
678 }
679}
680
681static Value *emitFrexpBuiltin(CodeGenFunction &CGF, const CallExpr *E,
682 Intrinsic::ID IntrinsicID) {
683 llvm::Value *Src0 = CGF.EmitScalarExpr(E: E->getArg(Arg: 0));
684 llvm::Value *Src1 = CGF.EmitScalarExpr(E: E->getArg(Arg: 1));
685
686 QualType IntPtrTy = E->getArg(Arg: 1)->getType()->getPointeeType();
687 llvm::Type *IntTy = CGF.ConvertType(T: IntPtrTy);
688 llvm::Function *F =
689 CGF.CGM.getIntrinsic(IID: IntrinsicID, Tys: {Src0->getType(), IntTy});
690 llvm::Value *Call = CGF.Builder.CreateCall(Callee: F, Args: Src0);
691
692 llvm::Value *Exp = CGF.Builder.CreateExtractValue(Agg: Call, Idxs: 1);
693 LValue LV = CGF.MakeNaturalAlignAddrLValue(V: Src1, T: IntPtrTy);
694 CGF.EmitStoreOfScalar(value: Exp, lvalue: LV);
695
696 return CGF.Builder.CreateExtractValue(Agg: Call, Idxs: 0);
697}
698
699static void emitSincosBuiltin(CodeGenFunction &CGF, const CallExpr *E,
700 Intrinsic::ID IntrinsicID) {
701 llvm::Value *Val = CGF.EmitScalarExpr(E: E->getArg(Arg: 0));
702 llvm::Value *Dest0 = CGF.EmitScalarExpr(E: E->getArg(Arg: 1));
703 llvm::Value *Dest1 = CGF.EmitScalarExpr(E: E->getArg(Arg: 2));
704
705 llvm::Function *F = CGF.CGM.getIntrinsic(IID: IntrinsicID, Tys: {Val->getType()});
706 llvm::Value *Call = CGF.Builder.CreateCall(Callee: F, Args: Val);
707
708 llvm::Value *SinResult = CGF.Builder.CreateExtractValue(Agg: Call, Idxs: 0);
709 llvm::Value *CosResult = CGF.Builder.CreateExtractValue(Agg: Call, Idxs: 1);
710
711 QualType DestPtrType = E->getArg(Arg: 1)->getType()->getPointeeType();
712 LValue SinLV = CGF.MakeNaturalAlignAddrLValue(V: Dest0, T: DestPtrType);
713 LValue CosLV = CGF.MakeNaturalAlignAddrLValue(V: Dest1, T: DestPtrType);
714
715 llvm::StoreInst *StoreSin =
716 CGF.Builder.CreateStore(Val: SinResult, Addr: SinLV.getAddress());
717 llvm::StoreInst *StoreCos =
718 CGF.Builder.CreateStore(Val: CosResult, Addr: CosLV.getAddress());
719
720 // Mark the two stores as non-aliasing with each other. The order of stores
721 // emitted by this builtin is arbitrary, enforcing a particular order will
722 // prevent optimizations later on.
723 llvm::MDBuilder MDHelper(CGF.getLLVMContext());
724 MDNode *Domain = MDHelper.createAnonymousAliasScopeDomain();
725 MDNode *AliasScope = MDHelper.createAnonymousAliasScope(Domain);
726 MDNode *AliasScopeList = MDNode::get(Context&: Call->getContext(), MDs: AliasScope);
727 StoreSin->setMetadata(KindID: LLVMContext::MD_alias_scope, Node: AliasScopeList);
728 StoreCos->setMetadata(KindID: LLVMContext::MD_noalias, Node: AliasScopeList);
729}
730
731static llvm::Value *emitModfBuiltin(CodeGenFunction &CGF, const CallExpr *E,
732 Intrinsic::ID IntrinsicID) {
733 llvm::Value *Val = CGF.EmitScalarExpr(E: E->getArg(Arg: 0));
734 llvm::Value *IntPartDest = CGF.EmitScalarExpr(E: E->getArg(Arg: 1));
735
736 llvm::Value *Call =
737 CGF.Builder.CreateIntrinsic(ID: IntrinsicID, Types: {Val->getType()}, Args: Val);
738
739 llvm::Value *FractionalResult = CGF.Builder.CreateExtractValue(Agg: Call, Idxs: 0);
740 llvm::Value *IntegralResult = CGF.Builder.CreateExtractValue(Agg: Call, Idxs: 1);
741
742 QualType DestPtrType = E->getArg(Arg: 1)->getType()->getPointeeType();
743 LValue IntegralLV = CGF.MakeNaturalAlignAddrLValue(V: IntPartDest, T: DestPtrType);
744 CGF.EmitStoreOfScalar(value: IntegralResult, lvalue: IntegralLV);
745
746 return FractionalResult;
747}
748
749/// EmitFAbs - Emit a call to @llvm.fabs().
750static Value *EmitFAbs(CodeGenFunction &CGF, Value *V) {
751 Function *F = CGF.CGM.getIntrinsic(Intrinsic::fabs, V->getType());
752 llvm::CallInst *Call = CGF.Builder.CreateCall(Callee: F, Args: V);
753 Call->setDoesNotAccessMemory();
754 return Call;
755}
756
757/// Emit the computation of the sign bit for a floating point value. Returns
758/// the i1 sign bit value.
759static Value *EmitSignBit(CodeGenFunction &CGF, Value *V) {
760 LLVMContext &C = CGF.CGM.getLLVMContext();
761
762 llvm::Type *Ty = V->getType();
763 int Width = Ty->getPrimitiveSizeInBits();
764 llvm::Type *IntTy = llvm::IntegerType::get(C, NumBits: Width);
765 V = CGF.Builder.CreateBitCast(V, DestTy: IntTy);
766 if (Ty->isPPC_FP128Ty()) {
767 // We want the sign bit of the higher-order double. The bitcast we just
768 // did works as if the double-double was stored to memory and then
769 // read as an i128. The "store" will put the higher-order double in the
770 // lower address in both little- and big-Endian modes, but the "load"
771 // will treat those bits as a different part of the i128: the low bits in
772 // little-Endian, the high bits in big-Endian. Therefore, on big-Endian
773 // we need to shift the high bits down to the low before truncating.
774 Width >>= 1;
775 if (CGF.getTarget().isBigEndian()) {
776 Value *ShiftCst = llvm::ConstantInt::get(Ty: IntTy, V: Width);
777 V = CGF.Builder.CreateLShr(LHS: V, RHS: ShiftCst);
778 }
779 // We are truncating value in order to extract the higher-order
780 // double, which we will be using to extract the sign from.
781 IntTy = llvm::IntegerType::get(C, NumBits: Width);
782 V = CGF.Builder.CreateTrunc(V, DestTy: IntTy);
783 }
784 Value *Zero = llvm::Constant::getNullValue(Ty: IntTy);
785 return CGF.Builder.CreateICmpSLT(LHS: V, RHS: Zero);
786}
787
788/// Checks no arguments or results are passed indirectly in the ABI (i.e. via a
789/// hidden pointer). This is used to check annotating FP libcalls (that could
790/// set `errno`) with "int" TBAA metadata is safe. If any floating-point
791/// arguments are passed indirectly, setup for the call could be incorrectly
792/// optimized out.
793static bool HasNoIndirectArgumentsOrResults(CGFunctionInfo const &FnInfo) {
794 auto IsIndirect = [&](ABIArgInfo const &info) {
795 return info.isIndirect() || info.isIndirectAliased() || info.isInAlloca();
796 };
797 return !IsIndirect(FnInfo.getReturnInfo()) &&
798 llvm::none_of(Range: FnInfo.arguments(),
799 P: [&](CGFunctionInfoArgInfo const &ArgInfo) {
800 return IsIndirect(ArgInfo.info);
801 });
802}
803
804static RValue emitLibraryCall(CodeGenFunction &CGF, const FunctionDecl *FD,
805 const CallExpr *E, llvm::Constant *calleeValue) {
806 CodeGenFunction::CGFPOptionsRAII FPOptsRAII(CGF, E);
807 CGCallee callee = CGCallee::forDirect(functionPtr: calleeValue, abstractInfo: GlobalDecl(FD));
808 llvm::CallBase *callOrInvoke = nullptr;
809 CGFunctionInfo const *FnInfo = nullptr;
810 RValue Call =
811 CGF.EmitCall(FnType: E->getCallee()->getType(), Callee: callee, E, ReturnValue: ReturnValueSlot(),
812 /*Chain=*/nullptr, CallOrInvoke: &callOrInvoke, ResolvedFnInfo: &FnInfo);
813
814 if (unsigned BuiltinID = FD->getBuiltinID()) {
815 // Check whether a FP math builtin function, such as BI__builtin_expf
816 ASTContext &Context = CGF.getContext();
817 bool ConstWithoutErrnoAndExceptions =
818 Context.BuiltinInfo.isConstWithoutErrnoAndExceptions(ID: BuiltinID);
819 // Restrict to target with errno, for example, MacOS doesn't set errno.
820 // TODO: Support builtin function with complex type returned, eg: cacosh
821 if (ConstWithoutErrnoAndExceptions && CGF.CGM.getLangOpts().MathErrno &&
822 !CGF.Builder.getIsFPConstrained() && Call.isScalar() &&
823 HasNoIndirectArgumentsOrResults(FnInfo: *FnInfo)) {
824 // Emit "int" TBAA metadata on FP math libcalls.
825 clang::QualType IntTy = Context.IntTy;
826 TBAAAccessInfo TBAAInfo = CGF.CGM.getTBAAAccessInfo(AccessType: IntTy);
827 CGF.CGM.DecorateInstructionWithTBAA(Inst: callOrInvoke, TBAAInfo);
828 }
829 }
830 return Call;
831}
832
833/// Emit a call to llvm.{sadd,uadd,ssub,usub,smul,umul}.with.overflow.*
834/// depending on IntrinsicID.
835///
836/// \arg CGF The current codegen function.
837/// \arg IntrinsicID The ID for the Intrinsic we wish to generate.
838/// \arg X The first argument to the llvm.*.with.overflow.*.
839/// \arg Y The second argument to the llvm.*.with.overflow.*.
840/// \arg Carry The carry returned by the llvm.*.with.overflow.*.
841/// \returns The result (i.e. sum/product) returned by the intrinsic.
842llvm::Value *EmitOverflowIntrinsic(CodeGenFunction &CGF,
843 const Intrinsic::ID IntrinsicID,
844 llvm::Value *X, llvm::Value *Y,
845 llvm::Value *&Carry) {
846 // Make sure we have integers of the same width.
847 assert(X->getType() == Y->getType() &&
848 "Arguments must be the same type. (Did you forget to make sure both "
849 "arguments have the same integer width?)");
850
851 Function *Callee = CGF.CGM.getIntrinsic(IID: IntrinsicID, Tys: X->getType());
852 llvm::Value *Tmp = CGF.Builder.CreateCall(Callee, Args: {X, Y});
853 Carry = CGF.Builder.CreateExtractValue(Agg: Tmp, Idxs: 1);
854 return CGF.Builder.CreateExtractValue(Agg: Tmp, Idxs: 0);
855}
856
857namespace {
858 struct WidthAndSignedness {
859 unsigned Width;
860 bool Signed;
861 };
862}
863
864static WidthAndSignedness
865getIntegerWidthAndSignedness(const clang::ASTContext &context,
866 const clang::QualType Type) {
867 assert(Type->isIntegerType() && "Given type is not an integer.");
868 unsigned Width = context.getIntWidth(T: Type);
869 bool Signed = Type->isSignedIntegerType();
870 return {.Width: Width, .Signed: Signed};
871}
872
873// Given one or more integer types, this function produces an integer type that
874// encompasses them: any value in one of the given types could be expressed in
875// the encompassing type.
876static struct WidthAndSignedness
877EncompassingIntegerType(ArrayRef<struct WidthAndSignedness> Types) {
878 assert(Types.size() > 0 && "Empty list of types.");
879
880 // If any of the given types is signed, we must return a signed type.
881 bool Signed = false;
882 for (const auto &Type : Types) {
883 Signed |= Type.Signed;
884 }
885
886 // The encompassing type must have a width greater than or equal to the width
887 // of the specified types. Additionally, if the encompassing type is signed,
888 // its width must be strictly greater than the width of any unsigned types
889 // given.
890 unsigned Width = 0;
891 for (const auto &Type : Types) {
892 unsigned MinWidth = Type.Width + (Signed && !Type.Signed);
893 if (Width < MinWidth) {
894 Width = MinWidth;
895 }
896 }
897
898 return {.Width: Width, .Signed: Signed};
899}
900
901Value *CodeGenFunction::EmitVAStartEnd(Value *ArgValue, bool IsStart) {
902 Intrinsic::ID inst = IsStart ? Intrinsic::vastart : Intrinsic::vaend;
903 return Builder.CreateCall(Callee: CGM.getIntrinsic(IID: inst, Tys: {ArgValue->getType()}),
904 Args: ArgValue);
905}
906
907/// Checks if using the result of __builtin_object_size(p, @p From) in place of
908/// __builtin_object_size(p, @p To) is correct
909static bool areBOSTypesCompatible(int From, int To) {
910 // Note: Our __builtin_object_size implementation currently treats Type=0 and
911 // Type=2 identically. Encoding this implementation detail here may make
912 // improving __builtin_object_size difficult in the future, so it's omitted.
913 return From == To || (From == 0 && To == 1) || (From == 3 && To == 2);
914}
915
916static llvm::Value *
917getDefaultBuiltinObjectSizeResult(unsigned Type, llvm::IntegerType *ResType) {
918 return ConstantInt::get(Ty: ResType, V: (Type & 2) ? 0 : -1, /*isSigned=*/IsSigned: true);
919}
920
921llvm::Value *
922CodeGenFunction::evaluateOrEmitBuiltinObjectSize(const Expr *E, unsigned Type,
923 llvm::IntegerType *ResType,
924 llvm::Value *EmittedE,
925 bool IsDynamic) {
926 uint64_t ObjectSize;
927 if (!E->tryEvaluateObjectSize(Result&: ObjectSize, Ctx&: getContext(), Type))
928 return emitBuiltinObjectSize(E, Type, ResType, EmittedE, IsDynamic);
929 return ConstantInt::get(Ty: ResType, V: ObjectSize, /*isSigned=*/IsSigned: true);
930}
931
932namespace {
933
934/// StructFieldAccess is a simple visitor class to grab the first MemberExpr
935/// from an Expr. It records any ArraySubscriptExpr we meet along the way.
936class StructFieldAccess
937 : public ConstStmtVisitor<StructFieldAccess, const Expr *> {
938 bool AddrOfSeen = false;
939
940public:
941 const Expr *ArrayIndex = nullptr;
942 QualType ArrayElementTy;
943
944 const Expr *VisitMemberExpr(const MemberExpr *E) {
945 if (AddrOfSeen && E->getType()->isArrayType())
946 // Avoid forms like '&ptr->array'.
947 return nullptr;
948 return E;
949 }
950
951 const Expr *VisitArraySubscriptExpr(const ArraySubscriptExpr *E) {
952 if (ArrayIndex)
953 // We don't support multiple subscripts.
954 return nullptr;
955
956 AddrOfSeen = false; // '&ptr->array[idx]' is okay.
957 ArrayIndex = E->getIdx();
958 ArrayElementTy = E->getBase()->getType();
959 return Visit(E->getBase());
960 }
961 const Expr *VisitCastExpr(const CastExpr *E) {
962 if (E->getCastKind() == CK_LValueToRValue)
963 return E;
964 return Visit(E->getSubExpr());
965 }
966 const Expr *VisitParenExpr(const ParenExpr *E) {
967 return Visit(E->getSubExpr());
968 }
969 const Expr *VisitUnaryAddrOf(const clang::UnaryOperator *E) {
970 AddrOfSeen = true;
971 return Visit(E->getSubExpr());
972 }
973 const Expr *VisitUnaryDeref(const clang::UnaryOperator *E) {
974 AddrOfSeen = false;
975 return Visit(E->getSubExpr());
976 }
977};
978
979} // end anonymous namespace
980
981/// Find a struct's flexible array member. It may be embedded inside multiple
982/// sub-structs, but must still be the last field.
983static const FieldDecl *FindFlexibleArrayMemberField(CodeGenFunction &CGF,
984 ASTContext &Ctx,
985 const RecordDecl *RD) {
986 const LangOptions::StrictFlexArraysLevelKind StrictFlexArraysLevel =
987 CGF.getLangOpts().getStrictFlexArraysLevel();
988
989 if (RD->isImplicit())
990 return nullptr;
991
992 for (const FieldDecl *FD : RD->fields()) {
993 if (Decl::isFlexibleArrayMemberLike(
994 Context: Ctx, D: FD, Ty: FD->getType(), StrictFlexArraysLevel,
995 /*IgnoreTemplateOrMacroSubstitution=*/true))
996 return FD;
997
998 if (auto RT = FD->getType()->getAs<RecordType>())
999 if (const FieldDecl *FD =
1000 FindFlexibleArrayMemberField(CGF, Ctx, RT->getAsRecordDecl()))
1001 return FD;
1002 }
1003
1004 return nullptr;
1005}
1006
1007/// Calculate the offset of a struct field. It may be embedded inside multiple
1008/// sub-structs.
1009static bool GetFieldOffset(ASTContext &Ctx, const RecordDecl *RD,
1010 const FieldDecl *FD, int64_t &Offset) {
1011 if (RD->isImplicit())
1012 return false;
1013
1014 // Keep track of the field number ourselves, because the other methods
1015 // (CGRecordLayout::getLLVMFieldNo) aren't always equivalent to how the AST
1016 // is laid out.
1017 uint32_t FieldNo = 0;
1018 const ASTRecordLayout &Layout = Ctx.getASTRecordLayout(D: RD);
1019
1020 for (const FieldDecl *Field : RD->fields()) {
1021 if (Field == FD) {
1022 Offset += Layout.getFieldOffset(FieldNo);
1023 return true;
1024 }
1025
1026 if (auto RT = Field->getType()->getAs<RecordType>()) {
1027 if (GetFieldOffset(Ctx, RT->getAsRecordDecl(), FD, Offset)) {
1028 Offset += Layout.getFieldOffset(FieldNo);
1029 return true;
1030 }
1031 }
1032
1033 if (!RD->isUnion())
1034 ++FieldNo;
1035 }
1036
1037 return false;
1038}
1039
1040static std::optional<int64_t>
1041GetFieldOffset(ASTContext &Ctx, const RecordDecl *RD, const FieldDecl *FD) {
1042 int64_t Offset = 0;
1043
1044 if (GetFieldOffset(Ctx, RD, FD, Offset))
1045 return std::optional<int64_t>(Offset);
1046
1047 return std::nullopt;
1048}
1049
1050llvm::Value *CodeGenFunction::emitCountedBySize(const Expr *E,
1051 llvm::Value *EmittedE,
1052 unsigned Type,
1053 llvm::IntegerType *ResType) {
1054 // Note: If the whole struct is specificed in the __bdos (i.e. Visitor
1055 // returns a DeclRefExpr). The calculation of the whole size of the structure
1056 // with a flexible array member can be done in two ways:
1057 //
1058 // 1) sizeof(struct S) + count * sizeof(typeof(fam))
1059 // 2) offsetof(struct S, fam) + count * sizeof(typeof(fam))
1060 //
1061 // The first will add additional padding after the end of the array
1062 // allocation while the second method is more precise, but not quite expected
1063 // from programmers. See
1064 // https://lore.kernel.org/lkml/ZvV6X5FPBBW7CO1f@archlinux/ for a discussion
1065 // of the topic.
1066 //
1067 // GCC isn't (currently) able to calculate __bdos on a pointer to the whole
1068 // structure. Therefore, because of the above issue, we choose to match what
1069 // GCC does for consistency's sake.
1070
1071 StructFieldAccess Visitor;
1072 E = Visitor.Visit(E);
1073 if (!E)
1074 return nullptr;
1075
1076 const Expr *Idx = Visitor.ArrayIndex;
1077 if (Idx) {
1078 if (Idx->HasSideEffects(Ctx: getContext()))
1079 // We can't have side-effects.
1080 return getDefaultBuiltinObjectSizeResult(Type, ResType);
1081
1082 if (const auto *IL = dyn_cast<IntegerLiteral>(Val: Idx)) {
1083 int64_t Val = IL->getValue().getSExtValue();
1084 if (Val < 0)
1085 return getDefaultBuiltinObjectSizeResult(Type, ResType);
1086
1087 // The index is 0, so we don't need to take it into account.
1088 if (Val == 0)
1089 Idx = nullptr;
1090 }
1091 }
1092
1093 // __counted_by on either a flexible array member or a pointer into a struct
1094 // with a flexible array member.
1095 if (const auto *ME = dyn_cast<MemberExpr>(Val: E))
1096 return emitCountedByMemberSize(E: ME, Idx, EmittedE, CastedArrayElementTy: Visitor.ArrayElementTy,
1097 Type, ResType);
1098
1099 // __counted_by on a pointer in a struct.
1100 if (const auto *ICE = dyn_cast<ImplicitCastExpr>(Val: E);
1101 ICE && ICE->getCastKind() == CK_LValueToRValue)
1102 return emitCountedByPointerSize(E: ICE, Idx, EmittedE, CastedArrayElementTy: Visitor.ArrayElementTy,
1103 Type, ResType);
1104
1105 return nullptr;
1106}
1107
1108static llvm::Value *EmitPositiveResultOrZero(CodeGenFunction &CGF,
1109 llvm::Value *Res,
1110 llvm::Value *Index,
1111 llvm::IntegerType *ResType,
1112 bool IsSigned) {
1113 // cmp = (array_size >= 0)
1114 Value *Cmp = CGF.Builder.CreateIsNotNeg(Arg: Res);
1115 if (Index)
1116 // cmp = (cmp && index >= 0)
1117 Cmp = CGF.Builder.CreateAnd(LHS: CGF.Builder.CreateIsNotNeg(Arg: Index), RHS: Cmp);
1118
1119 // return cmp ? result : 0
1120 return CGF.Builder.CreateSelect(C: Cmp, True: Res,
1121 False: ConstantInt::get(Ty: ResType, V: 0, IsSigned));
1122}
1123
1124static std::pair<llvm::Value *, llvm::Value *>
1125GetCountFieldAndIndex(CodeGenFunction &CGF, const MemberExpr *ME,
1126 const FieldDecl *ArrayFD, const FieldDecl *CountFD,
1127 const Expr *Idx, llvm::IntegerType *ResType,
1128 bool IsSigned) {
1129 // count = ptr->count;
1130 Value *Count = CGF.EmitLoadOfCountedByField(ME, ArrayFD, CountFD);
1131 if (!Count)
1132 return std::make_pair<Value *>(x: nullptr, y: nullptr);
1133 Count = CGF.Builder.CreateIntCast(V: Count, DestTy: ResType, isSigned: IsSigned, Name: "count");
1134
1135 // index = ptr->index;
1136 Value *Index = nullptr;
1137 if (Idx) {
1138 bool IdxSigned = Idx->getType()->isSignedIntegerType();
1139 Index = CGF.EmitScalarExpr(E: Idx);
1140 Index = CGF.Builder.CreateIntCast(V: Index, DestTy: ResType, isSigned: IdxSigned, Name: "index");
1141 }
1142
1143 return std::make_pair(x&: Count, y&: Index);
1144}
1145
1146llvm::Value *CodeGenFunction::emitCountedByPointerSize(
1147 const ImplicitCastExpr *E, const Expr *Idx, llvm::Value *EmittedE,
1148 QualType CastedArrayElementTy, unsigned Type, llvm::IntegerType *ResType) {
1149 assert(E->getCastKind() == CK_LValueToRValue &&
1150 "must be an LValue to RValue cast");
1151
1152 const MemberExpr *ME = dyn_cast<MemberExpr>(E->getSubExpr());
1153 if (!ME)
1154 return nullptr;
1155
1156 const auto *ArrayBaseFD = dyn_cast<FieldDecl>(Val: ME->getMemberDecl());
1157 if (!ArrayBaseFD || !ArrayBaseFD->getType()->isPointerType() ||
1158 !ArrayBaseFD->getType()->isCountAttributedType())
1159 return nullptr;
1160
1161 // Get the 'count' FieldDecl.
1162 const FieldDecl *CountFD = ArrayBaseFD->findCountedByField();
1163 if (!CountFD)
1164 // Can't find the field referenced by the "counted_by" attribute.
1165 return nullptr;
1166
1167 // Calculate the array's object size using these formulae. (Note: if the
1168 // calculation is negative, we return 0.):
1169 //
1170 // struct p;
1171 // struct s {
1172 // /* ... */
1173 // struct p **array __attribute__((counted_by(count)));
1174 // int count;
1175 // };
1176 //
1177 // 1) 'ptr->array':
1178 //
1179 // count = ptr->count;
1180 //
1181 // array_element_size = sizeof (*ptr->array);
1182 // array_size = count * array_element_size;
1183 //
1184 // result = array_size;
1185 //
1186 // cmp = (result >= 0)
1187 // return cmp ? result : 0;
1188 //
1189 // 2) '&((cast) ptr->array)[idx]':
1190 //
1191 // count = ptr->count;
1192 // index = idx;
1193 //
1194 // array_element_size = sizeof (*ptr->array);
1195 // array_size = count * array_element_size;
1196 //
1197 // casted_array_element_size = sizeof (*((cast) ptr->array));
1198 //
1199 // index_size = index * casted_array_element_size;
1200 // result = array_size - index_size;
1201 //
1202 // cmp = (result >= 0)
1203 // if (index)
1204 // cmp = (cmp && index > 0)
1205 // return cmp ? result : 0;
1206
1207 auto GetElementBaseSize = [&](QualType ElementTy) {
1208 CharUnits ElementSize =
1209 getContext().getTypeSizeInChars(T: ElementTy->getPointeeType());
1210
1211 if (ElementSize.isZero()) {
1212 // This might be a __sized_by on a 'void *', which counts bytes, not
1213 // elements.
1214 auto *CAT = ElementTy->getAs<CountAttributedType>();
1215 if (!CAT || (CAT->getKind() != CountAttributedType::SizedBy &&
1216 CAT->getKind() != CountAttributedType::SizedByOrNull))
1217 // Okay, not sure what it is now.
1218 // FIXME: Should this be an assert?
1219 return std::optional<CharUnits>();
1220
1221 ElementSize = CharUnits::One();
1222 }
1223
1224 return std::optional<CharUnits>(ElementSize);
1225 };
1226
1227 // Get the sizes of the original array element and the casted array element,
1228 // if different.
1229 std::optional<CharUnits> ArrayElementBaseSize =
1230 GetElementBaseSize(ArrayBaseFD->getType());
1231 if (!ArrayElementBaseSize)
1232 return nullptr;
1233
1234 std::optional<CharUnits> CastedArrayElementBaseSize = ArrayElementBaseSize;
1235 if (!CastedArrayElementTy.isNull() && CastedArrayElementTy->isPointerType()) {
1236 CastedArrayElementBaseSize = GetElementBaseSize(CastedArrayElementTy);
1237 if (!CastedArrayElementBaseSize)
1238 return nullptr;
1239 }
1240
1241 bool IsSigned = CountFD->getType()->isSignedIntegerType();
1242
1243 // count = ptr->count;
1244 // index = ptr->index;
1245 Value *Count, *Index;
1246 std::tie(args&: Count, args&: Index) = GetCountFieldAndIndex(
1247 *this, ME, ArrayBaseFD, CountFD, Idx, ResType, IsSigned);
1248 if (!Count)
1249 return nullptr;
1250
1251 // array_element_size = sizeof (*ptr->array)
1252 auto *ArrayElementSize = llvm::ConstantInt::get(
1253 Ty: ResType, V: ArrayElementBaseSize->getQuantity(), IsSigned);
1254
1255 // casted_array_element_size = sizeof (*((cast) ptr->array));
1256 auto *CastedArrayElementSize = llvm::ConstantInt::get(
1257 Ty: ResType, V: CastedArrayElementBaseSize->getQuantity(), IsSigned);
1258
1259 // array_size = count * array_element_size;
1260 Value *ArraySize = Builder.CreateMul(LHS: Count, RHS: ArrayElementSize, Name: "array_size",
1261 HasNUW: !IsSigned, HasNSW: IsSigned);
1262
1263 // Option (1) 'ptr->array'
1264 // result = array_size
1265 Value *Result = ArraySize;
1266
1267 if (Idx) { // Option (2) '&((cast) ptr->array)[idx]'
1268 // index_size = index * casted_array_element_size;
1269 Value *IndexSize = Builder.CreateMul(LHS: Index, RHS: CastedArrayElementSize,
1270 Name: "index_size", HasNUW: !IsSigned, HasNSW: IsSigned);
1271
1272 // result = result - index_size;
1273 Result =
1274 Builder.CreateSub(LHS: Result, RHS: IndexSize, Name: "result", HasNUW: !IsSigned, HasNSW: IsSigned);
1275 }
1276
1277 return EmitPositiveResultOrZero(CGF&: *this, Res: Result, Index, ResType, IsSigned);
1278}
1279
1280llvm::Value *CodeGenFunction::emitCountedByMemberSize(
1281 const MemberExpr *ME, const Expr *Idx, llvm::Value *EmittedE,
1282 QualType CastedArrayElementTy, unsigned Type, llvm::IntegerType *ResType) {
1283 const auto *FD = dyn_cast<FieldDecl>(Val: ME->getMemberDecl());
1284 if (!FD)
1285 return nullptr;
1286
1287 // Find the flexible array member and check that it has the __counted_by
1288 // attribute.
1289 ASTContext &Ctx = getContext();
1290 const RecordDecl *RD = FD->getDeclContext()->getOuterLexicalRecordContext();
1291 const FieldDecl *FlexibleArrayMemberFD = nullptr;
1292
1293 if (Decl::isFlexibleArrayMemberLike(
1294 Context: Ctx, D: FD, Ty: FD->getType(), StrictFlexArraysLevel: getLangOpts().getStrictFlexArraysLevel(),
1295 /*IgnoreTemplateOrMacroSubstitution=*/true))
1296 FlexibleArrayMemberFD = FD;
1297 else
1298 FlexibleArrayMemberFD = FindFlexibleArrayMemberField(CGF&: *this, Ctx, RD);
1299
1300 if (!FlexibleArrayMemberFD ||
1301 !FlexibleArrayMemberFD->getType()->isCountAttributedType())
1302 return nullptr;
1303
1304 // Get the 'count' FieldDecl.
1305 const FieldDecl *CountFD = FlexibleArrayMemberFD->findCountedByField();
1306 if (!CountFD)
1307 // Can't find the field referenced by the "counted_by" attribute.
1308 return nullptr;
1309
1310 // Calculate the flexible array member's object size using these formulae.
1311 // (Note: if the calculation is negative, we return 0.):
1312 //
1313 // struct p;
1314 // struct s {
1315 // /* ... */
1316 // int count;
1317 // struct p *array[] __attribute__((counted_by(count)));
1318 // };
1319 //
1320 // 1) 'ptr->array':
1321 //
1322 // count = ptr->count;
1323 //
1324 // flexible_array_member_element_size = sizeof (*ptr->array);
1325 // flexible_array_member_size =
1326 // count * flexible_array_member_element_size;
1327 //
1328 // result = flexible_array_member_size;
1329 //
1330 // cmp = (result >= 0)
1331 // return cmp ? result : 0;
1332 //
1333 // 2) '&((cast) ptr->array)[idx]':
1334 //
1335 // count = ptr->count;
1336 // index = idx;
1337 //
1338 // flexible_array_member_element_size = sizeof (*ptr->array);
1339 // flexible_array_member_size =
1340 // count * flexible_array_member_element_size;
1341 //
1342 // casted_flexible_array_member_element_size =
1343 // sizeof (*((cast) ptr->array));
1344 // index_size = index * casted_flexible_array_member_element_size;
1345 //
1346 // result = flexible_array_member_size - index_size;
1347 //
1348 // cmp = (result >= 0)
1349 // if (index != 0)
1350 // cmp = (cmp && index >= 0)
1351 // return cmp ? result : 0;
1352 //
1353 // 3) '&ptr->field':
1354 //
1355 // count = ptr->count;
1356 // sizeof_struct = sizeof (struct s);
1357 //
1358 // flexible_array_member_element_size = sizeof (*ptr->array);
1359 // flexible_array_member_size =
1360 // count * flexible_array_member_element_size;
1361 //
1362 // field_offset = offsetof (struct s, field);
1363 // offset_diff = sizeof_struct - field_offset;
1364 //
1365 // result = offset_diff + flexible_array_member_size;
1366 //
1367 // cmp = (result >= 0)
1368 // return cmp ? result : 0;
1369 //
1370 // 4) '&((cast) ptr->field_array)[idx]':
1371 //
1372 // count = ptr->count;
1373 // index = idx;
1374 // sizeof_struct = sizeof (struct s);
1375 //
1376 // flexible_array_member_element_size = sizeof (*ptr->array);
1377 // flexible_array_member_size =
1378 // count * flexible_array_member_element_size;
1379 //
1380 // casted_field_element_size = sizeof (*((cast) ptr->field_array));
1381 // field_offset = offsetof (struct s, field)
1382 // field_offset += index * casted_field_element_size;
1383 //
1384 // offset_diff = sizeof_struct - field_offset;
1385 //
1386 // result = offset_diff + flexible_array_member_size;
1387 //
1388 // cmp = (result >= 0)
1389 // if (index != 0)
1390 // cmp = (cmp && index >= 0)
1391 // return cmp ? result : 0;
1392
1393 bool IsSigned = CountFD->getType()->isSignedIntegerType();
1394
1395 QualType FlexibleArrayMemberTy = FlexibleArrayMemberFD->getType();
1396
1397 // Explicit cast because otherwise the CharWidth will promote an i32's into
1398 // u64's leading to overflows.
1399 int64_t CharWidth = static_cast<int64_t>(CGM.getContext().getCharWidth());
1400
1401 // field_offset = offsetof (struct s, field);
1402 Value *FieldOffset = nullptr;
1403 if (FlexibleArrayMemberFD != FD) {
1404 std::optional<int64_t> Offset = GetFieldOffset(Ctx, RD, FD);
1405 if (!Offset)
1406 return nullptr;
1407 FieldOffset =
1408 llvm::ConstantInt::get(Ty: ResType, V: *Offset / CharWidth, IsSigned);
1409 }
1410
1411 // count = ptr->count;
1412 // index = ptr->index;
1413 Value *Count, *Index;
1414 std::tie(args&: Count, args&: Index) = GetCountFieldAndIndex(
1415 CGF&: *this, ME, ArrayFD: FlexibleArrayMemberFD, CountFD, Idx, ResType, IsSigned);
1416 if (!Count)
1417 return nullptr;
1418
1419 // flexible_array_member_element_size = sizeof (*ptr->array);
1420 const ArrayType *ArrayTy = Ctx.getAsArrayType(T: FlexibleArrayMemberTy);
1421 CharUnits BaseSize = Ctx.getTypeSizeInChars(T: ArrayTy->getElementType());
1422 auto *FlexibleArrayMemberElementSize =
1423 llvm::ConstantInt::get(Ty: ResType, V: BaseSize.getQuantity(), IsSigned);
1424
1425 // flexible_array_member_size = count * flexible_array_member_element_size;
1426 Value *FlexibleArrayMemberSize =
1427 Builder.CreateMul(LHS: Count, RHS: FlexibleArrayMemberElementSize,
1428 Name: "flexible_array_member_size", HasNUW: !IsSigned, HasNSW: IsSigned);
1429
1430 Value *Result = nullptr;
1431 if (FlexibleArrayMemberFD == FD) {
1432 if (Idx) { // Option (2) '&((cast) ptr->array)[idx]'
1433 // casted_flexible_array_member_element_size =
1434 // sizeof (*((cast) ptr->array));
1435 llvm::ConstantInt *CastedFlexibleArrayMemberElementSize =
1436 FlexibleArrayMemberElementSize;
1437 if (!CastedArrayElementTy.isNull() &&
1438 CastedArrayElementTy->isPointerType()) {
1439 CharUnits BaseSize =
1440 Ctx.getTypeSizeInChars(T: CastedArrayElementTy->getPointeeType());
1441 CastedFlexibleArrayMemberElementSize =
1442 llvm::ConstantInt::get(Ty: ResType, V: BaseSize.getQuantity(), IsSigned);
1443 }
1444
1445 // index_size = index * casted_flexible_array_member_element_size;
1446 Value *IndexSize =
1447 Builder.CreateMul(LHS: Index, RHS: CastedFlexibleArrayMemberElementSize,
1448 Name: "index_size", HasNUW: !IsSigned, HasNSW: IsSigned);
1449
1450 // result = flexible_array_member_size - index_size;
1451 Result = Builder.CreateSub(LHS: FlexibleArrayMemberSize, RHS: IndexSize, Name: "result",
1452 HasNUW: !IsSigned, HasNSW: IsSigned);
1453 } else { // Option (1) 'ptr->array'
1454 // result = flexible_array_member_size;
1455 Result = FlexibleArrayMemberSize;
1456 }
1457 } else {
1458 // sizeof_struct = sizeof (struct s);
1459 llvm::StructType *StructTy = getTypes().getCGRecordLayout(RD).getLLVMType();
1460 const llvm::DataLayout &Layout = CGM.getDataLayout();
1461 TypeSize Size = Layout.getTypeSizeInBits(Ty: StructTy);
1462 Value *SizeofStruct =
1463 llvm::ConstantInt::get(Ty: ResType, V: Size.getKnownMinValue() / CharWidth);
1464
1465 if (Idx) { // Option (4) '&((cast) ptr->field_array)[idx]'
1466 // casted_field_element_size = sizeof (*((cast) ptr->field_array));
1467 CharUnits BaseSize;
1468 if (!CastedArrayElementTy.isNull() &&
1469 CastedArrayElementTy->isPointerType()) {
1470 BaseSize =
1471 Ctx.getTypeSizeInChars(T: CastedArrayElementTy->getPointeeType());
1472 } else {
1473 const ArrayType *ArrayTy = Ctx.getAsArrayType(T: FD->getType());
1474 BaseSize = Ctx.getTypeSizeInChars(T: ArrayTy->getElementType());
1475 }
1476
1477 llvm::ConstantInt *CastedFieldElementSize =
1478 llvm::ConstantInt::get(Ty: ResType, V: BaseSize.getQuantity(), IsSigned);
1479
1480 // field_offset += index * casted_field_element_size;
1481 Value *Mul = Builder.CreateMul(LHS: Index, RHS: CastedFieldElementSize,
1482 Name: "field_offset", HasNUW: !IsSigned, HasNSW: IsSigned);
1483 FieldOffset = Builder.CreateAdd(LHS: FieldOffset, RHS: Mul);
1484 }
1485 // Option (3) '&ptr->field', and Option (4) continuation.
1486 // offset_diff = flexible_array_member_offset - field_offset;
1487 Value *OffsetDiff = Builder.CreateSub(LHS: SizeofStruct, RHS: FieldOffset,
1488 Name: "offset_diff", HasNUW: !IsSigned, HasNSW: IsSigned);
1489
1490 // result = offset_diff + flexible_array_member_size;
1491 Result = Builder.CreateAdd(LHS: FlexibleArrayMemberSize, RHS: OffsetDiff, Name: "result");
1492 }
1493
1494 return EmitPositiveResultOrZero(CGF&: *this, Res: Result, Index, ResType, IsSigned);
1495}
1496
1497/// Returns a Value corresponding to the size of the given expression.
1498/// This Value may be either of the following:
1499/// - A llvm::Argument (if E is a param with the pass_object_size attribute on
1500/// it)
1501/// - A call to the @llvm.objectsize intrinsic
1502///
1503/// EmittedE is the result of emitting `E` as a scalar expr. If it's non-null
1504/// and we wouldn't otherwise try to reference a pass_object_size parameter,
1505/// we'll call @llvm.objectsize on EmittedE, rather than emitting E.
1506llvm::Value *
1507CodeGenFunction::emitBuiltinObjectSize(const Expr *E, unsigned Type,
1508 llvm::IntegerType *ResType,
1509 llvm::Value *EmittedE, bool IsDynamic) {
1510 // We need to reference an argument if the pointer is a parameter with the
1511 // pass_object_size attribute.
1512 if (auto *D = dyn_cast<DeclRefExpr>(Val: E->IgnoreParenImpCasts())) {
1513 auto *Param = dyn_cast<ParmVarDecl>(Val: D->getDecl());
1514 auto *PS = D->getDecl()->getAttr<PassObjectSizeAttr>();
1515 if (Param != nullptr && PS != nullptr &&
1516 areBOSTypesCompatible(PS->getType(), Type)) {
1517 auto Iter = SizeArguments.find(Val: Param);
1518 assert(Iter != SizeArguments.end());
1519
1520 const ImplicitParamDecl *D = Iter->second;
1521 auto DIter = LocalDeclMap.find(D);
1522 assert(DIter != LocalDeclMap.end());
1523
1524 return EmitLoadOfScalar(DIter->second, /*Volatile=*/false,
1525 getContext().getSizeType(), E->getBeginLoc());
1526 }
1527 }
1528
1529 // LLVM can't handle Type=3 appropriately, and __builtin_object_size shouldn't
1530 // evaluate E for side-effects. In either case, we shouldn't lower to
1531 // @llvm.objectsize.
1532 if (Type == 3 || (!EmittedE && E->HasSideEffects(Ctx: getContext())))
1533 return getDefaultBuiltinObjectSizeResult(Type, ResType);
1534
1535 Value *Ptr = EmittedE ? EmittedE : EmitScalarExpr(E);
1536 assert(Ptr->getType()->isPointerTy() &&
1537 "Non-pointer passed to __builtin_object_size?");
1538
1539 if (IsDynamic)
1540 // Emit special code for a flexible array member with the "counted_by"
1541 // attribute.
1542 if (Value *V = emitCountedBySize(E, EmittedE: Ptr, Type, ResType))
1543 return V;
1544
1545 Function *F =
1546 CGM.getIntrinsic(Intrinsic::objectsize, {ResType, Ptr->getType()});
1547
1548 // LLVM only supports 0 and 2, make sure that we pass along that as a boolean.
1549 Value *Min = Builder.getInt1(V: (Type & 2) != 0);
1550 // For GCC compatibility, __builtin_object_size treat NULL as unknown size.
1551 Value *NullIsUnknown = Builder.getTrue();
1552 Value *Dynamic = Builder.getInt1(V: IsDynamic);
1553 return Builder.CreateCall(Callee: F, Args: {Ptr, Min, NullIsUnknown, Dynamic});
1554}
1555
1556namespace {
1557/// A struct to generically describe a bit test intrinsic.
1558struct BitTest {
1559 enum ActionKind : uint8_t { TestOnly, Complement, Reset, Set };
1560 enum InterlockingKind : uint8_t {
1561 Unlocked,
1562 Sequential,
1563 Acquire,
1564 Release,
1565 NoFence
1566 };
1567
1568 ActionKind Action;
1569 InterlockingKind Interlocking;
1570 bool Is64Bit;
1571
1572 static BitTest decodeBitTestBuiltin(unsigned BuiltinID);
1573};
1574
1575} // namespace
1576
1577BitTest BitTest::decodeBitTestBuiltin(unsigned BuiltinID) {
1578 switch (BuiltinID) {
1579 // Main portable variants.
1580 case Builtin::BI_bittest:
1581 return {.Action: TestOnly, .Interlocking: Unlocked, .Is64Bit: false};
1582 case Builtin::BI_bittestandcomplement:
1583 return {.Action: Complement, .Interlocking: Unlocked, .Is64Bit: false};
1584 case Builtin::BI_bittestandreset:
1585 return {.Action: Reset, .Interlocking: Unlocked, .Is64Bit: false};
1586 case Builtin::BI_bittestandset:
1587 return {.Action: Set, .Interlocking: Unlocked, .Is64Bit: false};
1588 case Builtin::BI_interlockedbittestandreset:
1589 return {.Action: Reset, .Interlocking: Sequential, .Is64Bit: false};
1590 case Builtin::BI_interlockedbittestandset:
1591 return {.Action: Set, .Interlocking: Sequential, .Is64Bit: false};
1592
1593 // X86-specific 64-bit variants.
1594 case Builtin::BI_bittest64:
1595 return {.Action: TestOnly, .Interlocking: Unlocked, .Is64Bit: true};
1596 case Builtin::BI_bittestandcomplement64:
1597 return {.Action: Complement, .Interlocking: Unlocked, .Is64Bit: true};
1598 case Builtin::BI_bittestandreset64:
1599 return {.Action: Reset, .Interlocking: Unlocked, .Is64Bit: true};
1600 case Builtin::BI_bittestandset64:
1601 return {.Action: Set, .Interlocking: Unlocked, .Is64Bit: true};
1602 case Builtin::BI_interlockedbittestandreset64:
1603 return {.Action: Reset, .Interlocking: Sequential, .Is64Bit: true};
1604 case Builtin::BI_interlockedbittestandset64:
1605 return {.Action: Set, .Interlocking: Sequential, .Is64Bit: true};
1606
1607 // ARM/AArch64-specific ordering variants.
1608 case Builtin::BI_interlockedbittestandset_acq:
1609 return {.Action: Set, .Interlocking: Acquire, .Is64Bit: false};
1610 case Builtin::BI_interlockedbittestandset_rel:
1611 return {.Action: Set, .Interlocking: Release, .Is64Bit: false};
1612 case Builtin::BI_interlockedbittestandset_nf:
1613 return {.Action: Set, .Interlocking: NoFence, .Is64Bit: false};
1614 case Builtin::BI_interlockedbittestandreset_acq:
1615 return {.Action: Reset, .Interlocking: Acquire, .Is64Bit: false};
1616 case Builtin::BI_interlockedbittestandreset_rel:
1617 return {.Action: Reset, .Interlocking: Release, .Is64Bit: false};
1618 case Builtin::BI_interlockedbittestandreset_nf:
1619 return {.Action: Reset, .Interlocking: NoFence, .Is64Bit: false};
1620 }
1621 llvm_unreachable("expected only bittest intrinsics");
1622}
1623
1624static char bitActionToX86BTCode(BitTest::ActionKind A) {
1625 switch (A) {
1626 case BitTest::TestOnly: return '\0';
1627 case BitTest::Complement: return 'c';
1628 case BitTest::Reset: return 'r';
1629 case BitTest::Set: return 's';
1630 }
1631 llvm_unreachable("invalid action");
1632}
1633
1634static llvm::Value *EmitX86BitTestIntrinsic(CodeGenFunction &CGF,
1635 BitTest BT,
1636 const CallExpr *E, Value *BitBase,
1637 Value *BitPos) {
1638 char Action = bitActionToX86BTCode(A: BT.Action);
1639 char SizeSuffix = BT.Is64Bit ? 'q' : 'l';
1640
1641 // Build the assembly.
1642 SmallString<64> Asm;
1643 raw_svector_ostream AsmOS(Asm);
1644 if (BT.Interlocking != BitTest::Unlocked)
1645 AsmOS << "lock ";
1646 AsmOS << "bt";
1647 if (Action)
1648 AsmOS << Action;
1649 AsmOS << SizeSuffix << " $2, ($1)";
1650
1651 // Build the constraints. FIXME: We should support immediates when possible.
1652 std::string Constraints = "={@ccc},r,r,~{cc},~{memory}";
1653 std::string_view MachineClobbers = CGF.getTarget().getClobbers();
1654 if (!MachineClobbers.empty()) {
1655 Constraints += ',';
1656 Constraints += MachineClobbers;
1657 }
1658 llvm::IntegerType *IntType = llvm::IntegerType::get(
1659 C&: CGF.getLLVMContext(),
1660 NumBits: CGF.getContext().getTypeSize(T: E->getArg(Arg: 1)->getType()));
1661 llvm::FunctionType *FTy =
1662 llvm::FunctionType::get(Result: CGF.Int8Ty, Params: {CGF.UnqualPtrTy, IntType}, isVarArg: false);
1663
1664 llvm::InlineAsm *IA =
1665 llvm::InlineAsm::get(Ty: FTy, AsmString: Asm, Constraints, /*hasSideEffects=*/true);
1666 return CGF.Builder.CreateCall(Callee: IA, Args: {BitBase, BitPos});
1667}
1668
1669static llvm::AtomicOrdering
1670getBitTestAtomicOrdering(BitTest::InterlockingKind I) {
1671 switch (I) {
1672 case BitTest::Unlocked: return llvm::AtomicOrdering::NotAtomic;
1673 case BitTest::Sequential: return llvm::AtomicOrdering::SequentiallyConsistent;
1674 case BitTest::Acquire: return llvm::AtomicOrdering::Acquire;
1675 case BitTest::Release: return llvm::AtomicOrdering::Release;
1676 case BitTest::NoFence: return llvm::AtomicOrdering::Monotonic;
1677 }
1678 llvm_unreachable("invalid interlocking");
1679}
1680
1681/// Emit a _bittest* intrinsic. These intrinsics take a pointer to an array of
1682/// bits and a bit position and read and optionally modify the bit at that
1683/// position. The position index can be arbitrarily large, i.e. it can be larger
1684/// than 31 or 63, so we need an indexed load in the general case.
1685static llvm::Value *EmitBitTestIntrinsic(CodeGenFunction &CGF,
1686 unsigned BuiltinID,
1687 const CallExpr *E) {
1688 Value *BitBase = CGF.EmitScalarExpr(E: E->getArg(Arg: 0));
1689 Value *BitPos = CGF.EmitScalarExpr(E: E->getArg(Arg: 1));
1690
1691 BitTest BT = BitTest::decodeBitTestBuiltin(BuiltinID);
1692
1693 // X86 has special BT, BTC, BTR, and BTS instructions that handle the array
1694 // indexing operation internally. Use them if possible.
1695 if (CGF.getTarget().getTriple().isX86())
1696 return EmitX86BitTestIntrinsic(CGF, BT, E, BitBase, BitPos);
1697
1698 // Otherwise, use generic code to load one byte and test the bit. Use all but
1699 // the bottom three bits as the array index, and the bottom three bits to form
1700 // a mask.
1701 // Bit = BitBaseI8[BitPos >> 3] & (1 << (BitPos & 0x7)) != 0;
1702 Value *ByteIndex = CGF.Builder.CreateAShr(
1703 LHS: BitPos, RHS: llvm::ConstantInt::get(Ty: BitPos->getType(), V: 3), Name: "bittest.byteidx");
1704 Address ByteAddr(CGF.Builder.CreateInBoundsGEP(Ty: CGF.Int8Ty, Ptr: BitBase, IdxList: ByteIndex,
1705 Name: "bittest.byteaddr"),
1706 CGF.Int8Ty, CharUnits::One());
1707 Value *PosLow =
1708 CGF.Builder.CreateAnd(LHS: CGF.Builder.CreateTrunc(V: BitPos, DestTy: CGF.Int8Ty),
1709 RHS: llvm::ConstantInt::get(Ty: CGF.Int8Ty, V: 0x7));
1710
1711 // The updating instructions will need a mask.
1712 Value *Mask = nullptr;
1713 if (BT.Action != BitTest::TestOnly) {
1714 Mask = CGF.Builder.CreateShl(LHS: llvm::ConstantInt::get(Ty: CGF.Int8Ty, V: 1), RHS: PosLow,
1715 Name: "bittest.mask");
1716 }
1717
1718 // Check the action and ordering of the interlocked intrinsics.
1719 llvm::AtomicOrdering Ordering = getBitTestAtomicOrdering(I: BT.Interlocking);
1720
1721 Value *OldByte = nullptr;
1722 if (Ordering != llvm::AtomicOrdering::NotAtomic) {
1723 // Emit a combined atomicrmw load/store operation for the interlocked
1724 // intrinsics.
1725 llvm::AtomicRMWInst::BinOp RMWOp = llvm::AtomicRMWInst::Or;
1726 if (BT.Action == BitTest::Reset) {
1727 Mask = CGF.Builder.CreateNot(V: Mask);
1728 RMWOp = llvm::AtomicRMWInst::And;
1729 }
1730 OldByte = CGF.Builder.CreateAtomicRMW(Op: RMWOp, Addr: ByteAddr, Val: Mask, Ordering);
1731 } else {
1732 // Emit a plain load for the non-interlocked intrinsics.
1733 OldByte = CGF.Builder.CreateLoad(Addr: ByteAddr, Name: "bittest.byte");
1734 Value *NewByte = nullptr;
1735 switch (BT.Action) {
1736 case BitTest::TestOnly:
1737 // Don't store anything.
1738 break;
1739 case BitTest::Complement:
1740 NewByte = CGF.Builder.CreateXor(LHS: OldByte, RHS: Mask);
1741 break;
1742 case BitTest::Reset:
1743 NewByte = CGF.Builder.CreateAnd(LHS: OldByte, RHS: CGF.Builder.CreateNot(V: Mask));
1744 break;
1745 case BitTest::Set:
1746 NewByte = CGF.Builder.CreateOr(LHS: OldByte, RHS: Mask);
1747 break;
1748 }
1749 if (NewByte)
1750 CGF.Builder.CreateStore(Val: NewByte, Addr: ByteAddr);
1751 }
1752
1753 // However we loaded the old byte, either by plain load or atomicrmw, shift
1754 // the bit into the low position and mask it to 0 or 1.
1755 Value *ShiftedByte = CGF.Builder.CreateLShr(LHS: OldByte, RHS: PosLow, Name: "bittest.shr");
1756 return CGF.Builder.CreateAnd(
1757 LHS: ShiftedByte, RHS: llvm::ConstantInt::get(Ty: CGF.Int8Ty, V: 1), Name: "bittest.res");
1758}
1759
1760namespace {
1761enum class MSVCSetJmpKind {
1762 _setjmpex,
1763 _setjmp3,
1764 _setjmp
1765};
1766}
1767
1768/// MSVC handles setjmp a bit differently on different platforms. On every
1769/// architecture except 32-bit x86, the frame address is passed. On x86, extra
1770/// parameters can be passed as variadic arguments, but we always pass none.
1771static RValue EmitMSVCRTSetJmp(CodeGenFunction &CGF, MSVCSetJmpKind SJKind,
1772 const CallExpr *E) {
1773 llvm::Value *Arg1 = nullptr;
1774 llvm::Type *Arg1Ty = nullptr;
1775 StringRef Name;
1776 bool IsVarArg = false;
1777 if (SJKind == MSVCSetJmpKind::_setjmp3) {
1778 Name = "_setjmp3";
1779 Arg1Ty = CGF.Int32Ty;
1780 Arg1 = llvm::ConstantInt::get(Ty: CGF.IntTy, V: 0);
1781 IsVarArg = true;
1782 } else {
1783 Name = SJKind == MSVCSetJmpKind::_setjmp ? "_setjmp" : "_setjmpex";
1784 Arg1Ty = CGF.Int8PtrTy;
1785 if (CGF.getTarget().getTriple().getArch() == llvm::Triple::aarch64) {
1786 Arg1 = CGF.Builder.CreateCall(
1787 CGF.CGM.getIntrinsic(Intrinsic::sponentry, CGF.AllocaInt8PtrTy));
1788 } else
1789 Arg1 = CGF.Builder.CreateCall(
1790 CGF.CGM.getIntrinsic(Intrinsic::frameaddress, CGF.AllocaInt8PtrTy),
1791 llvm::ConstantInt::get(CGF.Int32Ty, 0));
1792 }
1793
1794 // Mark the call site and declaration with ReturnsTwice.
1795 llvm::Type *ArgTypes[2] = {CGF.Int8PtrTy, Arg1Ty};
1796 llvm::AttributeList ReturnsTwiceAttr = llvm::AttributeList::get(
1797 CGF.getLLVMContext(), llvm::AttributeList::FunctionIndex,
1798 llvm::Attribute::ReturnsTwice);
1799 llvm::FunctionCallee SetJmpFn = CGF.CGM.CreateRuntimeFunction(
1800 Ty: llvm::FunctionType::get(Result: CGF.IntTy, Params: ArgTypes, isVarArg: IsVarArg), Name,
1801 ExtraAttrs: ReturnsTwiceAttr, /*Local=*/true);
1802
1803 llvm::Value *Buf = CGF.Builder.CreateBitOrPointerCast(
1804 V: CGF.EmitScalarExpr(E: E->getArg(Arg: 0)), DestTy: CGF.Int8PtrTy);
1805 llvm::Value *Args[] = {Buf, Arg1};
1806 llvm::CallBase *CB = CGF.EmitRuntimeCallOrInvoke(callee: SetJmpFn, args: Args);
1807 CB->setAttributes(ReturnsTwiceAttr);
1808 return RValue::get(V: CB);
1809}
1810
1811// Emit an MSVC intrinsic. Assumes that arguments have *not* been evaluated.
1812Value *CodeGenFunction::EmitMSVCBuiltinExpr(MSVCIntrin BuiltinID,
1813 const CallExpr *E) {
1814 switch (BuiltinID) {
1815 case MSVCIntrin::_BitScanForward:
1816 case MSVCIntrin::_BitScanReverse: {
1817 Address IndexAddress(EmitPointerWithAlignment(Addr: E->getArg(Arg: 0)));
1818 Value *ArgValue = EmitScalarExpr(E: E->getArg(Arg: 1));
1819
1820 llvm::Type *ArgType = ArgValue->getType();
1821 llvm::Type *IndexType = IndexAddress.getElementType();
1822 llvm::Type *ResultType = ConvertType(E->getType());
1823
1824 Value *ArgZero = llvm::Constant::getNullValue(Ty: ArgType);
1825 Value *ResZero = llvm::Constant::getNullValue(Ty: ResultType);
1826 Value *ResOne = llvm::ConstantInt::get(Ty: ResultType, V: 1);
1827
1828 BasicBlock *Begin = Builder.GetInsertBlock();
1829 BasicBlock *End = createBasicBlock(name: "bitscan_end", parent: this->CurFn);
1830 Builder.SetInsertPoint(End);
1831 PHINode *Result = Builder.CreatePHI(Ty: ResultType, NumReservedValues: 2, Name: "bitscan_result");
1832
1833 Builder.SetInsertPoint(Begin);
1834 Value *IsZero = Builder.CreateICmpEQ(LHS: ArgValue, RHS: ArgZero);
1835 BasicBlock *NotZero = createBasicBlock(name: "bitscan_not_zero", parent: this->CurFn);
1836 Builder.CreateCondBr(Cond: IsZero, True: End, False: NotZero);
1837 Result->addIncoming(V: ResZero, BB: Begin);
1838
1839 Builder.SetInsertPoint(NotZero);
1840
1841 if (BuiltinID == MSVCIntrin::_BitScanForward) {
1842 Function *F = CGM.getIntrinsic(Intrinsic::cttz, ArgType);
1843 Value *ZeroCount = Builder.CreateCall(Callee: F, Args: {ArgValue, Builder.getTrue()});
1844 ZeroCount = Builder.CreateIntCast(V: ZeroCount, DestTy: IndexType, isSigned: false);
1845 Builder.CreateStore(Val: ZeroCount, Addr: IndexAddress, IsVolatile: false);
1846 } else {
1847 unsigned ArgWidth = cast<llvm::IntegerType>(Val: ArgType)->getBitWidth();
1848 Value *ArgTypeLastIndex = llvm::ConstantInt::get(Ty: IndexType, V: ArgWidth - 1);
1849
1850 Function *F = CGM.getIntrinsic(Intrinsic::ctlz, ArgType);
1851 Value *ZeroCount = Builder.CreateCall(Callee: F, Args: {ArgValue, Builder.getTrue()});
1852 ZeroCount = Builder.CreateIntCast(V: ZeroCount, DestTy: IndexType, isSigned: false);
1853 Value *Index = Builder.CreateNSWSub(LHS: ArgTypeLastIndex, RHS: ZeroCount);
1854 Builder.CreateStore(Val: Index, Addr: IndexAddress, IsVolatile: false);
1855 }
1856 Builder.CreateBr(Dest: End);
1857 Result->addIncoming(V: ResOne, BB: NotZero);
1858
1859 Builder.SetInsertPoint(End);
1860 return Result;
1861 }
1862 case MSVCIntrin::_InterlockedAnd:
1863 return MakeBinaryAtomicValue(CGF&: *this, Kind: AtomicRMWInst::And, E);
1864 case MSVCIntrin::_InterlockedExchange:
1865 return MakeBinaryAtomicValue(CGF&: *this, Kind: AtomicRMWInst::Xchg, E);
1866 case MSVCIntrin::_InterlockedExchangeAdd:
1867 return MakeBinaryAtomicValue(CGF&: *this, Kind: AtomicRMWInst::Add, E);
1868 case MSVCIntrin::_InterlockedExchangeSub:
1869 return MakeBinaryAtomicValue(CGF&: *this, Kind: AtomicRMWInst::Sub, E);
1870 case MSVCIntrin::_InterlockedOr:
1871 return MakeBinaryAtomicValue(CGF&: *this, Kind: AtomicRMWInst::Or, E);
1872 case MSVCIntrin::_InterlockedXor:
1873 return MakeBinaryAtomicValue(CGF&: *this, Kind: AtomicRMWInst::Xor, E);
1874 case MSVCIntrin::_InterlockedExchangeAdd_acq:
1875 return MakeBinaryAtomicValue(CGF&: *this, Kind: AtomicRMWInst::Add, E,
1876 Ordering: AtomicOrdering::Acquire);
1877 case MSVCIntrin::_InterlockedExchangeAdd_rel:
1878 return MakeBinaryAtomicValue(CGF&: *this, Kind: AtomicRMWInst::Add, E,
1879 Ordering: AtomicOrdering::Release);
1880 case MSVCIntrin::_InterlockedExchangeAdd_nf:
1881 return MakeBinaryAtomicValue(CGF&: *this, Kind: AtomicRMWInst::Add, E,
1882 Ordering: AtomicOrdering::Monotonic);
1883 case MSVCIntrin::_InterlockedExchange_acq:
1884 return MakeBinaryAtomicValue(CGF&: *this, Kind: AtomicRMWInst::Xchg, E,
1885 Ordering: AtomicOrdering::Acquire);
1886 case MSVCIntrin::_InterlockedExchange_rel:
1887 return MakeBinaryAtomicValue(CGF&: *this, Kind: AtomicRMWInst::Xchg, E,
1888 Ordering: AtomicOrdering::Release);
1889 case MSVCIntrin::_InterlockedExchange_nf:
1890 return MakeBinaryAtomicValue(CGF&: *this, Kind: AtomicRMWInst::Xchg, E,
1891 Ordering: AtomicOrdering::Monotonic);
1892 case MSVCIntrin::_InterlockedCompareExchange:
1893 return EmitAtomicCmpXchgForMSIntrin(CGF&: *this, E);
1894 case MSVCIntrin::_InterlockedCompareExchange_acq:
1895 return EmitAtomicCmpXchgForMSIntrin(CGF&: *this, E, SuccessOrdering: AtomicOrdering::Acquire);
1896 case MSVCIntrin::_InterlockedCompareExchange_rel:
1897 return EmitAtomicCmpXchgForMSIntrin(CGF&: *this, E, SuccessOrdering: AtomicOrdering::Release);
1898 case MSVCIntrin::_InterlockedCompareExchange_nf:
1899 return EmitAtomicCmpXchgForMSIntrin(CGF&: *this, E, SuccessOrdering: AtomicOrdering::Monotonic);
1900 case MSVCIntrin::_InterlockedCompareExchange128:
1901 return EmitAtomicCmpXchg128ForMSIntrin(
1902 CGF&: *this, E, SuccessOrdering: AtomicOrdering::SequentiallyConsistent);
1903 case MSVCIntrin::_InterlockedCompareExchange128_acq:
1904 return EmitAtomicCmpXchg128ForMSIntrin(CGF&: *this, E, SuccessOrdering: AtomicOrdering::Acquire);
1905 case MSVCIntrin::_InterlockedCompareExchange128_rel:
1906 return EmitAtomicCmpXchg128ForMSIntrin(CGF&: *this, E, SuccessOrdering: AtomicOrdering::Release);
1907 case MSVCIntrin::_InterlockedCompareExchange128_nf:
1908 return EmitAtomicCmpXchg128ForMSIntrin(CGF&: *this, E, SuccessOrdering: AtomicOrdering::Monotonic);
1909 case MSVCIntrin::_InterlockedOr_acq:
1910 return MakeBinaryAtomicValue(CGF&: *this, Kind: AtomicRMWInst::Or, E,
1911 Ordering: AtomicOrdering::Acquire);
1912 case MSVCIntrin::_InterlockedOr_rel:
1913 return MakeBinaryAtomicValue(CGF&: *this, Kind: AtomicRMWInst::Or, E,
1914 Ordering: AtomicOrdering::Release);
1915 case MSVCIntrin::_InterlockedOr_nf:
1916 return MakeBinaryAtomicValue(CGF&: *this, Kind: AtomicRMWInst::Or, E,
1917 Ordering: AtomicOrdering::Monotonic);
1918 case MSVCIntrin::_InterlockedXor_acq:
1919 return MakeBinaryAtomicValue(CGF&: *this, Kind: AtomicRMWInst::Xor, E,
1920 Ordering: AtomicOrdering::Acquire);
1921 case MSVCIntrin::_InterlockedXor_rel:
1922 return MakeBinaryAtomicValue(CGF&: *this, Kind: AtomicRMWInst::Xor, E,
1923 Ordering: AtomicOrdering::Release);
1924 case MSVCIntrin::_InterlockedXor_nf:
1925 return MakeBinaryAtomicValue(CGF&: *this, Kind: AtomicRMWInst::Xor, E,
1926 Ordering: AtomicOrdering::Monotonic);
1927 case MSVCIntrin::_InterlockedAnd_acq:
1928 return MakeBinaryAtomicValue(CGF&: *this, Kind: AtomicRMWInst::And, E,
1929 Ordering: AtomicOrdering::Acquire);
1930 case MSVCIntrin::_InterlockedAnd_rel:
1931 return MakeBinaryAtomicValue(CGF&: *this, Kind: AtomicRMWInst::And, E,
1932 Ordering: AtomicOrdering::Release);
1933 case MSVCIntrin::_InterlockedAnd_nf:
1934 return MakeBinaryAtomicValue(CGF&: *this, Kind: AtomicRMWInst::And, E,
1935 Ordering: AtomicOrdering::Monotonic);
1936 case MSVCIntrin::_InterlockedIncrement_acq:
1937 return EmitAtomicIncrementValue(CGF&: *this, E, Ordering: AtomicOrdering::Acquire);
1938 case MSVCIntrin::_InterlockedIncrement_rel:
1939 return EmitAtomicIncrementValue(CGF&: *this, E, Ordering: AtomicOrdering::Release);
1940 case MSVCIntrin::_InterlockedIncrement_nf:
1941 return EmitAtomicIncrementValue(CGF&: *this, E, Ordering: AtomicOrdering::Monotonic);
1942 case MSVCIntrin::_InterlockedDecrement_acq:
1943 return EmitAtomicDecrementValue(CGF&: *this, E, Ordering: AtomicOrdering::Acquire);
1944 case MSVCIntrin::_InterlockedDecrement_rel:
1945 return EmitAtomicDecrementValue(CGF&: *this, E, Ordering: AtomicOrdering::Release);
1946 case MSVCIntrin::_InterlockedDecrement_nf:
1947 return EmitAtomicDecrementValue(CGF&: *this, E, Ordering: AtomicOrdering::Monotonic);
1948
1949 case MSVCIntrin::_InterlockedDecrement:
1950 return EmitAtomicDecrementValue(CGF&: *this, E);
1951 case MSVCIntrin::_InterlockedIncrement:
1952 return EmitAtomicIncrementValue(CGF&: *this, E);
1953
1954 case MSVCIntrin::__fastfail: {
1955 // Request immediate process termination from the kernel. The instruction
1956 // sequences to do this are documented on MSDN:
1957 // https://msdn.microsoft.com/en-us/library/dn774154.aspx
1958 llvm::Triple::ArchType ISA = getTarget().getTriple().getArch();
1959 StringRef Asm, Constraints;
1960 switch (ISA) {
1961 default:
1962 ErrorUnsupported(E, "__fastfail call for this architecture");
1963 break;
1964 case llvm::Triple::x86:
1965 case llvm::Triple::x86_64:
1966 Asm = "int $$0x29";
1967 Constraints = "{cx}";
1968 break;
1969 case llvm::Triple::thumb:
1970 Asm = "udf #251";
1971 Constraints = "{r0}";
1972 break;
1973 case llvm::Triple::aarch64:
1974 Asm = "brk #0xF003";
1975 Constraints = "{w0}";
1976 }
1977 llvm::FunctionType *FTy = llvm::FunctionType::get(Result: VoidTy, Params: {Int32Ty}, isVarArg: false);
1978 llvm::InlineAsm *IA =
1979 llvm::InlineAsm::get(Ty: FTy, AsmString: Asm, Constraints, /*hasSideEffects=*/true);
1980 llvm::AttributeList NoReturnAttr = llvm::AttributeList::get(
1981 getLLVMContext(), llvm::AttributeList::FunctionIndex,
1982 llvm::Attribute::NoReturn);
1983 llvm::CallInst *CI = Builder.CreateCall(Callee: IA, Args: EmitScalarExpr(E: E->getArg(Arg: 0)));
1984 CI->setAttributes(NoReturnAttr);
1985 return CI;
1986 }
1987 }
1988 llvm_unreachable("Incorrect MSVC intrinsic!");
1989}
1990
1991namespace {
1992// ARC cleanup for __builtin_os_log_format
1993struct CallObjCArcUse final : EHScopeStack::Cleanup {
1994 CallObjCArcUse(llvm::Value *object) : object(object) {}
1995 llvm::Value *object;
1996
1997 void Emit(CodeGenFunction &CGF, Flags flags) override {
1998 CGF.EmitARCIntrinsicUse(values: object);
1999 }
2000};
2001}
2002
2003Value *CodeGenFunction::EmitCheckedArgForBuiltin(const Expr *E,
2004 BuiltinCheckKind Kind) {
2005 assert((Kind == BCK_CLZPassedZero || Kind == BCK_CTZPassedZero) &&
2006 "Unsupported builtin check kind");
2007
2008 Value *ArgValue = EmitScalarExpr(E);
2009 if (!SanOpts.has(K: SanitizerKind::Builtin))
2010 return ArgValue;
2011
2012 auto CheckOrdinal = SanitizerKind::SO_Builtin;
2013 auto CheckHandler = SanitizerHandler::InvalidBuiltin;
2014 SanitizerDebugLocation SanScope(this, {CheckOrdinal}, CheckHandler);
2015 Value *Cond = Builder.CreateICmpNE(
2016 LHS: ArgValue, RHS: llvm::Constant::getNullValue(Ty: ArgValue->getType()));
2017 EmitCheck(Checked: std::make_pair(x&: Cond, y&: CheckOrdinal), Check: CheckHandler,
2018 StaticArgs: {EmitCheckSourceLocation(Loc: E->getExprLoc()),
2019 llvm::ConstantInt::get(Ty: Builder.getInt8Ty(), V: Kind)},
2020 DynamicArgs: {});
2021 return ArgValue;
2022}
2023
2024Value *CodeGenFunction::EmitCheckedArgForAssume(const Expr *E) {
2025 Value *ArgValue = EvaluateExprAsBool(E);
2026 if (!SanOpts.has(K: SanitizerKind::Builtin))
2027 return ArgValue;
2028
2029 auto CheckOrdinal = SanitizerKind::SO_Builtin;
2030 auto CheckHandler = SanitizerHandler::InvalidBuiltin;
2031 SanitizerDebugLocation SanScope(this, {CheckOrdinal}, CheckHandler);
2032 EmitCheck(
2033 Checked: std::make_pair(x&: ArgValue, y&: CheckOrdinal), Check: CheckHandler,
2034 StaticArgs: {EmitCheckSourceLocation(Loc: E->getExprLoc()),
2035 llvm::ConstantInt::get(Ty: Builder.getInt8Ty(), V: BCK_AssumePassedFalse)},
2036 DynamicArgs: std::nullopt);
2037 return ArgValue;
2038}
2039
2040static Value *EmitAbs(CodeGenFunction &CGF, Value *ArgValue, bool HasNSW) {
2041 return CGF.Builder.CreateBinaryIntrinsic(
2042 Intrinsic::abs, ArgValue,
2043 ConstantInt::get(CGF.Builder.getInt1Ty(), HasNSW));
2044}
2045
2046static Value *EmitOverflowCheckedAbs(CodeGenFunction &CGF, const CallExpr *E,
2047 bool SanitizeOverflow) {
2048 Value *ArgValue = CGF.EmitScalarExpr(E: E->getArg(Arg: 0));
2049
2050 // Try to eliminate overflow check.
2051 if (const auto *VCI = dyn_cast<llvm::ConstantInt>(Val: ArgValue)) {
2052 if (!VCI->isMinSignedValue())
2053 return EmitAbs(CGF, ArgValue, HasNSW: true);
2054 }
2055
2056 SmallVector<SanitizerKind::SanitizerOrdinal, 1> Ordinals;
2057 SanitizerHandler CheckHandler;
2058 if (SanitizeOverflow) {
2059 Ordinals.push_back(Elt: SanitizerKind::SO_SignedIntegerOverflow);
2060 CheckHandler = SanitizerHandler::NegateOverflow;
2061 } else
2062 CheckHandler = SanitizerHandler::SubOverflow;
2063
2064 SanitizerDebugLocation SanScope(&CGF, Ordinals, CheckHandler);
2065
2066 Constant *Zero = Constant::getNullValue(Ty: ArgValue->getType());
2067 Value *ResultAndOverflow = CGF.Builder.CreateBinaryIntrinsic(
2068 Intrinsic::ssub_with_overflow, Zero, ArgValue);
2069 Value *Result = CGF.Builder.CreateExtractValue(Agg: ResultAndOverflow, Idxs: 0);
2070 Value *NotOverflow = CGF.Builder.CreateNot(
2071 V: CGF.Builder.CreateExtractValue(Agg: ResultAndOverflow, Idxs: 1));
2072
2073 // TODO: support -ftrapv-handler.
2074 if (SanitizeOverflow) {
2075 CGF.EmitCheck(Checked: {{NotOverflow, SanitizerKind::SO_SignedIntegerOverflow}},
2076 Check: CheckHandler,
2077 StaticArgs: {CGF.EmitCheckSourceLocation(Loc: E->getArg(Arg: 0)->getExprLoc()),
2078 CGF.EmitCheckTypeDescriptor(T: E->getType())},
2079 DynamicArgs: {ArgValue});
2080 } else
2081 CGF.EmitTrapCheck(Checked: NotOverflow, CheckHandlerID: CheckHandler);
2082
2083 Value *CmpResult = CGF.Builder.CreateICmpSLT(LHS: ArgValue, RHS: Zero, Name: "abscond");
2084 return CGF.Builder.CreateSelect(C: CmpResult, True: Result, False: ArgValue, Name: "abs");
2085}
2086
2087/// Get the argument type for arguments to os_log_helper.
2088static CanQualType getOSLogArgType(ASTContext &C, int Size) {
2089 QualType UnsignedTy = C.getIntTypeForBitwidth(DestWidth: Size * 8, /*Signed=*/false);
2090 return C.getCanonicalType(T: UnsignedTy);
2091}
2092
2093llvm::Function *CodeGenFunction::generateBuiltinOSLogHelperFunction(
2094 const analyze_os_log::OSLogBufferLayout &Layout,
2095 CharUnits BufferAlignment) {
2096 ASTContext &Ctx = getContext();
2097
2098 llvm::SmallString<64> Name;
2099 {
2100 raw_svector_ostream OS(Name);
2101 OS << "__os_log_helper";
2102 OS << "_" << BufferAlignment.getQuantity();
2103 OS << "_" << int(Layout.getSummaryByte());
2104 OS << "_" << int(Layout.getNumArgsByte());
2105 for (const auto &Item : Layout.Items)
2106 OS << "_" << int(Item.getSizeByte()) << "_"
2107 << int(Item.getDescriptorByte());
2108 }
2109
2110 if (llvm::Function *F = CGM.getModule().getFunction(Name))
2111 return F;
2112
2113 llvm::SmallVector<QualType, 4> ArgTys;
2114 FunctionArgList Args;
2115 Args.push_back(Elt: ImplicitParamDecl::Create(
2116 Ctx, nullptr, SourceLocation(), &Ctx.Idents.get(Name: "buffer"), Ctx.VoidPtrTy,
2117 ImplicitParamKind::Other));
2118 ArgTys.emplace_back(Ctx.VoidPtrTy);
2119
2120 for (unsigned int I = 0, E = Layout.Items.size(); I < E; ++I) {
2121 char Size = Layout.Items[I].getSizeByte();
2122 if (!Size)
2123 continue;
2124
2125 QualType ArgTy = getOSLogArgType(C&: Ctx, Size);
2126 Args.push_back(ImplicitParamDecl::Create(
2127 C&: Ctx, DC: nullptr, IdLoc: SourceLocation(),
2128 Id: &Ctx.Idents.get(Name: std::string("arg") + llvm::to_string(Value: I)), T: ArgTy,
2129 ParamKind: ImplicitParamKind::Other));
2130 ArgTys.emplace_back(Args&: ArgTy);
2131 }
2132
2133 QualType ReturnTy = Ctx.VoidTy;
2134
2135 // The helper function has linkonce_odr linkage to enable the linker to merge
2136 // identical functions. To ensure the merging always happens, 'noinline' is
2137 // attached to the function when compiling with -Oz.
2138 const CGFunctionInfo &FI =
2139 CGM.getTypes().arrangeBuiltinFunctionDeclaration(resultType: ReturnTy, args: Args);
2140 llvm::FunctionType *FuncTy = CGM.getTypes().GetFunctionType(Info: FI);
2141 llvm::Function *Fn = llvm::Function::Create(
2142 Ty: FuncTy, Linkage: llvm::GlobalValue::LinkOnceODRLinkage, N: Name, M: &CGM.getModule());
2143 Fn->setVisibility(llvm::GlobalValue::HiddenVisibility);
2144 CGM.SetLLVMFunctionAttributes(GD: GlobalDecl(), Info: FI, F: Fn, /*IsThunk=*/false);
2145 CGM.SetLLVMFunctionAttributesForDefinition(D: nullptr, F: Fn);
2146 Fn->setDoesNotThrow();
2147
2148 // Attach 'noinline' at -Oz.
2149 if (CGM.getCodeGenOpts().OptimizeSize == 2)
2150 Fn->addFnAttr(llvm::Attribute::NoInline);
2151
2152 auto NL = ApplyDebugLocation::CreateEmpty(CGF&: *this);
2153 StartFunction(GD: GlobalDecl(), RetTy: ReturnTy, Fn, FnInfo: FI, Args);
2154
2155 // Create a scope with an artificial location for the body of this function.
2156 auto AL = ApplyDebugLocation::CreateArtificial(CGF&: *this);
2157
2158 CharUnits Offset;
2159 Address BufAddr = makeNaturalAddressForPointer(
2160 Ptr: Builder.CreateLoad(Addr: GetAddrOfLocalVar(VD: Args[0]), Name: "buf"), T: Ctx.VoidTy,
2161 Alignment: BufferAlignment);
2162 Builder.CreateStore(Val: Builder.getInt8(C: Layout.getSummaryByte()),
2163 Addr: Builder.CreateConstByteGEP(Addr: BufAddr, Offset: Offset++, Name: "summary"));
2164 Builder.CreateStore(Val: Builder.getInt8(C: Layout.getNumArgsByte()),
2165 Addr: Builder.CreateConstByteGEP(Addr: BufAddr, Offset: Offset++, Name: "numArgs"));
2166
2167 unsigned I = 1;
2168 for (const auto &Item : Layout.Items) {
2169 Builder.CreateStore(
2170 Val: Builder.getInt8(C: Item.getDescriptorByte()),
2171 Addr: Builder.CreateConstByteGEP(Addr: BufAddr, Offset: Offset++, Name: "argDescriptor"));
2172 Builder.CreateStore(
2173 Val: Builder.getInt8(C: Item.getSizeByte()),
2174 Addr: Builder.CreateConstByteGEP(Addr: BufAddr, Offset: Offset++, Name: "argSize"));
2175
2176 CharUnits Size = Item.size();
2177 if (!Size.getQuantity())
2178 continue;
2179
2180 Address Arg = GetAddrOfLocalVar(VD: Args[I]);
2181 Address Addr = Builder.CreateConstByteGEP(Addr: BufAddr, Offset, Name: "argData");
2182 Addr = Addr.withElementType(ElemTy: Arg.getElementType());
2183 Builder.CreateStore(Val: Builder.CreateLoad(Addr: Arg), Addr);
2184 Offset += Size;
2185 ++I;
2186 }
2187
2188 FinishFunction();
2189
2190 return Fn;
2191}
2192
2193RValue CodeGenFunction::emitBuiltinOSLogFormat(const CallExpr &E) {
2194 assert(E.getNumArgs() >= 2 &&
2195 "__builtin_os_log_format takes at least 2 arguments");
2196 ASTContext &Ctx = getContext();
2197 analyze_os_log::OSLogBufferLayout Layout;
2198 analyze_os_log::computeOSLogBufferLayout(Ctx, E: &E, layout&: Layout);
2199 Address BufAddr = EmitPointerWithAlignment(Addr: E.getArg(Arg: 0));
2200
2201 // Ignore argument 1, the format string. It is not currently used.
2202 CallArgList Args;
2203 Args.add(rvalue: RValue::get(V: BufAddr.emitRawPointer(CGF&: *this)), type: Ctx.VoidPtrTy);
2204
2205 for (const auto &Item : Layout.Items) {
2206 int Size = Item.getSizeByte();
2207 if (!Size)
2208 continue;
2209
2210 llvm::Value *ArgVal;
2211
2212 if (Item.getKind() == analyze_os_log::OSLogBufferItem::MaskKind) {
2213 uint64_t Val = 0;
2214 for (unsigned I = 0, E = Item.getMaskType().size(); I < E; ++I)
2215 Val |= ((uint64_t)Item.getMaskType()[I]) << I * 8;
2216 ArgVal = llvm::Constant::getIntegerValue(Ty: Int64Ty, V: llvm::APInt(64, Val));
2217 } else if (const Expr *TheExpr = Item.getExpr()) {
2218 ArgVal = EmitScalarExpr(E: TheExpr, /*Ignore*/ IgnoreResultAssign: false);
2219
2220 // If a temporary object that requires destruction after the full
2221 // expression is passed, push a lifetime-extended cleanup to extend its
2222 // lifetime to the end of the enclosing block scope.
2223 auto LifetimeExtendObject = [&](const Expr *E) {
2224 E = E->IgnoreParenCasts();
2225 // Extend lifetimes of objects returned by function calls and message
2226 // sends.
2227
2228 // FIXME: We should do this in other cases in which temporaries are
2229 // created including arguments of non-ARC types (e.g., C++
2230 // temporaries).
2231 if (isa<CallExpr>(Val: E) || isa<ObjCMessageExpr>(Val: E))
2232 return true;
2233 return false;
2234 };
2235
2236 if (TheExpr->getType()->isObjCRetainableType() &&
2237 getLangOpts().ObjCAutoRefCount && LifetimeExtendObject(TheExpr)) {
2238 assert(getEvaluationKind(TheExpr->getType()) == TEK_Scalar &&
2239 "Only scalar can be a ObjC retainable type");
2240 if (!isa<Constant>(Val: ArgVal)) {
2241 CleanupKind Cleanup = getARCCleanupKind();
2242 QualType Ty = TheExpr->getType();
2243 RawAddress Alloca = RawAddress::invalid();
2244 RawAddress Addr = CreateMemTemp(T: Ty, Name: "os.log.arg", Alloca: &Alloca);
2245 ArgVal = EmitARCRetain(type: Ty, value: ArgVal);
2246 Builder.CreateStore(Val: ArgVal, Addr);
2247 pushLifetimeExtendedDestroy(kind: Cleanup, addr: Alloca, type: Ty,
2248 destroyer: CodeGenFunction::destroyARCStrongPrecise,
2249 useEHCleanupForArray: Cleanup & EHCleanup);
2250
2251 // Push a clang.arc.use call to ensure ARC optimizer knows that the
2252 // argument has to be alive.
2253 if (CGM.getCodeGenOpts().OptimizationLevel != 0)
2254 pushCleanupAfterFullExpr<CallObjCArcUse>(Kind: Cleanup, A: ArgVal);
2255 }
2256 }
2257 } else {
2258 ArgVal = Builder.getInt32(C: Item.getConstValue().getQuantity());
2259 }
2260
2261 unsigned ArgValSize =
2262 CGM.getDataLayout().getTypeSizeInBits(Ty: ArgVal->getType());
2263 llvm::IntegerType *IntTy = llvm::Type::getIntNTy(C&: getLLVMContext(),
2264 N: ArgValSize);
2265 ArgVal = Builder.CreateBitOrPointerCast(V: ArgVal, DestTy: IntTy);
2266 CanQualType ArgTy = getOSLogArgType(C&: Ctx, Size);
2267 // If ArgVal has type x86_fp80, zero-extend ArgVal.
2268 ArgVal = Builder.CreateZExtOrBitCast(V: ArgVal, DestTy: ConvertType(T: ArgTy));
2269 Args.add(rvalue: RValue::get(V: ArgVal), type: ArgTy);
2270 }
2271
2272 const CGFunctionInfo &FI =
2273 CGM.getTypes().arrangeBuiltinFunctionCall(resultType: Ctx.VoidTy, args: Args);
2274 llvm::Function *F = CodeGenFunction(CGM).generateBuiltinOSLogHelperFunction(
2275 Layout, BufferAlignment: BufAddr.getAlignment());
2276 EmitCall(CallInfo: FI, Callee: CGCallee::forDirect(functionPtr: F), ReturnValue: ReturnValueSlot(), Args);
2277 return RValue::get(Addr: BufAddr, CGF&: *this);
2278}
2279
2280static bool isSpecialUnsignedMultiplySignedResult(
2281 unsigned BuiltinID, WidthAndSignedness Op1Info, WidthAndSignedness Op2Info,
2282 WidthAndSignedness ResultInfo) {
2283 return BuiltinID == Builtin::BI__builtin_mul_overflow &&
2284 Op1Info.Width == Op2Info.Width && Op2Info.Width == ResultInfo.Width &&
2285 !Op1Info.Signed && !Op2Info.Signed && ResultInfo.Signed;
2286}
2287
2288static RValue EmitCheckedUnsignedMultiplySignedResult(
2289 CodeGenFunction &CGF, const clang::Expr *Op1, WidthAndSignedness Op1Info,
2290 const clang::Expr *Op2, WidthAndSignedness Op2Info,
2291 const clang::Expr *ResultArg, QualType ResultQTy,
2292 WidthAndSignedness ResultInfo) {
2293 assert(isSpecialUnsignedMultiplySignedResult(
2294 Builtin::BI__builtin_mul_overflow, Op1Info, Op2Info, ResultInfo) &&
2295 "Cannot specialize this multiply");
2296
2297 llvm::Value *V1 = CGF.EmitScalarExpr(E: Op1);
2298 llvm::Value *V2 = CGF.EmitScalarExpr(E: Op2);
2299
2300 llvm::Value *HasOverflow;
2301 llvm::Value *Result = EmitOverflowIntrinsic(
2302 CGF, Intrinsic::umul_with_overflow, V1, V2, HasOverflow);
2303
2304 // The intrinsic call will detect overflow when the value is > UINT_MAX,
2305 // however, since the original builtin had a signed result, we need to report
2306 // an overflow when the result is greater than INT_MAX.
2307 auto IntMax = llvm::APInt::getSignedMaxValue(numBits: ResultInfo.Width);
2308 llvm::Value *IntMaxValue = llvm::ConstantInt::get(Ty: Result->getType(), V: IntMax);
2309
2310 llvm::Value *IntMaxOverflow = CGF.Builder.CreateICmpUGT(LHS: Result, RHS: IntMaxValue);
2311 HasOverflow = CGF.Builder.CreateOr(LHS: HasOverflow, RHS: IntMaxOverflow);
2312
2313 bool isVolatile =
2314 ResultArg->getType()->getPointeeType().isVolatileQualified();
2315 Address ResultPtr = CGF.EmitPointerWithAlignment(Addr: ResultArg);
2316 CGF.Builder.CreateStore(Val: CGF.EmitToMemory(Value: Result, Ty: ResultQTy), Addr: ResultPtr,
2317 IsVolatile: isVolatile);
2318 return RValue::get(V: HasOverflow);
2319}
2320
2321/// Determine if a binop is a checked mixed-sign multiply we can specialize.
2322static bool isSpecialMixedSignMultiply(unsigned BuiltinID,
2323 WidthAndSignedness Op1Info,
2324 WidthAndSignedness Op2Info,
2325 WidthAndSignedness ResultInfo) {
2326 return BuiltinID == Builtin::BI__builtin_mul_overflow &&
2327 std::max(Op1Info.Width, Op2Info.Width) >= ResultInfo.Width &&
2328 Op1Info.Signed != Op2Info.Signed;
2329}
2330
2331/// Emit a checked mixed-sign multiply. This is a cheaper specialization of
2332/// the generic checked-binop irgen.
2333static RValue
2334EmitCheckedMixedSignMultiply(CodeGenFunction &CGF, const clang::Expr *Op1,
2335 WidthAndSignedness Op1Info, const clang::Expr *Op2,
2336 WidthAndSignedness Op2Info,
2337 const clang::Expr *ResultArg, QualType ResultQTy,
2338 WidthAndSignedness ResultInfo) {
2339 assert(isSpecialMixedSignMultiply(Builtin::BI__builtin_mul_overflow, Op1Info,
2340 Op2Info, ResultInfo) &&
2341 "Not a mixed-sign multipliction we can specialize");
2342
2343 // Emit the signed and unsigned operands.
2344 const clang::Expr *SignedOp = Op1Info.Signed ? Op1 : Op2;
2345 const clang::Expr *UnsignedOp = Op1Info.Signed ? Op2 : Op1;
2346 llvm::Value *Signed = CGF.EmitScalarExpr(E: SignedOp);
2347 llvm::Value *Unsigned = CGF.EmitScalarExpr(E: UnsignedOp);
2348 unsigned SignedOpWidth = Op1Info.Signed ? Op1Info.Width : Op2Info.Width;
2349 unsigned UnsignedOpWidth = Op1Info.Signed ? Op2Info.Width : Op1Info.Width;
2350
2351 // One of the operands may be smaller than the other. If so, [s|z]ext it.
2352 if (SignedOpWidth < UnsignedOpWidth)
2353 Signed = CGF.Builder.CreateSExt(V: Signed, DestTy: Unsigned->getType(), Name: "op.sext");
2354 if (UnsignedOpWidth < SignedOpWidth)
2355 Unsigned = CGF.Builder.CreateZExt(V: Unsigned, DestTy: Signed->getType(), Name: "op.zext");
2356
2357 llvm::Type *OpTy = Signed->getType();
2358 llvm::Value *Zero = llvm::Constant::getNullValue(Ty: OpTy);
2359 Address ResultPtr = CGF.EmitPointerWithAlignment(Addr: ResultArg);
2360 llvm::Type *ResTy = ResultPtr.getElementType();
2361 unsigned OpWidth = std::max(a: Op1Info.Width, b: Op2Info.Width);
2362
2363 // Take the absolute value of the signed operand.
2364 llvm::Value *IsNegative = CGF.Builder.CreateICmpSLT(LHS: Signed, RHS: Zero);
2365 llvm::Value *AbsOfNegative = CGF.Builder.CreateSub(LHS: Zero, RHS: Signed);
2366 llvm::Value *AbsSigned =
2367 CGF.Builder.CreateSelect(C: IsNegative, True: AbsOfNegative, False: Signed);
2368
2369 // Perform a checked unsigned multiplication.
2370 llvm::Value *UnsignedOverflow;
2371 llvm::Value *UnsignedResult =
2372 EmitOverflowIntrinsic(CGF, Intrinsic::umul_with_overflow, AbsSigned,
2373 Unsigned, UnsignedOverflow);
2374
2375 llvm::Value *Overflow, *Result;
2376 if (ResultInfo.Signed) {
2377 // Signed overflow occurs if the result is greater than INT_MAX or lesser
2378 // than INT_MIN, i.e when |Result| > (INT_MAX + IsNegative).
2379 auto IntMax =
2380 llvm::APInt::getSignedMaxValue(numBits: ResultInfo.Width).zext(width: OpWidth);
2381 llvm::Value *MaxResult =
2382 CGF.Builder.CreateAdd(LHS: llvm::ConstantInt::get(Ty: OpTy, V: IntMax),
2383 RHS: CGF.Builder.CreateZExt(V: IsNegative, DestTy: OpTy));
2384 llvm::Value *SignedOverflow =
2385 CGF.Builder.CreateICmpUGT(LHS: UnsignedResult, RHS: MaxResult);
2386 Overflow = CGF.Builder.CreateOr(LHS: UnsignedOverflow, RHS: SignedOverflow);
2387
2388 // Prepare the signed result (possibly by negating it).
2389 llvm::Value *NegativeResult = CGF.Builder.CreateNeg(V: UnsignedResult);
2390 llvm::Value *SignedResult =
2391 CGF.Builder.CreateSelect(C: IsNegative, True: NegativeResult, False: UnsignedResult);
2392 Result = CGF.Builder.CreateTrunc(V: SignedResult, DestTy: ResTy);
2393 } else {
2394 // Unsigned overflow occurs if the result is < 0 or greater than UINT_MAX.
2395 llvm::Value *Underflow = CGF.Builder.CreateAnd(
2396 LHS: IsNegative, RHS: CGF.Builder.CreateIsNotNull(Arg: UnsignedResult));
2397 Overflow = CGF.Builder.CreateOr(LHS: UnsignedOverflow, RHS: Underflow);
2398 if (ResultInfo.Width < OpWidth) {
2399 auto IntMax =
2400 llvm::APInt::getMaxValue(numBits: ResultInfo.Width).zext(width: OpWidth);
2401 llvm::Value *TruncOverflow = CGF.Builder.CreateICmpUGT(
2402 LHS: UnsignedResult, RHS: llvm::ConstantInt::get(Ty: OpTy, V: IntMax));
2403 Overflow = CGF.Builder.CreateOr(LHS: Overflow, RHS: TruncOverflow);
2404 }
2405
2406 // Negate the product if it would be negative in infinite precision.
2407 Result = CGF.Builder.CreateSelect(
2408 C: IsNegative, True: CGF.Builder.CreateNeg(V: UnsignedResult), False: UnsignedResult);
2409
2410 Result = CGF.Builder.CreateTrunc(V: Result, DestTy: ResTy);
2411 }
2412 assert(Overflow && Result && "Missing overflow or result");
2413
2414 bool isVolatile =
2415 ResultArg->getType()->getPointeeType().isVolatileQualified();
2416 CGF.Builder.CreateStore(Val: CGF.EmitToMemory(Value: Result, Ty: ResultQTy), Addr: ResultPtr,
2417 IsVolatile: isVolatile);
2418 return RValue::get(V: Overflow);
2419}
2420
2421static bool
2422TypeRequiresBuiltinLaunderImp(const ASTContext &Ctx, QualType Ty,
2423 llvm::SmallPtrSetImpl<const Decl *> &Seen) {
2424 if (const auto *Arr = Ctx.getAsArrayType(T: Ty))
2425 Ty = Ctx.getBaseElementType(VAT: Arr);
2426
2427 const auto *Record = Ty->getAsCXXRecordDecl();
2428 if (!Record)
2429 return false;
2430
2431 // We've already checked this type, or are in the process of checking it.
2432 if (!Seen.insert(Record).second)
2433 return false;
2434
2435 assert(Record->hasDefinition() &&
2436 "Incomplete types should already be diagnosed");
2437
2438 if (Record->isDynamicClass())
2439 return true;
2440
2441 for (FieldDecl *F : Record->fields()) {
2442 if (TypeRequiresBuiltinLaunderImp(Ctx, F->getType(), Seen))
2443 return true;
2444 }
2445 return false;
2446}
2447
2448/// Determine if the specified type requires laundering by checking if it is a
2449/// dynamic class type or contains a subobject which is a dynamic class type.
2450static bool TypeRequiresBuiltinLaunder(CodeGenModule &CGM, QualType Ty) {
2451 if (!CGM.getCodeGenOpts().StrictVTablePointers)
2452 return false;
2453 llvm::SmallPtrSet<const Decl *, 16> Seen;
2454 return TypeRequiresBuiltinLaunderImp(Ctx: CGM.getContext(), Ty, Seen);
2455}
2456
2457RValue CodeGenFunction::emitRotate(const CallExpr *E, bool IsRotateRight) {
2458 llvm::Value *Src = EmitScalarExpr(E: E->getArg(Arg: 0));
2459 llvm::Value *ShiftAmt = EmitScalarExpr(E: E->getArg(Arg: 1));
2460
2461 // The builtin's shift arg may have a different type than the source arg and
2462 // result, but the LLVM intrinsic uses the same type for all values.
2463 llvm::Type *Ty = Src->getType();
2464 ShiftAmt = Builder.CreateIntCast(V: ShiftAmt, DestTy: Ty, isSigned: false);
2465
2466 // Rotate is a special case of LLVM funnel shift - 1st 2 args are the same.
2467 unsigned IID = IsRotateRight ? Intrinsic::fshr : Intrinsic::fshl;
2468 Function *F = CGM.getIntrinsic(IID, Tys: Ty);
2469 return RValue::get(V: Builder.CreateCall(Callee: F, Args: { Src, Src, ShiftAmt }));
2470}
2471
2472// Map math builtins for long-double to f128 version.
2473static unsigned mutateLongDoubleBuiltin(unsigned BuiltinID) {
2474 switch (BuiltinID) {
2475#define MUTATE_LDBL(func) \
2476 case Builtin::BI__builtin_##func##l: \
2477 return Builtin::BI__builtin_##func##f128;
2478 MUTATE_LDBL(sqrt)
2479 MUTATE_LDBL(cbrt)
2480 MUTATE_LDBL(fabs)
2481 MUTATE_LDBL(log)
2482 MUTATE_LDBL(log2)
2483 MUTATE_LDBL(log10)
2484 MUTATE_LDBL(log1p)
2485 MUTATE_LDBL(logb)
2486 MUTATE_LDBL(exp)
2487 MUTATE_LDBL(exp2)
2488 MUTATE_LDBL(expm1)
2489 MUTATE_LDBL(fdim)
2490 MUTATE_LDBL(hypot)
2491 MUTATE_LDBL(ilogb)
2492 MUTATE_LDBL(pow)
2493 MUTATE_LDBL(fmin)
2494 MUTATE_LDBL(fmax)
2495 MUTATE_LDBL(ceil)
2496 MUTATE_LDBL(trunc)
2497 MUTATE_LDBL(rint)
2498 MUTATE_LDBL(nearbyint)
2499 MUTATE_LDBL(round)
2500 MUTATE_LDBL(floor)
2501 MUTATE_LDBL(lround)
2502 MUTATE_LDBL(llround)
2503 MUTATE_LDBL(lrint)
2504 MUTATE_LDBL(llrint)
2505 MUTATE_LDBL(fmod)
2506 MUTATE_LDBL(modf)
2507 MUTATE_LDBL(nan)
2508 MUTATE_LDBL(nans)
2509 MUTATE_LDBL(inf)
2510 MUTATE_LDBL(fma)
2511 MUTATE_LDBL(sin)
2512 MUTATE_LDBL(cos)
2513 MUTATE_LDBL(tan)
2514 MUTATE_LDBL(sinh)
2515 MUTATE_LDBL(cosh)
2516 MUTATE_LDBL(tanh)
2517 MUTATE_LDBL(asin)
2518 MUTATE_LDBL(acos)
2519 MUTATE_LDBL(atan)
2520 MUTATE_LDBL(asinh)
2521 MUTATE_LDBL(acosh)
2522 MUTATE_LDBL(atanh)
2523 MUTATE_LDBL(atan2)
2524 MUTATE_LDBL(erf)
2525 MUTATE_LDBL(erfc)
2526 MUTATE_LDBL(ldexp)
2527 MUTATE_LDBL(frexp)
2528 MUTATE_LDBL(huge_val)
2529 MUTATE_LDBL(copysign)
2530 MUTATE_LDBL(nextafter)
2531 MUTATE_LDBL(nexttoward)
2532 MUTATE_LDBL(remainder)
2533 MUTATE_LDBL(remquo)
2534 MUTATE_LDBL(scalbln)
2535 MUTATE_LDBL(scalbn)
2536 MUTATE_LDBL(tgamma)
2537 MUTATE_LDBL(lgamma)
2538#undef MUTATE_LDBL
2539 default:
2540 return BuiltinID;
2541 }
2542}
2543
2544static Value *tryUseTestFPKind(CodeGenFunction &CGF, unsigned BuiltinID,
2545 Value *V) {
2546 if (CGF.Builder.getIsFPConstrained() &&
2547 CGF.Builder.getDefaultConstrainedExcept() != fp::ebIgnore) {
2548 if (Value *Result =
2549 CGF.getTargetHooks().testFPKind(V, BuiltinID, Builder&: CGF.Builder, CGM&: CGF.CGM))
2550 return Result;
2551 }
2552 return nullptr;
2553}
2554
2555static RValue EmitHipStdParUnsupportedBuiltin(CodeGenFunction *CGF,
2556 const FunctionDecl *FD) {
2557 auto Name = FD->getNameAsString() + "__hipstdpar_unsupported";
2558 auto FnTy = CGF->CGM.getTypes().GetFunctionType(GD: FD);
2559 auto UBF = CGF->CGM.getModule().getOrInsertFunction(Name, FnTy);
2560
2561 SmallVector<Value *, 16> Args;
2562 for (auto &&FormalTy : FnTy->params())
2563 Args.push_back(Elt: llvm::PoisonValue::get(T: FormalTy));
2564
2565 return RValue::get(CGF->Builder.CreateCall(UBF, Args));
2566}
2567
2568RValue CodeGenFunction::EmitBuiltinExpr(const GlobalDecl GD, unsigned BuiltinID,
2569 const CallExpr *E,
2570 ReturnValueSlot ReturnValue) {
2571 assert(!getContext().BuiltinInfo.isImmediate(BuiltinID) &&
2572 "Should not codegen for consteval builtins");
2573
2574 const FunctionDecl *FD = GD.getDecl()->getAsFunction();
2575 // See if we can constant fold this builtin. If so, don't emit it at all.
2576 // TODO: Extend this handling to all builtin calls that we can constant-fold.
2577 Expr::EvalResult Result;
2578 if (E->isPRValue() && E->EvaluateAsRValue(Result, CGM.getContext()) &&
2579 !Result.hasSideEffects()) {
2580 if (Result.Val.isInt())
2581 return RValue::get(V: llvm::ConstantInt::get(Context&: getLLVMContext(),
2582 V: Result.Val.getInt()));
2583 if (Result.Val.isFloat())
2584 return RValue::get(V: llvm::ConstantFP::get(Context&: getLLVMContext(),
2585 V: Result.Val.getFloat()));
2586 }
2587
2588 // If current long-double semantics is IEEE 128-bit, replace math builtins
2589 // of long-double with f128 equivalent.
2590 // TODO: This mutation should also be applied to other targets other than PPC,
2591 // after backend supports IEEE 128-bit style libcalls.
2592 if (getTarget().getTriple().isPPC64() &&
2593 &getTarget().getLongDoubleFormat() == &llvm::APFloat::IEEEquad())
2594 BuiltinID = mutateLongDoubleBuiltin(BuiltinID);
2595
2596 // If the builtin has been declared explicitly with an assembler label,
2597 // disable the specialized emitting below. Ideally we should communicate the
2598 // rename in IR, or at least avoid generating the intrinsic calls that are
2599 // likely to get lowered to the renamed library functions.
2600 const unsigned BuiltinIDIfNoAsmLabel =
2601 FD->hasAttr<AsmLabelAttr>() ? 0 : BuiltinID;
2602
2603 std::optional<bool> ErrnoOverriden;
2604 // ErrnoOverriden is true if math-errno is overriden via the
2605 // '#pragma float_control(precise, on)'. This pragma disables fast-math,
2606 // which implies math-errno.
2607 if (E->hasStoredFPFeatures()) {
2608 FPOptionsOverride OP = E->getFPFeatures();
2609 if (OP.hasMathErrnoOverride())
2610 ErrnoOverriden = OP.getMathErrnoOverride();
2611 }
2612 // True if 'attribute__((optnone))' is used. This attribute overrides
2613 // fast-math which implies math-errno.
2614 bool OptNone = CurFuncDecl && CurFuncDecl->hasAttr<OptimizeNoneAttr>();
2615
2616 // True if we are compiling at -O2 and errno has been disabled
2617 // using the '#pragma float_control(precise, off)', and
2618 // attribute opt-none hasn't been seen.
2619 bool ErrnoOverridenToFalseWithOpt =
2620 ErrnoOverriden.has_value() && !ErrnoOverriden.value() && !OptNone &&
2621 CGM.getCodeGenOpts().OptimizationLevel != 0;
2622
2623 // There are LLVM math intrinsics/instructions corresponding to math library
2624 // functions except the LLVM op will never set errno while the math library
2625 // might. Also, math builtins have the same semantics as their math library
2626 // twins. Thus, we can transform math library and builtin calls to their
2627 // LLVM counterparts if the call is marked 'const' (known to never set errno).
2628 // In case FP exceptions are enabled, the experimental versions of the
2629 // intrinsics model those.
2630 bool ConstAlways =
2631 getContext().BuiltinInfo.isConst(ID: BuiltinID);
2632
2633 // There's a special case with the fma builtins where they are always const
2634 // if the target environment is GNU or the target is OS is Windows and we're
2635 // targeting the MSVCRT.dll environment.
2636 // FIXME: This list can be become outdated. Need to find a way to get it some
2637 // other way.
2638 switch (BuiltinID) {
2639 case Builtin::BI__builtin_fma:
2640 case Builtin::BI__builtin_fmaf:
2641 case Builtin::BI__builtin_fmal:
2642 case Builtin::BI__builtin_fmaf16:
2643 case Builtin::BIfma:
2644 case Builtin::BIfmaf:
2645 case Builtin::BIfmal: {
2646 auto &Trip = CGM.getTriple();
2647 if (Trip.isGNUEnvironment() || Trip.isOSMSVCRT())
2648 ConstAlways = true;
2649 break;
2650 }
2651 default:
2652 break;
2653 }
2654
2655 bool ConstWithoutErrnoAndExceptions =
2656 getContext().BuiltinInfo.isConstWithoutErrnoAndExceptions(ID: BuiltinID);
2657 bool ConstWithoutExceptions =
2658 getContext().BuiltinInfo.isConstWithoutExceptions(ID: BuiltinID);
2659
2660 // ConstAttr is enabled in fast-math mode. In fast-math mode, math-errno is
2661 // disabled.
2662 // Math intrinsics are generated only when math-errno is disabled. Any pragmas
2663 // or attributes that affect math-errno should prevent or allow math
2664 // intrinsics to be generated. Intrinsics are generated:
2665 // 1- In fast math mode, unless math-errno is overriden
2666 // via '#pragma float_control(precise, on)', or via an
2667 // 'attribute__((optnone))'.
2668 // 2- If math-errno was enabled on command line but overriden
2669 // to false via '#pragma float_control(precise, off))' and
2670 // 'attribute__((optnone))' hasn't been used.
2671 // 3- If we are compiling with optimization and errno has been disabled
2672 // via '#pragma float_control(precise, off)', and
2673 // 'attribute__((optnone))' hasn't been used.
2674
2675 bool ConstWithoutErrnoOrExceptions =
2676 ConstWithoutErrnoAndExceptions || ConstWithoutExceptions;
2677 bool GenerateIntrinsics =
2678 (ConstAlways && !OptNone) ||
2679 (!getLangOpts().MathErrno &&
2680 !(ErrnoOverriden.has_value() && ErrnoOverriden.value()) && !OptNone);
2681 if (!GenerateIntrinsics) {
2682 GenerateIntrinsics =
2683 ConstWithoutErrnoOrExceptions && !ConstWithoutErrnoAndExceptions;
2684 if (!GenerateIntrinsics)
2685 GenerateIntrinsics =
2686 ConstWithoutErrnoOrExceptions &&
2687 (!getLangOpts().MathErrno &&
2688 !(ErrnoOverriden.has_value() && ErrnoOverriden.value()) && !OptNone);
2689 if (!GenerateIntrinsics)
2690 GenerateIntrinsics =
2691 ConstWithoutErrnoOrExceptions && ErrnoOverridenToFalseWithOpt;
2692 }
2693 if (GenerateIntrinsics) {
2694 switch (BuiltinIDIfNoAsmLabel) {
2695 case Builtin::BIacos:
2696 case Builtin::BIacosf:
2697 case Builtin::BIacosl:
2698 case Builtin::BI__builtin_acos:
2699 case Builtin::BI__builtin_acosf:
2700 case Builtin::BI__builtin_acosf16:
2701 case Builtin::BI__builtin_acosl:
2702 case Builtin::BI__builtin_acosf128:
2703 return RValue::get(emitUnaryMaybeConstrainedFPBuiltin(
2704 *this, E, Intrinsic::acos, Intrinsic::experimental_constrained_acos));
2705
2706 case Builtin::BIasin:
2707 case Builtin::BIasinf:
2708 case Builtin::BIasinl:
2709 case Builtin::BI__builtin_asin:
2710 case Builtin::BI__builtin_asinf:
2711 case Builtin::BI__builtin_asinf16:
2712 case Builtin::BI__builtin_asinl:
2713 case Builtin::BI__builtin_asinf128:
2714 return RValue::get(emitUnaryMaybeConstrainedFPBuiltin(
2715 *this, E, Intrinsic::asin, Intrinsic::experimental_constrained_asin));
2716
2717 case Builtin::BIatan:
2718 case Builtin::BIatanf:
2719 case Builtin::BIatanl:
2720 case Builtin::BI__builtin_atan:
2721 case Builtin::BI__builtin_atanf:
2722 case Builtin::BI__builtin_atanf16:
2723 case Builtin::BI__builtin_atanl:
2724 case Builtin::BI__builtin_atanf128:
2725 return RValue::get(emitUnaryMaybeConstrainedFPBuiltin(
2726 *this, E, Intrinsic::atan, Intrinsic::experimental_constrained_atan));
2727
2728 case Builtin::BIatan2:
2729 case Builtin::BIatan2f:
2730 case Builtin::BIatan2l:
2731 case Builtin::BI__builtin_atan2:
2732 case Builtin::BI__builtin_atan2f:
2733 case Builtin::BI__builtin_atan2f16:
2734 case Builtin::BI__builtin_atan2l:
2735 case Builtin::BI__builtin_atan2f128:
2736 return RValue::get(emitBinaryMaybeConstrainedFPBuiltin(
2737 *this, E, Intrinsic::atan2,
2738 Intrinsic::experimental_constrained_atan2));
2739
2740 case Builtin::BIceil:
2741 case Builtin::BIceilf:
2742 case Builtin::BIceill:
2743 case Builtin::BI__builtin_ceil:
2744 case Builtin::BI__builtin_ceilf:
2745 case Builtin::BI__builtin_ceilf16:
2746 case Builtin::BI__builtin_ceill:
2747 case Builtin::BI__builtin_ceilf128:
2748 return RValue::get(emitUnaryMaybeConstrainedFPBuiltin(*this, E,
2749 Intrinsic::ceil,
2750 Intrinsic::experimental_constrained_ceil));
2751
2752 case Builtin::BIcopysign:
2753 case Builtin::BIcopysignf:
2754 case Builtin::BIcopysignl:
2755 case Builtin::BI__builtin_copysign:
2756 case Builtin::BI__builtin_copysignf:
2757 case Builtin::BI__builtin_copysignf16:
2758 case Builtin::BI__builtin_copysignl:
2759 case Builtin::BI__builtin_copysignf128:
2760 return RValue::get(
2761 emitBuiltinWithOneOverloadedType<2>(*this, E, Intrinsic::copysign));
2762
2763 case Builtin::BIcos:
2764 case Builtin::BIcosf:
2765 case Builtin::BIcosl:
2766 case Builtin::BI__builtin_cos:
2767 case Builtin::BI__builtin_cosf:
2768 case Builtin::BI__builtin_cosf16:
2769 case Builtin::BI__builtin_cosl:
2770 case Builtin::BI__builtin_cosf128:
2771 return RValue::get(emitUnaryMaybeConstrainedFPBuiltin(*this, E,
2772 Intrinsic::cos,
2773 Intrinsic::experimental_constrained_cos));
2774
2775 case Builtin::BIcosh:
2776 case Builtin::BIcoshf:
2777 case Builtin::BIcoshl:
2778 case Builtin::BI__builtin_cosh:
2779 case Builtin::BI__builtin_coshf:
2780 case Builtin::BI__builtin_coshf16:
2781 case Builtin::BI__builtin_coshl:
2782 case Builtin::BI__builtin_coshf128:
2783 return RValue::get(emitUnaryMaybeConstrainedFPBuiltin(
2784 *this, E, Intrinsic::cosh, Intrinsic::experimental_constrained_cosh));
2785
2786 case Builtin::BIexp:
2787 case Builtin::BIexpf:
2788 case Builtin::BIexpl:
2789 case Builtin::BI__builtin_exp:
2790 case Builtin::BI__builtin_expf:
2791 case Builtin::BI__builtin_expf16:
2792 case Builtin::BI__builtin_expl:
2793 case Builtin::BI__builtin_expf128:
2794 return RValue::get(emitUnaryMaybeConstrainedFPBuiltin(*this, E,
2795 Intrinsic::exp,
2796 Intrinsic::experimental_constrained_exp));
2797
2798 case Builtin::BIexp2:
2799 case Builtin::BIexp2f:
2800 case Builtin::BIexp2l:
2801 case Builtin::BI__builtin_exp2:
2802 case Builtin::BI__builtin_exp2f:
2803 case Builtin::BI__builtin_exp2f16:
2804 case Builtin::BI__builtin_exp2l:
2805 case Builtin::BI__builtin_exp2f128:
2806 return RValue::get(emitUnaryMaybeConstrainedFPBuiltin(*this, E,
2807 Intrinsic::exp2,
2808 Intrinsic::experimental_constrained_exp2));
2809 case Builtin::BI__builtin_exp10:
2810 case Builtin::BI__builtin_exp10f:
2811 case Builtin::BI__builtin_exp10f16:
2812 case Builtin::BI__builtin_exp10l:
2813 case Builtin::BI__builtin_exp10f128: {
2814 // TODO: strictfp support
2815 if (Builder.getIsFPConstrained())
2816 break;
2817 return RValue::get(
2818 emitBuiltinWithOneOverloadedType<1>(*this, E, Intrinsic::exp10));
2819 }
2820 case Builtin::BIfabs:
2821 case Builtin::BIfabsf:
2822 case Builtin::BIfabsl:
2823 case Builtin::BI__builtin_fabs:
2824 case Builtin::BI__builtin_fabsf:
2825 case Builtin::BI__builtin_fabsf16:
2826 case Builtin::BI__builtin_fabsl:
2827 case Builtin::BI__builtin_fabsf128:
2828 return RValue::get(
2829 emitBuiltinWithOneOverloadedType<1>(*this, E, Intrinsic::fabs));
2830
2831 case Builtin::BIfloor:
2832 case Builtin::BIfloorf:
2833 case Builtin::BIfloorl:
2834 case Builtin::BI__builtin_floor:
2835 case Builtin::BI__builtin_floorf:
2836 case Builtin::BI__builtin_floorf16:
2837 case Builtin::BI__builtin_floorl:
2838 case Builtin::BI__builtin_floorf128:
2839 return RValue::get(emitUnaryMaybeConstrainedFPBuiltin(*this, E,
2840 Intrinsic::floor,
2841 Intrinsic::experimental_constrained_floor));
2842
2843 case Builtin::BIfma:
2844 case Builtin::BIfmaf:
2845 case Builtin::BIfmal:
2846 case Builtin::BI__builtin_fma:
2847 case Builtin::BI__builtin_fmaf:
2848 case Builtin::BI__builtin_fmaf16:
2849 case Builtin::BI__builtin_fmal:
2850 case Builtin::BI__builtin_fmaf128:
2851 return RValue::get(emitTernaryMaybeConstrainedFPBuiltin(*this, E,
2852 Intrinsic::fma,
2853 Intrinsic::experimental_constrained_fma));
2854
2855 case Builtin::BIfmax:
2856 case Builtin::BIfmaxf:
2857 case Builtin::BIfmaxl:
2858 case Builtin::BI__builtin_fmax:
2859 case Builtin::BI__builtin_fmaxf:
2860 case Builtin::BI__builtin_fmaxf16:
2861 case Builtin::BI__builtin_fmaxl:
2862 case Builtin::BI__builtin_fmaxf128:
2863 return RValue::get(emitBinaryMaybeConstrainedFPBuiltin(*this, E,
2864 Intrinsic::maxnum,
2865 Intrinsic::experimental_constrained_maxnum));
2866
2867 case Builtin::BIfmin:
2868 case Builtin::BIfminf:
2869 case Builtin::BIfminl:
2870 case Builtin::BI__builtin_fmin:
2871 case Builtin::BI__builtin_fminf:
2872 case Builtin::BI__builtin_fminf16:
2873 case Builtin::BI__builtin_fminl:
2874 case Builtin::BI__builtin_fminf128:
2875 return RValue::get(emitBinaryMaybeConstrainedFPBuiltin(*this, E,
2876 Intrinsic::minnum,
2877 Intrinsic::experimental_constrained_minnum));
2878
2879 case Builtin::BIfmaximum_num:
2880 case Builtin::BIfmaximum_numf:
2881 case Builtin::BIfmaximum_numl:
2882 case Builtin::BI__builtin_fmaximum_num:
2883 case Builtin::BI__builtin_fmaximum_numf:
2884 case Builtin::BI__builtin_fmaximum_numf16:
2885 case Builtin::BI__builtin_fmaximum_numl:
2886 case Builtin::BI__builtin_fmaximum_numf128:
2887 return RValue::get(
2888 emitBuiltinWithOneOverloadedType<2>(*this, E, Intrinsic::maximumnum));
2889
2890 case Builtin::BIfminimum_num:
2891 case Builtin::BIfminimum_numf:
2892 case Builtin::BIfminimum_numl:
2893 case Builtin::BI__builtin_fminimum_num:
2894 case Builtin::BI__builtin_fminimum_numf:
2895 case Builtin::BI__builtin_fminimum_numf16:
2896 case Builtin::BI__builtin_fminimum_numl:
2897 case Builtin::BI__builtin_fminimum_numf128:
2898 return RValue::get(
2899 emitBuiltinWithOneOverloadedType<2>(*this, E, Intrinsic::minimumnum));
2900
2901 // fmod() is a special-case. It maps to the frem instruction rather than an
2902 // LLVM intrinsic.
2903 case Builtin::BIfmod:
2904 case Builtin::BIfmodf:
2905 case Builtin::BIfmodl:
2906 case Builtin::BI__builtin_fmod:
2907 case Builtin::BI__builtin_fmodf:
2908 case Builtin::BI__builtin_fmodf16:
2909 case Builtin::BI__builtin_fmodl:
2910 case Builtin::BI__builtin_fmodf128:
2911 case Builtin::BI__builtin_elementwise_fmod: {
2912 CodeGenFunction::CGFPOptionsRAII FPOptsRAII(*this, E);
2913 Value *Arg1 = EmitScalarExpr(E: E->getArg(Arg: 0));
2914 Value *Arg2 = EmitScalarExpr(E: E->getArg(Arg: 1));
2915 return RValue::get(V: Builder.CreateFRem(L: Arg1, R: Arg2, Name: "fmod"));
2916 }
2917
2918 case Builtin::BIlog:
2919 case Builtin::BIlogf:
2920 case Builtin::BIlogl:
2921 case Builtin::BI__builtin_log:
2922 case Builtin::BI__builtin_logf:
2923 case Builtin::BI__builtin_logf16:
2924 case Builtin::BI__builtin_logl:
2925 case Builtin::BI__builtin_logf128:
2926 return RValue::get(emitUnaryMaybeConstrainedFPBuiltin(*this, E,
2927 Intrinsic::log,
2928 Intrinsic::experimental_constrained_log));
2929
2930 case Builtin::BIlog10:
2931 case Builtin::BIlog10f:
2932 case Builtin::BIlog10l:
2933 case Builtin::BI__builtin_log10:
2934 case Builtin::BI__builtin_log10f:
2935 case Builtin::BI__builtin_log10f16:
2936 case Builtin::BI__builtin_log10l:
2937 case Builtin::BI__builtin_log10f128:
2938 return RValue::get(emitUnaryMaybeConstrainedFPBuiltin(*this, E,
2939 Intrinsic::log10,
2940 Intrinsic::experimental_constrained_log10));
2941
2942 case Builtin::BIlog2:
2943 case Builtin::BIlog2f:
2944 case Builtin::BIlog2l:
2945 case Builtin::BI__builtin_log2:
2946 case Builtin::BI__builtin_log2f:
2947 case Builtin::BI__builtin_log2f16:
2948 case Builtin::BI__builtin_log2l:
2949 case Builtin::BI__builtin_log2f128:
2950 return RValue::get(emitUnaryMaybeConstrainedFPBuiltin(*this, E,
2951 Intrinsic::log2,
2952 Intrinsic::experimental_constrained_log2));
2953
2954 case Builtin::BInearbyint:
2955 case Builtin::BInearbyintf:
2956 case Builtin::BInearbyintl:
2957 case Builtin::BI__builtin_nearbyint:
2958 case Builtin::BI__builtin_nearbyintf:
2959 case Builtin::BI__builtin_nearbyintl:
2960 case Builtin::BI__builtin_nearbyintf128:
2961 return RValue::get(emitUnaryMaybeConstrainedFPBuiltin(*this, E,
2962 Intrinsic::nearbyint,
2963 Intrinsic::experimental_constrained_nearbyint));
2964
2965 case Builtin::BIpow:
2966 case Builtin::BIpowf:
2967 case Builtin::BIpowl:
2968 case Builtin::BI__builtin_pow:
2969 case Builtin::BI__builtin_powf:
2970 case Builtin::BI__builtin_powf16:
2971 case Builtin::BI__builtin_powl:
2972 case Builtin::BI__builtin_powf128:
2973 return RValue::get(emitBinaryMaybeConstrainedFPBuiltin(*this, E,
2974 Intrinsic::pow,
2975 Intrinsic::experimental_constrained_pow));
2976
2977 case Builtin::BIrint:
2978 case Builtin::BIrintf:
2979 case Builtin::BIrintl:
2980 case Builtin::BI__builtin_rint:
2981 case Builtin::BI__builtin_rintf:
2982 case Builtin::BI__builtin_rintf16:
2983 case Builtin::BI__builtin_rintl:
2984 case Builtin::BI__builtin_rintf128:
2985 return RValue::get(emitUnaryMaybeConstrainedFPBuiltin(*this, E,
2986 Intrinsic::rint,
2987 Intrinsic::experimental_constrained_rint));
2988
2989 case Builtin::BIround:
2990 case Builtin::BIroundf:
2991 case Builtin::BIroundl:
2992 case Builtin::BI__builtin_round:
2993 case Builtin::BI__builtin_roundf:
2994 case Builtin::BI__builtin_roundf16:
2995 case Builtin::BI__builtin_roundl:
2996 case Builtin::BI__builtin_roundf128:
2997 return RValue::get(emitUnaryMaybeConstrainedFPBuiltin(*this, E,
2998 Intrinsic::round,
2999 Intrinsic::experimental_constrained_round));
3000
3001 case Builtin::BIroundeven:
3002 case Builtin::BIroundevenf:
3003 case Builtin::BIroundevenl:
3004 case Builtin::BI__builtin_roundeven:
3005 case Builtin::BI__builtin_roundevenf:
3006 case Builtin::BI__builtin_roundevenf16:
3007 case Builtin::BI__builtin_roundevenl:
3008 case Builtin::BI__builtin_roundevenf128:
3009 return RValue::get(emitUnaryMaybeConstrainedFPBuiltin(*this, E,
3010 Intrinsic::roundeven,
3011 Intrinsic::experimental_constrained_roundeven));
3012
3013 case Builtin::BIsin:
3014 case Builtin::BIsinf:
3015 case Builtin::BIsinl:
3016 case Builtin::BI__builtin_sin:
3017 case Builtin::BI__builtin_sinf:
3018 case Builtin::BI__builtin_sinf16:
3019 case Builtin::BI__builtin_sinl:
3020 case Builtin::BI__builtin_sinf128:
3021 return RValue::get(emitUnaryMaybeConstrainedFPBuiltin(*this, E,
3022 Intrinsic::sin,
3023 Intrinsic::experimental_constrained_sin));
3024
3025 case Builtin::BIsinh:
3026 case Builtin::BIsinhf:
3027 case Builtin::BIsinhl:
3028 case Builtin::BI__builtin_sinh:
3029 case Builtin::BI__builtin_sinhf:
3030 case Builtin::BI__builtin_sinhf16:
3031 case Builtin::BI__builtin_sinhl:
3032 case Builtin::BI__builtin_sinhf128:
3033 return RValue::get(emitUnaryMaybeConstrainedFPBuiltin(
3034 *this, E, Intrinsic::sinh, Intrinsic::experimental_constrained_sinh));
3035
3036 case Builtin::BI__builtin_sincospi:
3037 case Builtin::BI__builtin_sincospif:
3038 case Builtin::BI__builtin_sincospil:
3039 if (Builder.getIsFPConstrained())
3040 break; // TODO: Emit constrained sincospi intrinsic once one exists.
3041 emitSincosBuiltin(*this, E, Intrinsic::sincospi);
3042 return RValue::get(V: nullptr);
3043
3044 case Builtin::BIsincos:
3045 case Builtin::BIsincosf:
3046 case Builtin::BIsincosl:
3047 case Builtin::BI__builtin_sincos:
3048 case Builtin::BI__builtin_sincosf:
3049 case Builtin::BI__builtin_sincosf16:
3050 case Builtin::BI__builtin_sincosl:
3051 case Builtin::BI__builtin_sincosf128:
3052 if (Builder.getIsFPConstrained())
3053 break; // TODO: Emit constrained sincos intrinsic once one exists.
3054 emitSincosBuiltin(*this, E, Intrinsic::sincos);
3055 return RValue::get(V: nullptr);
3056
3057 case Builtin::BIsqrt:
3058 case Builtin::BIsqrtf:
3059 case Builtin::BIsqrtl:
3060 case Builtin::BI__builtin_sqrt:
3061 case Builtin::BI__builtin_sqrtf:
3062 case Builtin::BI__builtin_sqrtf16:
3063 case Builtin::BI__builtin_sqrtl:
3064 case Builtin::BI__builtin_sqrtf128:
3065 case Builtin::BI__builtin_elementwise_sqrt: {
3066 llvm::Value *Call = emitUnaryMaybeConstrainedFPBuiltin(
3067 *this, E, Intrinsic::sqrt, Intrinsic::experimental_constrained_sqrt);
3068 SetSqrtFPAccuracy(Call);
3069 return RValue::get(V: Call);
3070 }
3071
3072 case Builtin::BItan:
3073 case Builtin::BItanf:
3074 case Builtin::BItanl:
3075 case Builtin::BI__builtin_tan:
3076 case Builtin::BI__builtin_tanf:
3077 case Builtin::BI__builtin_tanf16:
3078 case Builtin::BI__builtin_tanl:
3079 case Builtin::BI__builtin_tanf128:
3080 return RValue::get(emitUnaryMaybeConstrainedFPBuiltin(
3081 *this, E, Intrinsic::tan, Intrinsic::experimental_constrained_tan));
3082
3083 case Builtin::BItanh:
3084 case Builtin::BItanhf:
3085 case Builtin::BItanhl:
3086 case Builtin::BI__builtin_tanh:
3087 case Builtin::BI__builtin_tanhf:
3088 case Builtin::BI__builtin_tanhf16:
3089 case Builtin::BI__builtin_tanhl:
3090 case Builtin::BI__builtin_tanhf128:
3091 return RValue::get(emitUnaryMaybeConstrainedFPBuiltin(
3092 *this, E, Intrinsic::tanh, Intrinsic::experimental_constrained_tanh));
3093
3094 case Builtin::BItrunc:
3095 case Builtin::BItruncf:
3096 case Builtin::BItruncl:
3097 case Builtin::BI__builtin_trunc:
3098 case Builtin::BI__builtin_truncf:
3099 case Builtin::BI__builtin_truncf16:
3100 case Builtin::BI__builtin_truncl:
3101 case Builtin::BI__builtin_truncf128:
3102 return RValue::get(emitUnaryMaybeConstrainedFPBuiltin(*this, E,
3103 Intrinsic::trunc,
3104 Intrinsic::experimental_constrained_trunc));
3105
3106 case Builtin::BIlround:
3107 case Builtin::BIlroundf:
3108 case Builtin::BIlroundl:
3109 case Builtin::BI__builtin_lround:
3110 case Builtin::BI__builtin_lroundf:
3111 case Builtin::BI__builtin_lroundl:
3112 case Builtin::BI__builtin_lroundf128:
3113 return RValue::get(emitMaybeConstrainedFPToIntRoundBuiltin(
3114 *this, E, Intrinsic::lround,
3115 Intrinsic::experimental_constrained_lround));
3116
3117 case Builtin::BIllround:
3118 case Builtin::BIllroundf:
3119 case Builtin::BIllroundl:
3120 case Builtin::BI__builtin_llround:
3121 case Builtin::BI__builtin_llroundf:
3122 case Builtin::BI__builtin_llroundl:
3123 case Builtin::BI__builtin_llroundf128:
3124 return RValue::get(emitMaybeConstrainedFPToIntRoundBuiltin(
3125 *this, E, Intrinsic::llround,
3126 Intrinsic::experimental_constrained_llround));
3127
3128 case Builtin::BIlrint:
3129 case Builtin::BIlrintf:
3130 case Builtin::BIlrintl:
3131 case Builtin::BI__builtin_lrint:
3132 case Builtin::BI__builtin_lrintf:
3133 case Builtin::BI__builtin_lrintl:
3134 case Builtin::BI__builtin_lrintf128:
3135 return RValue::get(emitMaybeConstrainedFPToIntRoundBuiltin(
3136 *this, E, Intrinsic::lrint,
3137 Intrinsic::experimental_constrained_lrint));
3138
3139 case Builtin::BIllrint:
3140 case Builtin::BIllrintf:
3141 case Builtin::BIllrintl:
3142 case Builtin::BI__builtin_llrint:
3143 case Builtin::BI__builtin_llrintf:
3144 case Builtin::BI__builtin_llrintl:
3145 case Builtin::BI__builtin_llrintf128:
3146 return RValue::get(emitMaybeConstrainedFPToIntRoundBuiltin(
3147 *this, E, Intrinsic::llrint,
3148 Intrinsic::experimental_constrained_llrint));
3149 case Builtin::BI__builtin_ldexp:
3150 case Builtin::BI__builtin_ldexpf:
3151 case Builtin::BI__builtin_ldexpl:
3152 case Builtin::BI__builtin_ldexpf16:
3153 case Builtin::BI__builtin_ldexpf128: {
3154 return RValue::get(emitBinaryExpMaybeConstrainedFPBuiltin(
3155 *this, E, Intrinsic::ldexp,
3156 Intrinsic::experimental_constrained_ldexp));
3157 }
3158 default:
3159 break;
3160 }
3161 }
3162
3163 // Check NonnullAttribute/NullabilityArg and Alignment.
3164 auto EmitArgCheck = [&](TypeCheckKind Kind, Address A, const Expr *Arg,
3165 unsigned ParmNum) {
3166 Value *Val = A.emitRawPointer(CGF&: *this);
3167 EmitNonNullArgCheck(RV: RValue::get(V: Val), ArgType: Arg->getType(), ArgLoc: Arg->getExprLoc(), AC: FD,
3168 ParmNum);
3169
3170 if (SanOpts.has(K: SanitizerKind::Alignment)) {
3171 SanitizerSet SkippedChecks;
3172 SkippedChecks.set(SanitizerKind::All);
3173 SkippedChecks.clear(K: SanitizerKind::Alignment);
3174 SourceLocation Loc = Arg->getExprLoc();
3175 // Strip an implicit cast.
3176 if (auto *CE = dyn_cast<ImplicitCastExpr>(Val: Arg))
3177 if (CE->getCastKind() == CK_BitCast)
3178 Arg = CE->getSubExpr();
3179 EmitTypeCheck(TCK: Kind, Loc, V: Val, Type: Arg->getType(), Alignment: A.getAlignment(),
3180 SkippedChecks);
3181 }
3182 };
3183
3184 switch (BuiltinIDIfNoAsmLabel) {
3185 default: break;
3186 case Builtin::BI__builtin___CFStringMakeConstantString:
3187 case Builtin::BI__builtin___NSStringMakeConstantString:
3188 return RValue::get(ConstantEmitter(*this).emitAbstract(E, E->getType()));
3189 case Builtin::BI__builtin_stdarg_start:
3190 case Builtin::BI__builtin_va_start:
3191 case Builtin::BI__va_start:
3192 case Builtin::BI__builtin_c23_va_start:
3193 case Builtin::BI__builtin_va_end:
3194 EmitVAStartEnd(BuiltinID == Builtin::BI__va_start
3195 ? EmitScalarExpr(E->getArg(0))
3196 : EmitVAListRef(E->getArg(0)).emitRawPointer(*this),
3197 BuiltinID != Builtin::BI__builtin_va_end);
3198 return RValue::get(V: nullptr);
3199 case Builtin::BI__builtin_va_copy: {
3200 Value *DstPtr = EmitVAListRef(E: E->getArg(Arg: 0)).emitRawPointer(CGF&: *this);
3201 Value *SrcPtr = EmitVAListRef(E: E->getArg(Arg: 1)).emitRawPointer(CGF&: *this);
3202 Builder.CreateCall(CGM.getIntrinsic(Intrinsic::vacopy, {DstPtr->getType()}),
3203 {DstPtr, SrcPtr});
3204 return RValue::get(V: nullptr);
3205 }
3206 case Builtin::BIabs:
3207 case Builtin::BIlabs:
3208 case Builtin::BIllabs:
3209 case Builtin::BI__builtin_abs:
3210 case Builtin::BI__builtin_labs:
3211 case Builtin::BI__builtin_llabs: {
3212 bool SanitizeOverflow = SanOpts.has(K: SanitizerKind::SignedIntegerOverflow);
3213
3214 Value *Result;
3215 switch (getLangOpts().getSignedOverflowBehavior()) {
3216 case LangOptions::SOB_Defined:
3217 Result = EmitAbs(CGF&: *this, ArgValue: EmitScalarExpr(E: E->getArg(Arg: 0)), HasNSW: false);
3218 break;
3219 case LangOptions::SOB_Undefined:
3220 if (!SanitizeOverflow) {
3221 Result = EmitAbs(CGF&: *this, ArgValue: EmitScalarExpr(E: E->getArg(Arg: 0)), HasNSW: true);
3222 break;
3223 }
3224 [[fallthrough]];
3225 case LangOptions::SOB_Trapping:
3226 // TODO: Somehow handle the corner case when the address of abs is taken.
3227 Result = EmitOverflowCheckedAbs(CGF&: *this, E, SanitizeOverflow);
3228 break;
3229 }
3230 return RValue::get(V: Result);
3231 }
3232 case Builtin::BI__builtin_complex: {
3233 Value *Real = EmitScalarExpr(E: E->getArg(Arg: 0));
3234 Value *Imag = EmitScalarExpr(E: E->getArg(Arg: 1));
3235 return RValue::getComplex(C: {Real, Imag});
3236 }
3237 case Builtin::BI__builtin_conj:
3238 case Builtin::BI__builtin_conjf:
3239 case Builtin::BI__builtin_conjl:
3240 case Builtin::BIconj:
3241 case Builtin::BIconjf:
3242 case Builtin::BIconjl: {
3243 ComplexPairTy ComplexVal = EmitComplexExpr(E: E->getArg(Arg: 0));
3244 Value *Real = ComplexVal.first;
3245 Value *Imag = ComplexVal.second;
3246 Imag = Builder.CreateFNeg(V: Imag, Name: "neg");
3247 return RValue::getComplex(C: std::make_pair(x&: Real, y&: Imag));
3248 }
3249 case Builtin::BI__builtin_creal:
3250 case Builtin::BI__builtin_crealf:
3251 case Builtin::BI__builtin_creall:
3252 case Builtin::BIcreal:
3253 case Builtin::BIcrealf:
3254 case Builtin::BIcreall: {
3255 ComplexPairTy ComplexVal = EmitComplexExpr(E: E->getArg(Arg: 0));
3256 return RValue::get(V: ComplexVal.first);
3257 }
3258
3259 case Builtin::BI__builtin_preserve_access_index: {
3260 // Only enabled preserved access index region when debuginfo
3261 // is available as debuginfo is needed to preserve user-level
3262 // access pattern.
3263 if (!getDebugInfo()) {
3264 CGM.Error(loc: E->getExprLoc(), error: "using builtin_preserve_access_index() without -g");
3265 return RValue::get(V: EmitScalarExpr(E: E->getArg(Arg: 0)));
3266 }
3267
3268 // Nested builtin_preserve_access_index() not supported
3269 if (IsInPreservedAIRegion) {
3270 CGM.Error(loc: E->getExprLoc(), error: "nested builtin_preserve_access_index() not supported");
3271 return RValue::get(V: EmitScalarExpr(E: E->getArg(Arg: 0)));
3272 }
3273
3274 IsInPreservedAIRegion = true;
3275 Value *Res = EmitScalarExpr(E: E->getArg(Arg: 0));
3276 IsInPreservedAIRegion = false;
3277 return RValue::get(V: Res);
3278 }
3279
3280 case Builtin::BI__builtin_cimag:
3281 case Builtin::BI__builtin_cimagf:
3282 case Builtin::BI__builtin_cimagl:
3283 case Builtin::BIcimag:
3284 case Builtin::BIcimagf:
3285 case Builtin::BIcimagl: {
3286 ComplexPairTy ComplexVal = EmitComplexExpr(E: E->getArg(Arg: 0));
3287 return RValue::get(V: ComplexVal.second);
3288 }
3289
3290 case Builtin::BI__builtin_clrsb:
3291 case Builtin::BI__builtin_clrsbl:
3292 case Builtin::BI__builtin_clrsbll: {
3293 // clrsb(x) -> clz(x < 0 ? ~x : x) - 1 or
3294 Value *ArgValue = EmitScalarExpr(E: E->getArg(Arg: 0));
3295
3296 llvm::Type *ArgType = ArgValue->getType();
3297 Function *F = CGM.getIntrinsic(Intrinsic::ctlz, ArgType);
3298
3299 llvm::Type *ResultType = ConvertType(E->getType());
3300 Value *Zero = llvm::Constant::getNullValue(Ty: ArgType);
3301 Value *IsNeg = Builder.CreateICmpSLT(LHS: ArgValue, RHS: Zero, Name: "isneg");
3302 Value *Inverse = Builder.CreateNot(V: ArgValue, Name: "not");
3303 Value *Tmp = Builder.CreateSelect(C: IsNeg, True: Inverse, False: ArgValue);
3304 Value *Ctlz = Builder.CreateCall(Callee: F, Args: {Tmp, Builder.getFalse()});
3305 Value *Result = Builder.CreateSub(LHS: Ctlz, RHS: llvm::ConstantInt::get(Ty: ArgType, V: 1));
3306 Result = Builder.CreateIntCast(V: Result, DestTy: ResultType, /*isSigned*/true,
3307 Name: "cast");
3308 return RValue::get(V: Result);
3309 }
3310 case Builtin::BI__builtin_ctzs:
3311 case Builtin::BI__builtin_ctz:
3312 case Builtin::BI__builtin_ctzl:
3313 case Builtin::BI__builtin_ctzll:
3314 case Builtin::BI__builtin_ctzg: {
3315 bool HasFallback = BuiltinIDIfNoAsmLabel == Builtin::BI__builtin_ctzg &&
3316 E->getNumArgs() > 1;
3317
3318 Value *ArgValue =
3319 HasFallback ? EmitScalarExpr(E: E->getArg(Arg: 0))
3320 : EmitCheckedArgForBuiltin(E: E->getArg(Arg: 0), Kind: BCK_CTZPassedZero);
3321
3322 llvm::Type *ArgType = ArgValue->getType();
3323 Function *F = CGM.getIntrinsic(Intrinsic::cttz, ArgType);
3324
3325 llvm::Type *ResultType = ConvertType(E->getType());
3326 Value *ZeroUndef =
3327 Builder.getInt1(V: HasFallback || getTarget().isCLZForZeroUndef());
3328 Value *Result = Builder.CreateCall(Callee: F, Args: {ArgValue, ZeroUndef});
3329 if (Result->getType() != ResultType)
3330 Result =
3331 Builder.CreateIntCast(V: Result, DestTy: ResultType, /*isSigned*/ false, Name: "cast");
3332 if (!HasFallback)
3333 return RValue::get(V: Result);
3334
3335 Value *Zero = Constant::getNullValue(Ty: ArgType);
3336 Value *IsZero = Builder.CreateICmpEQ(LHS: ArgValue, RHS: Zero, Name: "iszero");
3337 Value *FallbackValue = EmitScalarExpr(E: E->getArg(Arg: 1));
3338 Value *ResultOrFallback =
3339 Builder.CreateSelect(C: IsZero, True: FallbackValue, False: Result, Name: "ctzg");
3340 return RValue::get(V: ResultOrFallback);
3341 }
3342 case Builtin::BI__builtin_clzs:
3343 case Builtin::BI__builtin_clz:
3344 case Builtin::BI__builtin_clzl:
3345 case Builtin::BI__builtin_clzll:
3346 case Builtin::BI__builtin_clzg: {
3347 bool HasFallback = BuiltinIDIfNoAsmLabel == Builtin::BI__builtin_clzg &&
3348 E->getNumArgs() > 1;
3349
3350 Value *ArgValue =
3351 HasFallback ? EmitScalarExpr(E: E->getArg(Arg: 0))
3352 : EmitCheckedArgForBuiltin(E: E->getArg(Arg: 0), Kind: BCK_CLZPassedZero);
3353
3354 llvm::Type *ArgType = ArgValue->getType();
3355 Function *F = CGM.getIntrinsic(Intrinsic::ctlz, ArgType);
3356
3357 llvm::Type *ResultType = ConvertType(E->getType());
3358 Value *ZeroUndef =
3359 Builder.getInt1(V: HasFallback || getTarget().isCLZForZeroUndef());
3360 Value *Result = Builder.CreateCall(Callee: F, Args: {ArgValue, ZeroUndef});
3361 if (Result->getType() != ResultType)
3362 Result =
3363 Builder.CreateIntCast(V: Result, DestTy: ResultType, /*isSigned*/ false, Name: "cast");
3364 if (!HasFallback)
3365 return RValue::get(V: Result);
3366
3367 Value *Zero = Constant::getNullValue(Ty: ArgType);
3368 Value *IsZero = Builder.CreateICmpEQ(LHS: ArgValue, RHS: Zero, Name: "iszero");
3369 Value *FallbackValue = EmitScalarExpr(E: E->getArg(Arg: 1));
3370 Value *ResultOrFallback =
3371 Builder.CreateSelect(C: IsZero, True: FallbackValue, False: Result, Name: "clzg");
3372 return RValue::get(V: ResultOrFallback);
3373 }
3374 case Builtin::BI__builtin_ffs:
3375 case Builtin::BI__builtin_ffsl:
3376 case Builtin::BI__builtin_ffsll: {
3377 // ffs(x) -> x ? cttz(x) + 1 : 0
3378 Value *ArgValue = EmitScalarExpr(E: E->getArg(Arg: 0));
3379
3380 llvm::Type *ArgType = ArgValue->getType();
3381 Function *F = CGM.getIntrinsic(Intrinsic::cttz, ArgType);
3382
3383 llvm::Type *ResultType = ConvertType(E->getType());
3384 Value *Tmp =
3385 Builder.CreateAdd(LHS: Builder.CreateCall(Callee: F, Args: {ArgValue, Builder.getTrue()}),
3386 RHS: llvm::ConstantInt::get(Ty: ArgType, V: 1));
3387 Value *Zero = llvm::Constant::getNullValue(Ty: ArgType);
3388 Value *IsZero = Builder.CreateICmpEQ(LHS: ArgValue, RHS: Zero, Name: "iszero");
3389 Value *Result = Builder.CreateSelect(C: IsZero, True: Zero, False: Tmp, Name: "ffs");
3390 if (Result->getType() != ResultType)
3391 Result = Builder.CreateIntCast(V: Result, DestTy: ResultType, /*isSigned*/true,
3392 Name: "cast");
3393 return RValue::get(V: Result);
3394 }
3395 case Builtin::BI__builtin_parity:
3396 case Builtin::BI__builtin_parityl:
3397 case Builtin::BI__builtin_parityll: {
3398 // parity(x) -> ctpop(x) & 1
3399 Value *ArgValue = EmitScalarExpr(E: E->getArg(Arg: 0));
3400
3401 llvm::Type *ArgType = ArgValue->getType();
3402 Function *F = CGM.getIntrinsic(Intrinsic::ctpop, ArgType);
3403
3404 llvm::Type *ResultType = ConvertType(E->getType());
3405 Value *Tmp = Builder.CreateCall(Callee: F, Args: ArgValue);
3406 Value *Result = Builder.CreateAnd(LHS: Tmp, RHS: llvm::ConstantInt::get(Ty: ArgType, V: 1));
3407 if (Result->getType() != ResultType)
3408 Result = Builder.CreateIntCast(V: Result, DestTy: ResultType, /*isSigned*/true,
3409 Name: "cast");
3410 return RValue::get(V: Result);
3411 }
3412 case Builtin::BI__lzcnt16:
3413 case Builtin::BI__lzcnt:
3414 case Builtin::BI__lzcnt64: {
3415 Value *ArgValue = EmitScalarExpr(E: E->getArg(Arg: 0));
3416
3417 llvm::Type *ArgType = ArgValue->getType();
3418 Function *F = CGM.getIntrinsic(Intrinsic::ctlz, ArgType);
3419
3420 llvm::Type *ResultType = ConvertType(E->getType());
3421 Value *Result = Builder.CreateCall(Callee: F, Args: {ArgValue, Builder.getFalse()});
3422 if (Result->getType() != ResultType)
3423 Result = Builder.CreateIntCast(V: Result, DestTy: ResultType, /*isSigned*/true,
3424 Name: "cast");
3425 return RValue::get(V: Result);
3426 }
3427 case Builtin::BI__popcnt16:
3428 case Builtin::BI__popcnt:
3429 case Builtin::BI__popcnt64:
3430 case Builtin::BI__builtin_popcount:
3431 case Builtin::BI__builtin_popcountl:
3432 case Builtin::BI__builtin_popcountll:
3433 case Builtin::BI__builtin_popcountg: {
3434 Value *ArgValue = EmitScalarExpr(E: E->getArg(Arg: 0));
3435
3436 llvm::Type *ArgType = ArgValue->getType();
3437 Function *F = CGM.getIntrinsic(Intrinsic::ctpop, ArgType);
3438
3439 llvm::Type *ResultType = ConvertType(E->getType());
3440 Value *Result = Builder.CreateCall(Callee: F, Args: ArgValue);
3441 if (Result->getType() != ResultType)
3442 Result =
3443 Builder.CreateIntCast(V: Result, DestTy: ResultType, /*isSigned*/ false, Name: "cast");
3444 return RValue::get(V: Result);
3445 }
3446 case Builtin::BI__builtin_unpredictable: {
3447 // Always return the argument of __builtin_unpredictable. LLVM does not
3448 // handle this builtin. Metadata for this builtin should be added directly
3449 // to instructions such as branches or switches that use it.
3450 return RValue::get(V: EmitScalarExpr(E: E->getArg(Arg: 0)));
3451 }
3452 case Builtin::BI__builtin_expect: {
3453 Value *ArgValue = EmitScalarExpr(E: E->getArg(Arg: 0));
3454 llvm::Type *ArgType = ArgValue->getType();
3455
3456 Value *ExpectedValue = EmitScalarExpr(E: E->getArg(Arg: 1));
3457 // Don't generate llvm.expect on -O0 as the backend won't use it for
3458 // anything.
3459 // Note, we still IRGen ExpectedValue because it could have side-effects.
3460 if (CGM.getCodeGenOpts().OptimizationLevel == 0)
3461 return RValue::get(V: ArgValue);
3462
3463 Function *FnExpect = CGM.getIntrinsic(Intrinsic::expect, ArgType);
3464 Value *Result =
3465 Builder.CreateCall(Callee: FnExpect, Args: {ArgValue, ExpectedValue}, Name: "expval");
3466 return RValue::get(V: Result);
3467 }
3468 case Builtin::BI__builtin_expect_with_probability: {
3469 Value *ArgValue = EmitScalarExpr(E: E->getArg(Arg: 0));
3470 llvm::Type *ArgType = ArgValue->getType();
3471
3472 Value *ExpectedValue = EmitScalarExpr(E: E->getArg(Arg: 1));
3473 llvm::APFloat Probability(0.0);
3474 const Expr *ProbArg = E->getArg(Arg: 2);
3475 bool EvalSucceed = ProbArg->EvaluateAsFloat(Result&: Probability, Ctx: CGM.getContext());
3476 assert(EvalSucceed && "probability should be able to evaluate as float");
3477 (void)EvalSucceed;
3478 bool LoseInfo = false;
3479 Probability.convert(ToSemantics: llvm::APFloat::IEEEdouble(),
3480 RM: llvm::RoundingMode::Dynamic, losesInfo: &LoseInfo);
3481 llvm::Type *Ty = ConvertType(T: ProbArg->getType());
3482 Constant *Confidence = ConstantFP::get(Ty, V: Probability);
3483 // Don't generate llvm.expect.with.probability on -O0 as the backend
3484 // won't use it for anything.
3485 // Note, we still IRGen ExpectedValue because it could have side-effects.
3486 if (CGM.getCodeGenOpts().OptimizationLevel == 0)
3487 return RValue::get(V: ArgValue);
3488
3489 Function *FnExpect =
3490 CGM.getIntrinsic(Intrinsic::expect_with_probability, ArgType);
3491 Value *Result = Builder.CreateCall(
3492 Callee: FnExpect, Args: {ArgValue, ExpectedValue, Confidence}, Name: "expval");
3493 return RValue::get(V: Result);
3494 }
3495 case Builtin::BI__builtin_assume_aligned: {
3496 const Expr *Ptr = E->getArg(Arg: 0);
3497 Value *PtrValue = EmitScalarExpr(E: Ptr);
3498 Value *OffsetValue =
3499 (E->getNumArgs() > 2) ? EmitScalarExpr(E: E->getArg(Arg: 2)) : nullptr;
3500
3501 Value *AlignmentValue = EmitScalarExpr(E: E->getArg(Arg: 1));
3502 ConstantInt *AlignmentCI = cast<ConstantInt>(Val: AlignmentValue);
3503 if (AlignmentCI->getValue().ugt(RHS: llvm::Value::MaximumAlignment))
3504 AlignmentCI = ConstantInt::get(Ty: AlignmentCI->getIntegerType(),
3505 V: llvm::Value::MaximumAlignment);
3506
3507 emitAlignmentAssumption(PtrValue, E: Ptr,
3508 /*The expr loc is sufficient.*/ AssumptionLoc: SourceLocation(),
3509 Alignment: AlignmentCI, OffsetValue);
3510 return RValue::get(V: PtrValue);
3511 }
3512 case Builtin::BI__builtin_assume_dereferenceable: {
3513 const Expr *Ptr = E->getArg(Arg: 0);
3514 const Expr *Size = E->getArg(Arg: 1);
3515 Value *PtrValue = EmitScalarExpr(E: Ptr);
3516 Value *SizeValue = EmitScalarExpr(E: Size);
3517 if (SizeValue->getType() != IntPtrTy)
3518 SizeValue =
3519 Builder.CreateIntCast(V: SizeValue, DestTy: IntPtrTy, isSigned: false, Name: "casted.size");
3520 Builder.CreateDereferenceableAssumption(PtrValue, SizeValue);
3521 return RValue::get(V: nullptr);
3522 }
3523 case Builtin::BI__assume:
3524 case Builtin::BI__builtin_assume: {
3525 if (E->getArg(Arg: 0)->HasSideEffects(Ctx: getContext()))
3526 return RValue::get(V: nullptr);
3527
3528 Value *ArgValue = EmitCheckedArgForAssume(E: E->getArg(Arg: 0));
3529 Function *FnAssume = CGM.getIntrinsic(Intrinsic::assume);
3530 Builder.CreateCall(Callee: FnAssume, Args: ArgValue);
3531 return RValue::get(V: nullptr);
3532 }
3533 case Builtin::BI__builtin_assume_separate_storage: {
3534 const Expr *Arg0 = E->getArg(Arg: 0);
3535 const Expr *Arg1 = E->getArg(Arg: 1);
3536
3537 Value *Value0 = EmitScalarExpr(E: Arg0);
3538 Value *Value1 = EmitScalarExpr(E: Arg1);
3539
3540 Value *Values[] = {Value0, Value1};
3541 OperandBundleDefT<Value *> OBD("separate_storage", Values);
3542 Builder.CreateAssumption(Cond: ConstantInt::getTrue(Context&: getLLVMContext()), OpBundles: {OBD});
3543 return RValue::get(V: nullptr);
3544 }
3545 case Builtin::BI__builtin_allow_runtime_check: {
3546 StringRef Kind =
3547 cast<StringLiteral>(Val: E->getArg(Arg: 0)->IgnoreParenCasts())->getString();
3548 LLVMContext &Ctx = CGM.getLLVMContext();
3549 llvm::Value *Allow = Builder.CreateCall(
3550 CGM.getIntrinsic(Intrinsic::allow_runtime_check),
3551 llvm::MetadataAsValue::get(Ctx, llvm::MDString::get(Ctx, Kind)));
3552 return RValue::get(V: Allow);
3553 }
3554 case Builtin::BI__arithmetic_fence: {
3555 // Create the builtin call if FastMath is selected, and the target
3556 // supports the builtin, otherwise just return the argument.
3557 CodeGenFunction::CGFPOptionsRAII FPOptsRAII(*this, E);
3558 llvm::FastMathFlags FMF = Builder.getFastMathFlags();
3559 bool isArithmeticFenceEnabled =
3560 FMF.allowReassoc() &&
3561 getContext().getTargetInfo().checkArithmeticFenceSupported();
3562 QualType ArgType = E->getArg(Arg: 0)->getType();
3563 if (ArgType->isComplexType()) {
3564 if (isArithmeticFenceEnabled) {
3565 QualType ElementType = ArgType->castAs<ComplexType>()->getElementType();
3566 ComplexPairTy ComplexVal = EmitComplexExpr(E: E->getArg(Arg: 0));
3567 Value *Real = Builder.CreateArithmeticFence(Val: ComplexVal.first,
3568 DstType: ConvertType(T: ElementType));
3569 Value *Imag = Builder.CreateArithmeticFence(Val: ComplexVal.second,
3570 DstType: ConvertType(T: ElementType));
3571 return RValue::getComplex(C: std::make_pair(x&: Real, y&: Imag));
3572 }
3573 ComplexPairTy ComplexVal = EmitComplexExpr(E: E->getArg(Arg: 0));
3574 Value *Real = ComplexVal.first;
3575 Value *Imag = ComplexVal.second;
3576 return RValue::getComplex(C: std::make_pair(x&: Real, y&: Imag));
3577 }
3578 Value *ArgValue = EmitScalarExpr(E: E->getArg(Arg: 0));
3579 if (isArithmeticFenceEnabled)
3580 return RValue::get(
3581 V: Builder.CreateArithmeticFence(Val: ArgValue, DstType: ConvertType(T: ArgType)));
3582 return RValue::get(V: ArgValue);
3583 }
3584 case Builtin::BI__builtin_bswap16:
3585 case Builtin::BI__builtin_bswap32:
3586 case Builtin::BI__builtin_bswap64:
3587 case Builtin::BI_byteswap_ushort:
3588 case Builtin::BI_byteswap_ulong:
3589 case Builtin::BI_byteswap_uint64: {
3590 return RValue::get(
3591 emitBuiltinWithOneOverloadedType<1>(*this, E, Intrinsic::bswap));
3592 }
3593 case Builtin::BI__builtin_bitreverse8:
3594 case Builtin::BI__builtin_bitreverse16:
3595 case Builtin::BI__builtin_bitreverse32:
3596 case Builtin::BI__builtin_bitreverse64: {
3597 return RValue::get(
3598 emitBuiltinWithOneOverloadedType<1>(*this, E, Intrinsic::bitreverse));
3599 }
3600 case Builtin::BI__builtin_rotateleft8:
3601 case Builtin::BI__builtin_rotateleft16:
3602 case Builtin::BI__builtin_rotateleft32:
3603 case Builtin::BI__builtin_rotateleft64:
3604 case Builtin::BI_rotl8: // Microsoft variants of rotate left
3605 case Builtin::BI_rotl16:
3606 case Builtin::BI_rotl:
3607 case Builtin::BI_lrotl:
3608 case Builtin::BI_rotl64:
3609 return emitRotate(E, IsRotateRight: false);
3610
3611 case Builtin::BI__builtin_rotateright8:
3612 case Builtin::BI__builtin_rotateright16:
3613 case Builtin::BI__builtin_rotateright32:
3614 case Builtin::BI__builtin_rotateright64:
3615 case Builtin::BI_rotr8: // Microsoft variants of rotate right
3616 case Builtin::BI_rotr16:
3617 case Builtin::BI_rotr:
3618 case Builtin::BI_lrotr:
3619 case Builtin::BI_rotr64:
3620 return emitRotate(E, IsRotateRight: true);
3621
3622 case Builtin::BI__builtin_constant_p: {
3623 llvm::Type *ResultType = ConvertType(E->getType());
3624
3625 const Expr *Arg = E->getArg(Arg: 0);
3626 QualType ArgType = Arg->getType();
3627 // FIXME: The allowance for Obj-C pointers and block pointers is historical
3628 // and likely a mistake.
3629 if (!ArgType->isIntegralOrEnumerationType() && !ArgType->isFloatingType() &&
3630 !ArgType->isObjCObjectPointerType() && !ArgType->isBlockPointerType())
3631 // Per the GCC documentation, only numeric constants are recognized after
3632 // inlining.
3633 return RValue::get(V: ConstantInt::get(Ty: ResultType, V: 0));
3634
3635 if (Arg->HasSideEffects(Ctx: getContext()))
3636 // The argument is unevaluated, so be conservative if it might have
3637 // side-effects.
3638 return RValue::get(V: ConstantInt::get(Ty: ResultType, V: 0));
3639
3640 Value *ArgValue = EmitScalarExpr(E: Arg);
3641 if (ArgType->isObjCObjectPointerType()) {
3642 // Convert Objective-C objects to id because we cannot distinguish between
3643 // LLVM types for Obj-C classes as they are opaque.
3644 ArgType = CGM.getContext().getObjCIdType();
3645 ArgValue = Builder.CreateBitCast(V: ArgValue, DestTy: ConvertType(T: ArgType));
3646 }
3647 Function *F =
3648 CGM.getIntrinsic(Intrinsic::is_constant, ConvertType(ArgType));
3649 Value *Result = Builder.CreateCall(Callee: F, Args: ArgValue);
3650 if (Result->getType() != ResultType)
3651 Result = Builder.CreateIntCast(V: Result, DestTy: ResultType, /*isSigned*/false);
3652 return RValue::get(V: Result);
3653 }
3654 case Builtin::BI__builtin_dynamic_object_size:
3655 case Builtin::BI__builtin_object_size: {
3656 unsigned Type =
3657 E->getArg(Arg: 1)->EvaluateKnownConstInt(Ctx: getContext()).getZExtValue();
3658 auto *ResType = cast<llvm::IntegerType>(ConvertType(E->getType()));
3659
3660 // We pass this builtin onto the optimizer so that it can figure out the
3661 // object size in more complex cases.
3662 bool IsDynamic = BuiltinID == Builtin::BI__builtin_dynamic_object_size;
3663 return RValue::get(emitBuiltinObjectSize(E: E->getArg(Arg: 0), Type, ResType: ResType,
3664 /*EmittedE=*/nullptr, IsDynamic));
3665 }
3666 case Builtin::BI__builtin_counted_by_ref: {
3667 // Default to returning '(void *) 0'.
3668 llvm::Value *Result = llvm::ConstantPointerNull::get(
3669 T: llvm::PointerType::getUnqual(C&: getLLVMContext()));
3670
3671 const Expr *Arg = E->getArg(Arg: 0)->IgnoreParenImpCasts();
3672
3673 if (auto *UO = dyn_cast<UnaryOperator>(Val: Arg);
3674 UO && UO->getOpcode() == UO_AddrOf) {
3675 Arg = UO->getSubExpr()->IgnoreParenImpCasts();
3676
3677 if (auto *ASE = dyn_cast<ArraySubscriptExpr>(Val: Arg))
3678 Arg = ASE->getBase()->IgnoreParenImpCasts();
3679 }
3680
3681 if (const MemberExpr *ME = dyn_cast_if_present<MemberExpr>(Val: Arg)) {
3682 if (auto *CATy =
3683 ME->getMemberDecl()->getType()->getAs<CountAttributedType>();
3684 CATy && CATy->getKind() == CountAttributedType::CountedBy) {
3685 const auto *FAMDecl = cast<FieldDecl>(Val: ME->getMemberDecl());
3686 if (const FieldDecl *CountFD = FAMDecl->findCountedByField())
3687 Result = GetCountedByFieldExprGEP(Base: Arg, FD: FAMDecl, CountDecl: CountFD);
3688 else
3689 llvm::report_fatal_error(reason: "Cannot find the counted_by 'count' field");
3690 }
3691 }
3692
3693 return RValue::get(V: Result);
3694 }
3695 case Builtin::BI__builtin_prefetch: {
3696 Value *Locality, *RW, *Address = EmitScalarExpr(E: E->getArg(Arg: 0));
3697 // FIXME: Technically these constants should of type 'int', yes?
3698 RW = (E->getNumArgs() > 1) ? EmitScalarExpr(E: E->getArg(Arg: 1)) :
3699 llvm::ConstantInt::get(Ty: Int32Ty, V: 0);
3700 Locality = (E->getNumArgs() > 2) ? EmitScalarExpr(E: E->getArg(Arg: 2)) :
3701 llvm::ConstantInt::get(Ty: Int32Ty, V: 3);
3702 Value *Data = llvm::ConstantInt::get(Ty: Int32Ty, V: 1);
3703 Function *F = CGM.getIntrinsic(Intrinsic::prefetch, Address->getType());
3704 Builder.CreateCall(Callee: F, Args: {Address, RW, Locality, Data});
3705 return RValue::get(V: nullptr);
3706 }
3707 case Builtin::BI__builtin_readcyclecounter: {
3708 Function *F = CGM.getIntrinsic(Intrinsic::readcyclecounter);
3709 return RValue::get(V: Builder.CreateCall(Callee: F));
3710 }
3711 case Builtin::BI__builtin_readsteadycounter: {
3712 Function *F = CGM.getIntrinsic(Intrinsic::readsteadycounter);
3713 return RValue::get(V: Builder.CreateCall(Callee: F));
3714 }
3715 case Builtin::BI__builtin___clear_cache: {
3716 Value *Begin = EmitScalarExpr(E: E->getArg(Arg: 0));
3717 Value *End = EmitScalarExpr(E: E->getArg(Arg: 1));
3718 Function *F = CGM.getIntrinsic(Intrinsic::clear_cache);
3719 return RValue::get(V: Builder.CreateCall(Callee: F, Args: {Begin, End}));
3720 }
3721 case Builtin::BI__builtin_trap:
3722 EmitTrapCall(Intrinsic::trap);
3723 return RValue::get(V: nullptr);
3724 case Builtin::BI__builtin_verbose_trap: {
3725 llvm::DILocation *TrapLocation = Builder.getCurrentDebugLocation();
3726 if (getDebugInfo()) {
3727 TrapLocation = getDebugInfo()->CreateTrapFailureMessageFor(
3728 TrapLocation, Category: *E->getArg(Arg: 0)->tryEvaluateString(Ctx&: getContext()),
3729 FailureMsg: *E->getArg(Arg: 1)->tryEvaluateString(Ctx&: getContext()));
3730 }
3731 ApplyDebugLocation ApplyTrapDI(*this, TrapLocation);
3732 // Currently no attempt is made to prevent traps from being merged.
3733 EmitTrapCall(Intrinsic::trap);
3734 return RValue::get(V: nullptr);
3735 }
3736 case Builtin::BI__debugbreak:
3737 EmitTrapCall(Intrinsic::debugtrap);
3738 return RValue::get(V: nullptr);
3739 case Builtin::BI__builtin_unreachable: {
3740 EmitUnreachable(Loc: E->getExprLoc());
3741
3742 // We do need to preserve an insertion point.
3743 EmitBlock(BB: createBasicBlock(name: "unreachable.cont"));
3744
3745 return RValue::get(V: nullptr);
3746 }
3747
3748 case Builtin::BI__builtin_powi:
3749 case Builtin::BI__builtin_powif:
3750 case Builtin::BI__builtin_powil: {
3751 llvm::Value *Src0 = EmitScalarExpr(E: E->getArg(Arg: 0));
3752 llvm::Value *Src1 = EmitScalarExpr(E: E->getArg(Arg: 1));
3753
3754 if (Builder.getIsFPConstrained()) {
3755 // FIXME: llvm.powi has 2 mangling types,
3756 // llvm.experimental.constrained.powi has one.
3757 CodeGenFunction::CGFPOptionsRAII FPOptsRAII(*this, E);
3758 Function *F = CGM.getIntrinsic(Intrinsic::experimental_constrained_powi,
3759 Src0->getType());
3760 return RValue::get(V: Builder.CreateConstrainedFPCall(Callee: F, Args: { Src0, Src1 }));
3761 }
3762
3763 Function *F = CGM.getIntrinsic(Intrinsic::powi,
3764 { Src0->getType(), Src1->getType() });
3765 return RValue::get(V: Builder.CreateCall(Callee: F, Args: { Src0, Src1 }));
3766 }
3767 case Builtin::BI__builtin_frexpl: {
3768 // Linux PPC will not be adding additional PPCDoubleDouble support.
3769 // WIP to switch default to IEEE long double. Will emit libcall for
3770 // frexpl instead of legalizing this type in the BE.
3771 if (&getTarget().getLongDoubleFormat() == &llvm::APFloat::PPCDoubleDouble())
3772 break;
3773 [[fallthrough]];
3774 }
3775 case Builtin::BI__builtin_frexp:
3776 case Builtin::BI__builtin_frexpf:
3777 case Builtin::BI__builtin_frexpf128:
3778 case Builtin::BI__builtin_frexpf16:
3779 return RValue::get(emitFrexpBuiltin(*this, E, Intrinsic::frexp));
3780 case Builtin::BImodf:
3781 case Builtin::BImodff:
3782 case Builtin::BImodfl:
3783 case Builtin::BI__builtin_modf:
3784 case Builtin::BI__builtin_modff:
3785 case Builtin::BI__builtin_modfl:
3786 if (Builder.getIsFPConstrained())
3787 break; // TODO: Emit constrained modf intrinsic once one exists.
3788 return RValue::get(emitModfBuiltin(*this, E, Intrinsic::modf));
3789 case Builtin::BI__builtin_isgreater:
3790 case Builtin::BI__builtin_isgreaterequal:
3791 case Builtin::BI__builtin_isless:
3792 case Builtin::BI__builtin_islessequal:
3793 case Builtin::BI__builtin_islessgreater:
3794 case Builtin::BI__builtin_isunordered: {
3795 // Ordered comparisons: we know the arguments to these are matching scalar
3796 // floating point values.
3797 CodeGenFunction::CGFPOptionsRAII FPOptsRAII(*this, E);
3798 Value *LHS = EmitScalarExpr(E: E->getArg(Arg: 0));
3799 Value *RHS = EmitScalarExpr(E: E->getArg(Arg: 1));
3800
3801 switch (BuiltinID) {
3802 default: llvm_unreachable("Unknown ordered comparison");
3803 case Builtin::BI__builtin_isgreater:
3804 LHS = Builder.CreateFCmpOGT(LHS, RHS, Name: "cmp");
3805 break;
3806 case Builtin::BI__builtin_isgreaterequal:
3807 LHS = Builder.CreateFCmpOGE(LHS, RHS, Name: "cmp");
3808 break;
3809 case Builtin::BI__builtin_isless:
3810 LHS = Builder.CreateFCmpOLT(LHS, RHS, Name: "cmp");
3811 break;
3812 case Builtin::BI__builtin_islessequal:
3813 LHS = Builder.CreateFCmpOLE(LHS, RHS, Name: "cmp");
3814 break;
3815 case Builtin::BI__builtin_islessgreater:
3816 LHS = Builder.CreateFCmpONE(LHS, RHS, Name: "cmp");
3817 break;
3818 case Builtin::BI__builtin_isunordered:
3819 LHS = Builder.CreateFCmpUNO(LHS, RHS, Name: "cmp");
3820 break;
3821 }
3822 // ZExt bool to int type.
3823 return RValue::get(Builder.CreateZExt(V: LHS, DestTy: ConvertType(E->getType())));
3824 }
3825
3826 case Builtin::BI__builtin_isnan: {
3827 CodeGenFunction::CGFPOptionsRAII FPOptsRAII(*this, E);
3828 Value *V = EmitScalarExpr(E: E->getArg(Arg: 0));
3829 if (Value *Result = tryUseTestFPKind(CGF&: *this, BuiltinID, V))
3830 return RValue::get(V: Result);
3831 return RValue::get(
3832 Builder.CreateZExt(V: Builder.createIsFPClass(FPNum: V, Test: FPClassTest::fcNan),
3833 DestTy: ConvertType(E->getType())));
3834 }
3835
3836 case Builtin::BI__builtin_issignaling: {
3837 CodeGenFunction::CGFPOptionsRAII FPOptsRAII(*this, E);
3838 Value *V = EmitScalarExpr(E: E->getArg(Arg: 0));
3839 return RValue::get(
3840 Builder.CreateZExt(V: Builder.createIsFPClass(FPNum: V, Test: FPClassTest::fcSNan),
3841 DestTy: ConvertType(E->getType())));
3842 }
3843
3844 case Builtin::BI__builtin_isinf: {
3845 CodeGenFunction::CGFPOptionsRAII FPOptsRAII(*this, E);
3846 Value *V = EmitScalarExpr(E: E->getArg(Arg: 0));
3847 if (Value *Result = tryUseTestFPKind(CGF&: *this, BuiltinID, V))
3848 return RValue::get(V: Result);
3849 return RValue::get(
3850 Builder.CreateZExt(V: Builder.createIsFPClass(FPNum: V, Test: FPClassTest::fcInf),
3851 DestTy: ConvertType(E->getType())));
3852 }
3853
3854 case Builtin::BIfinite:
3855 case Builtin::BI__finite:
3856 case Builtin::BIfinitef:
3857 case Builtin::BI__finitef:
3858 case Builtin::BIfinitel:
3859 case Builtin::BI__finitel:
3860 case Builtin::BI__builtin_isfinite: {
3861 CodeGenFunction::CGFPOptionsRAII FPOptsRAII(*this, E);
3862 Value *V = EmitScalarExpr(E: E->getArg(Arg: 0));
3863 if (Value *Result = tryUseTestFPKind(CGF&: *this, BuiltinID, V))
3864 return RValue::get(V: Result);
3865 return RValue::get(
3866 Builder.CreateZExt(V: Builder.createIsFPClass(FPNum: V, Test: FPClassTest::fcFinite),
3867 DestTy: ConvertType(E->getType())));
3868 }
3869
3870 case Builtin::BI__builtin_isnormal: {
3871 CodeGenFunction::CGFPOptionsRAII FPOptsRAII(*this, E);
3872 Value *V = EmitScalarExpr(E: E->getArg(Arg: 0));
3873 return RValue::get(
3874 Builder.CreateZExt(V: Builder.createIsFPClass(FPNum: V, Test: FPClassTest::fcNormal),
3875 DestTy: ConvertType(E->getType())));
3876 }
3877
3878 case Builtin::BI__builtin_issubnormal: {
3879 CodeGenFunction::CGFPOptionsRAII FPOptsRAII(*this, E);
3880 Value *V = EmitScalarExpr(E: E->getArg(Arg: 0));
3881 return RValue::get(
3882 Builder.CreateZExt(V: Builder.createIsFPClass(FPNum: V, Test: FPClassTest::fcSubnormal),
3883 DestTy: ConvertType(E->getType())));
3884 }
3885
3886 case Builtin::BI__builtin_iszero: {
3887 CodeGenFunction::CGFPOptionsRAII FPOptsRAII(*this, E);
3888 Value *V = EmitScalarExpr(E: E->getArg(Arg: 0));
3889 return RValue::get(
3890 Builder.CreateZExt(V: Builder.createIsFPClass(FPNum: V, Test: FPClassTest::fcZero),
3891 DestTy: ConvertType(E->getType())));
3892 }
3893
3894 case Builtin::BI__builtin_isfpclass: {
3895 Expr::EvalResult Result;
3896 if (!E->getArg(Arg: 1)->EvaluateAsInt(Result, Ctx: CGM.getContext()))
3897 break;
3898 uint64_t Test = Result.Val.getInt().getLimitedValue();
3899 CodeGenFunction::CGFPOptionsRAII FPOptsRAII(*this, E);
3900 Value *V = EmitScalarExpr(E: E->getArg(Arg: 0));
3901 return RValue::get(Builder.CreateZExt(V: Builder.createIsFPClass(FPNum: V, Test),
3902 DestTy: ConvertType(E->getType())));
3903 }
3904
3905 case Builtin::BI__builtin_nondeterministic_value: {
3906 llvm::Type *Ty = ConvertType(T: E->getArg(Arg: 0)->getType());
3907
3908 Value *Result = PoisonValue::get(T: Ty);
3909 Result = Builder.CreateFreeze(V: Result);
3910
3911 return RValue::get(V: Result);
3912 }
3913
3914 case Builtin::BI__builtin_elementwise_abs: {
3915 Value *Result;
3916 QualType QT = E->getArg(Arg: 0)->getType();
3917
3918 if (auto *VecTy = QT->getAs<VectorType>())
3919 QT = VecTy->getElementType();
3920 if (QT->isIntegerType())
3921 Result = Builder.CreateBinaryIntrinsic(
3922 Intrinsic::abs, EmitScalarExpr(E->getArg(0)), Builder.getFalse(),
3923 nullptr, "elt.abs");
3924 else
3925 Result = emitBuiltinWithOneOverloadedType<1>(*this, E, Intrinsic::fabs,
3926 "elt.abs");
3927
3928 return RValue::get(V: Result);
3929 }
3930 case Builtin::BI__builtin_elementwise_acos:
3931 return RValue::get(emitBuiltinWithOneOverloadedType<1>(
3932 *this, E, Intrinsic::acos, "elt.acos"));
3933 case Builtin::BI__builtin_elementwise_asin:
3934 return RValue::get(emitBuiltinWithOneOverloadedType<1>(
3935 *this, E, Intrinsic::asin, "elt.asin"));
3936 case Builtin::BI__builtin_elementwise_atan:
3937 return RValue::get(emitBuiltinWithOneOverloadedType<1>(
3938 *this, E, Intrinsic::atan, "elt.atan"));
3939 case Builtin::BI__builtin_elementwise_atan2:
3940 return RValue::get(emitBuiltinWithOneOverloadedType<2>(
3941 *this, E, Intrinsic::atan2, "elt.atan2"));
3942 case Builtin::BI__builtin_elementwise_ceil:
3943 return RValue::get(emitBuiltinWithOneOverloadedType<1>(
3944 *this, E, Intrinsic::ceil, "elt.ceil"));
3945 case Builtin::BI__builtin_elementwise_exp:
3946 return RValue::get(emitBuiltinWithOneOverloadedType<1>(
3947 *this, E, Intrinsic::exp, "elt.exp"));
3948 case Builtin::BI__builtin_elementwise_exp2:
3949 return RValue::get(emitBuiltinWithOneOverloadedType<1>(
3950 *this, E, Intrinsic::exp2, "elt.exp2"));
3951 case Builtin::BI__builtin_elementwise_exp10:
3952 return RValue::get(emitBuiltinWithOneOverloadedType<1>(
3953 *this, E, Intrinsic::exp10, "elt.exp10"));
3954 case Builtin::BI__builtin_elementwise_log:
3955 return RValue::get(emitBuiltinWithOneOverloadedType<1>(
3956 *this, E, Intrinsic::log, "elt.log"));
3957 case Builtin::BI__builtin_elementwise_log2:
3958 return RValue::get(emitBuiltinWithOneOverloadedType<1>(
3959 *this, E, Intrinsic::log2, "elt.log2"));
3960 case Builtin::BI__builtin_elementwise_log10:
3961 return RValue::get(emitBuiltinWithOneOverloadedType<1>(
3962 *this, E, Intrinsic::log10, "elt.log10"));
3963 case Builtin::BI__builtin_elementwise_pow: {
3964 return RValue::get(
3965 emitBuiltinWithOneOverloadedType<2>(*this, E, Intrinsic::pow));
3966 }
3967 case Builtin::BI__builtin_elementwise_bitreverse:
3968 return RValue::get(emitBuiltinWithOneOverloadedType<1>(
3969 *this, E, Intrinsic::bitreverse, "elt.bitreverse"));
3970 case Builtin::BI__builtin_elementwise_cos:
3971 return RValue::get(emitBuiltinWithOneOverloadedType<1>(
3972 *this, E, Intrinsic::cos, "elt.cos"));
3973 case Builtin::BI__builtin_elementwise_cosh:
3974 return RValue::get(emitBuiltinWithOneOverloadedType<1>(
3975 *this, E, Intrinsic::cosh, "elt.cosh"));
3976 case Builtin::BI__builtin_elementwise_floor:
3977 return RValue::get(emitBuiltinWithOneOverloadedType<1>(
3978 *this, E, Intrinsic::floor, "elt.floor"));
3979 case Builtin::BI__builtin_elementwise_popcount:
3980 return RValue::get(emitBuiltinWithOneOverloadedType<1>(
3981 *this, E, Intrinsic::ctpop, "elt.ctpop"));
3982 case Builtin::BI__builtin_elementwise_roundeven:
3983 return RValue::get(emitBuiltinWithOneOverloadedType<1>(
3984 *this, E, Intrinsic::roundeven, "elt.roundeven"));
3985 case Builtin::BI__builtin_elementwise_round:
3986 return RValue::get(emitBuiltinWithOneOverloadedType<1>(
3987 *this, E, Intrinsic::round, "elt.round"));
3988 case Builtin::BI__builtin_elementwise_rint:
3989 return RValue::get(emitBuiltinWithOneOverloadedType<1>(
3990 *this, E, Intrinsic::rint, "elt.rint"));
3991 case Builtin::BI__builtin_elementwise_nearbyint:
3992 return RValue::get(emitBuiltinWithOneOverloadedType<1>(
3993 *this, E, Intrinsic::nearbyint, "elt.nearbyint"));
3994 case Builtin::BI__builtin_elementwise_sin:
3995 return RValue::get(emitBuiltinWithOneOverloadedType<1>(
3996 *this, E, Intrinsic::sin, "elt.sin"));
3997 case Builtin::BI__builtin_elementwise_sinh:
3998 return RValue::get(emitBuiltinWithOneOverloadedType<1>(
3999 *this, E, Intrinsic::sinh, "elt.sinh"));
4000 case Builtin::BI__builtin_elementwise_tan:
4001 return RValue::get(emitBuiltinWithOneOverloadedType<1>(
4002 *this, E, Intrinsic::tan, "elt.tan"));
4003 case Builtin::BI__builtin_elementwise_tanh:
4004 return RValue::get(emitBuiltinWithOneOverloadedType<1>(
4005 *this, E, Intrinsic::tanh, "elt.tanh"));
4006 case Builtin::BI__builtin_elementwise_trunc:
4007 return RValue::get(emitBuiltinWithOneOverloadedType<1>(
4008 *this, E, Intrinsic::trunc, "elt.trunc"));
4009 case Builtin::BI__builtin_elementwise_canonicalize:
4010 return RValue::get(emitBuiltinWithOneOverloadedType<1>(
4011 *this, E, Intrinsic::canonicalize, "elt.canonicalize"));
4012 case Builtin::BI__builtin_elementwise_copysign:
4013 return RValue::get(
4014 emitBuiltinWithOneOverloadedType<2>(*this, E, Intrinsic::copysign));
4015 case Builtin::BI__builtin_elementwise_fma:
4016 return RValue::get(
4017 emitBuiltinWithOneOverloadedType<3>(*this, E, Intrinsic::fma));
4018 case Builtin::BI__builtin_elementwise_add_sat:
4019 case Builtin::BI__builtin_elementwise_sub_sat: {
4020 Value *Op0 = EmitScalarExpr(E: E->getArg(Arg: 0));
4021 Value *Op1 = EmitScalarExpr(E: E->getArg(Arg: 1));
4022 Value *Result;
4023 assert(Op0->getType()->isIntOrIntVectorTy() && "integer type expected");
4024 QualType Ty = E->getArg(Arg: 0)->getType();
4025 if (auto *VecTy = Ty->getAs<VectorType>())
4026 Ty = VecTy->getElementType();
4027 bool IsSigned = Ty->isSignedIntegerType();
4028 unsigned Opc;
4029 if (BuiltinIDIfNoAsmLabel == Builtin::BI__builtin_elementwise_add_sat)
4030 Opc = IsSigned ? Intrinsic::sadd_sat : Intrinsic::uadd_sat;
4031 else
4032 Opc = IsSigned ? Intrinsic::ssub_sat : Intrinsic::usub_sat;
4033 Result = Builder.CreateBinaryIntrinsic(ID: Opc, LHS: Op0, RHS: Op1, FMFSource: nullptr, Name: "elt.sat");
4034 return RValue::get(V: Result);
4035 }
4036
4037 case Builtin::BI__builtin_elementwise_max: {
4038 Value *Op0 = EmitScalarExpr(E: E->getArg(Arg: 0));
4039 Value *Op1 = EmitScalarExpr(E: E->getArg(Arg: 1));
4040 Value *Result;
4041 if (Op0->getType()->isIntOrIntVectorTy()) {
4042 QualType Ty = E->getArg(Arg: 0)->getType();
4043 if (auto *VecTy = Ty->getAs<VectorType>())
4044 Ty = VecTy->getElementType();
4045 Result = Builder.CreateBinaryIntrinsic(
4046 Ty->isSignedIntegerType() ? Intrinsic::smax : Intrinsic::umax, Op0,
4047 Op1, nullptr, "elt.max");
4048 } else
4049 Result = Builder.CreateMaxNum(LHS: Op0, RHS: Op1, /*FMFSource=*/nullptr, Name: "elt.max");
4050 return RValue::get(V: Result);
4051 }
4052 case Builtin::BI__builtin_elementwise_min: {
4053 Value *Op0 = EmitScalarExpr(E: E->getArg(Arg: 0));
4054 Value *Op1 = EmitScalarExpr(E: E->getArg(Arg: 1));
4055 Value *Result;
4056 if (Op0->getType()->isIntOrIntVectorTy()) {
4057 QualType Ty = E->getArg(Arg: 0)->getType();
4058 if (auto *VecTy = Ty->getAs<VectorType>())
4059 Ty = VecTy->getElementType();
4060 Result = Builder.CreateBinaryIntrinsic(
4061 Ty->isSignedIntegerType() ? Intrinsic::smin : Intrinsic::umin, Op0,
4062 Op1, nullptr, "elt.min");
4063 } else
4064 Result = Builder.CreateMinNum(LHS: Op0, RHS: Op1, /*FMFSource=*/nullptr, Name: "elt.min");
4065 return RValue::get(V: Result);
4066 }
4067
4068 case Builtin::BI__builtin_elementwise_maxnum: {
4069 Value *Op0 = EmitScalarExpr(E: E->getArg(Arg: 0));
4070 Value *Op1 = EmitScalarExpr(E: E->getArg(Arg: 1));
4071 Value *Result = Builder.CreateBinaryIntrinsic(llvm::Intrinsic::maxnum, Op0,
4072 Op1, nullptr, "elt.maxnum");
4073 return RValue::get(V: Result);
4074 }
4075
4076 case Builtin::BI__builtin_elementwise_minnum: {
4077 Value *Op0 = EmitScalarExpr(E: E->getArg(Arg: 0));
4078 Value *Op1 = EmitScalarExpr(E: E->getArg(Arg: 1));
4079 Value *Result = Builder.CreateBinaryIntrinsic(llvm::Intrinsic::minnum, Op0,
4080 Op1, nullptr, "elt.minnum");
4081 return RValue::get(V: Result);
4082 }
4083
4084 case Builtin::BI__builtin_elementwise_maximum: {
4085 Value *Op0 = EmitScalarExpr(E: E->getArg(Arg: 0));
4086 Value *Op1 = EmitScalarExpr(E: E->getArg(Arg: 1));
4087 Value *Result = Builder.CreateBinaryIntrinsic(Intrinsic::maximum, Op0, Op1,
4088 nullptr, "elt.maximum");
4089 return RValue::get(V: Result);
4090 }
4091
4092 case Builtin::BI__builtin_elementwise_minimum: {
4093 Value *Op0 = EmitScalarExpr(E: E->getArg(Arg: 0));
4094 Value *Op1 = EmitScalarExpr(E: E->getArg(Arg: 1));
4095 Value *Result = Builder.CreateBinaryIntrinsic(Intrinsic::minimum, Op0, Op1,
4096 nullptr, "elt.minimum");
4097 return RValue::get(V: Result);
4098 }
4099
4100 case Builtin::BI__builtin_reduce_max: {
4101 auto GetIntrinsicID = [this](QualType QT) {
4102 if (auto *VecTy = QT->getAs<VectorType>())
4103 QT = VecTy->getElementType();
4104 else if (QT->isSizelessVectorType())
4105 QT = QT->getSizelessVectorEltType(Ctx: CGM.getContext());
4106
4107 if (QT->isSignedIntegerType())
4108 return Intrinsic::vector_reduce_smax;
4109 if (QT->isUnsignedIntegerType())
4110 return Intrinsic::vector_reduce_umax;
4111 assert(QT->isFloatingType() && "must have a float here");
4112 return Intrinsic::vector_reduce_fmax;
4113 };
4114 return RValue::get(V: emitBuiltinWithOneOverloadedType<1>(
4115 *this, E, GetIntrinsicID(E->getArg(Arg: 0)->getType()), "rdx.min"));
4116 }
4117
4118 case Builtin::BI__builtin_reduce_min: {
4119 auto GetIntrinsicID = [this](QualType QT) {
4120 if (auto *VecTy = QT->getAs<VectorType>())
4121 QT = VecTy->getElementType();
4122 else if (QT->isSizelessVectorType())
4123 QT = QT->getSizelessVectorEltType(Ctx: CGM.getContext());
4124
4125 if (QT->isSignedIntegerType())
4126 return Intrinsic::vector_reduce_smin;
4127 if (QT->isUnsignedIntegerType())
4128 return Intrinsic::vector_reduce_umin;
4129 assert(QT->isFloatingType() && "must have a float here");
4130 return Intrinsic::vector_reduce_fmin;
4131 };
4132
4133 return RValue::get(V: emitBuiltinWithOneOverloadedType<1>(
4134 *this, E, GetIntrinsicID(E->getArg(Arg: 0)->getType()), "rdx.min"));
4135 }
4136
4137 case Builtin::BI__builtin_reduce_add:
4138 return RValue::get(emitBuiltinWithOneOverloadedType<1>(
4139 *this, E, Intrinsic::vector_reduce_add, "rdx.add"));
4140 case Builtin::BI__builtin_reduce_mul:
4141 return RValue::get(emitBuiltinWithOneOverloadedType<1>(
4142 *this, E, Intrinsic::vector_reduce_mul, "rdx.mul"));
4143 case Builtin::BI__builtin_reduce_xor:
4144 return RValue::get(emitBuiltinWithOneOverloadedType<1>(
4145 *this, E, Intrinsic::vector_reduce_xor, "rdx.xor"));
4146 case Builtin::BI__builtin_reduce_or:
4147 return RValue::get(emitBuiltinWithOneOverloadedType<1>(
4148 *this, E, Intrinsic::vector_reduce_or, "rdx.or"));
4149 case Builtin::BI__builtin_reduce_and:
4150 return RValue::get(emitBuiltinWithOneOverloadedType<1>(
4151 *this, E, Intrinsic::vector_reduce_and, "rdx.and"));
4152 case Builtin::BI__builtin_reduce_maximum:
4153 return RValue::get(emitBuiltinWithOneOverloadedType<1>(
4154 *this, E, Intrinsic::vector_reduce_fmaximum, "rdx.maximum"));
4155 case Builtin::BI__builtin_reduce_minimum:
4156 return RValue::get(emitBuiltinWithOneOverloadedType<1>(
4157 *this, E, Intrinsic::vector_reduce_fminimum, "rdx.minimum"));
4158
4159 case Builtin::BI__builtin_matrix_transpose: {
4160 auto *MatrixTy = E->getArg(Arg: 0)->getType()->castAs<ConstantMatrixType>();
4161 Value *MatValue = EmitScalarExpr(E: E->getArg(Arg: 0));
4162 MatrixBuilder MB(Builder);
4163 Value *Result = MB.CreateMatrixTranspose(Matrix: MatValue, Rows: MatrixTy->getNumRows(),
4164 Columns: MatrixTy->getNumColumns());
4165 return RValue::get(V: Result);
4166 }
4167
4168 case Builtin::BI__builtin_matrix_column_major_load: {
4169 MatrixBuilder MB(Builder);
4170 // Emit everything that isn't dependent on the first parameter type
4171 Value *Stride = EmitScalarExpr(E: E->getArg(Arg: 3));
4172 const auto *ResultTy = E->getType()->getAs<ConstantMatrixType>();
4173 auto *PtrTy = E->getArg(Arg: 0)->getType()->getAs<PointerType>();
4174 assert(PtrTy && "arg0 must be of pointer type");
4175 bool IsVolatile = PtrTy->getPointeeType().isVolatileQualified();
4176
4177 Address Src = EmitPointerWithAlignment(Addr: E->getArg(Arg: 0));
4178 EmitNonNullArgCheck(RV: RValue::get(V: Src.emitRawPointer(CGF&: *this)),
4179 ArgType: E->getArg(Arg: 0)->getType(), ArgLoc: E->getArg(Arg: 0)->getExprLoc(), AC: FD,
4180 ParmNum: 0);
4181 Value *Result = MB.CreateColumnMajorLoad(
4182 EltTy: Src.getElementType(), DataPtr: Src.emitRawPointer(CGF&: *this),
4183 Alignment: Align(Src.getAlignment().getQuantity()), Stride, IsVolatile,
4184 Rows: ResultTy->getNumRows(), Columns: ResultTy->getNumColumns(), Name: "matrix");
4185 return RValue::get(V: Result);
4186 }
4187
4188 case Builtin::BI__builtin_matrix_column_major_store: {
4189 MatrixBuilder MB(Builder);
4190 Value *Matrix = EmitScalarExpr(E: E->getArg(Arg: 0));
4191 Address Dst = EmitPointerWithAlignment(Addr: E->getArg(Arg: 1));
4192 Value *Stride = EmitScalarExpr(E: E->getArg(Arg: 2));
4193
4194 const auto *MatrixTy = E->getArg(Arg: 0)->getType()->getAs<ConstantMatrixType>();
4195 auto *PtrTy = E->getArg(Arg: 1)->getType()->getAs<PointerType>();
4196 assert(PtrTy && "arg1 must be of pointer type");
4197 bool IsVolatile = PtrTy->getPointeeType().isVolatileQualified();
4198
4199 EmitNonNullArgCheck(RV: RValue::get(V: Dst.emitRawPointer(CGF&: *this)),
4200 ArgType: E->getArg(Arg: 1)->getType(), ArgLoc: E->getArg(Arg: 1)->getExprLoc(), AC: FD,
4201 ParmNum: 0);
4202 Value *Result = MB.CreateColumnMajorStore(
4203 Matrix, Ptr: Dst.emitRawPointer(CGF&: *this),
4204 Alignment: Align(Dst.getAlignment().getQuantity()), Stride, IsVolatile,
4205 Rows: MatrixTy->getNumRows(), Columns: MatrixTy->getNumColumns());
4206 addInstToNewSourceAtom(KeyInstruction: cast<Instruction>(Val: Result), Backup: Matrix);
4207 return RValue::get(V: Result);
4208 }
4209
4210 case Builtin::BI__builtin_isinf_sign: {
4211 // isinf_sign(x) -> fabs(x) == infinity ? (signbit(x) ? -1 : 1) : 0
4212 CodeGenFunction::CGFPOptionsRAII FPOptsRAII(*this, E);
4213 // FIXME: for strictfp/IEEE-754 we need to not trap on SNaN here.
4214 Value *Arg = EmitScalarExpr(E: E->getArg(Arg: 0));
4215 Value *AbsArg = EmitFAbs(CGF&: *this, V: Arg);
4216 Value *IsInf = Builder.CreateFCmpOEQ(
4217 LHS: AbsArg, RHS: ConstantFP::getInfinity(Ty: Arg->getType()), Name: "isinf");
4218 Value *IsNeg = EmitSignBit(CGF&: *this, V: Arg);
4219
4220 llvm::Type *IntTy = ConvertType(E->getType());
4221 Value *Zero = Constant::getNullValue(Ty: IntTy);
4222 Value *One = ConstantInt::get(Ty: IntTy, V: 1);
4223 Value *NegativeOne = ConstantInt::get(Ty: IntTy, V: -1);
4224 Value *SignResult = Builder.CreateSelect(C: IsNeg, True: NegativeOne, False: One);
4225 Value *Result = Builder.CreateSelect(C: IsInf, True: SignResult, False: Zero);
4226 return RValue::get(V: Result);
4227 }
4228
4229 case Builtin::BI__builtin_flt_rounds: {
4230 Function *F = CGM.getIntrinsic(Intrinsic::get_rounding);
4231
4232 llvm::Type *ResultType = ConvertType(E->getType());
4233 Value *Result = Builder.CreateCall(Callee: F);
4234 if (Result->getType() != ResultType)
4235 Result = Builder.CreateIntCast(V: Result, DestTy: ResultType, /*isSigned*/true,
4236 Name: "cast");
4237 return RValue::get(V: Result);
4238 }
4239
4240 case Builtin::BI__builtin_set_flt_rounds: {
4241 Function *F = CGM.getIntrinsic(Intrinsic::set_rounding);
4242
4243 Value *V = EmitScalarExpr(E: E->getArg(Arg: 0));
4244 Builder.CreateCall(Callee: F, Args: V);
4245 return RValue::get(V: nullptr);
4246 }
4247
4248 case Builtin::BI__builtin_fpclassify: {
4249 CodeGenFunction::CGFPOptionsRAII FPOptsRAII(*this, E);
4250 // FIXME: for strictfp/IEEE-754 we need to not trap on SNaN here.
4251 Value *V = EmitScalarExpr(E: E->getArg(Arg: 5));
4252 llvm::Type *Ty = ConvertType(T: E->getArg(Arg: 5)->getType());
4253
4254 // Create Result
4255 BasicBlock *Begin = Builder.GetInsertBlock();
4256 BasicBlock *End = createBasicBlock(name: "fpclassify_end", parent: this->CurFn);
4257 Builder.SetInsertPoint(End);
4258 PHINode *Result =
4259 Builder.CreatePHI(Ty: ConvertType(T: E->getArg(Arg: 0)->getType()), NumReservedValues: 4,
4260 Name: "fpclassify_result");
4261
4262 // if (V==0) return FP_ZERO
4263 Builder.SetInsertPoint(Begin);
4264 Value *IsZero = Builder.CreateFCmpOEQ(LHS: V, RHS: Constant::getNullValue(Ty),
4265 Name: "iszero");
4266 Value *ZeroLiteral = EmitScalarExpr(E: E->getArg(Arg: 4));
4267 BasicBlock *NotZero = createBasicBlock(name: "fpclassify_not_zero", parent: this->CurFn);
4268 Builder.CreateCondBr(Cond: IsZero, True: End, False: NotZero);
4269 Result->addIncoming(V: ZeroLiteral, BB: Begin);
4270
4271 // if (V != V) return FP_NAN
4272 Builder.SetInsertPoint(NotZero);
4273 Value *IsNan = Builder.CreateFCmpUNO(LHS: V, RHS: V, Name: "cmp");
4274 Value *NanLiteral = EmitScalarExpr(E: E->getArg(Arg: 0));
4275 BasicBlock *NotNan = createBasicBlock(name: "fpclassify_not_nan", parent: this->CurFn);
4276 Builder.CreateCondBr(Cond: IsNan, True: End, False: NotNan);
4277 Result->addIncoming(V: NanLiteral, BB: NotZero);
4278
4279 // if (fabs(V) == infinity) return FP_INFINITY
4280 Builder.SetInsertPoint(NotNan);
4281 Value *VAbs = EmitFAbs(CGF&: *this, V);
4282 Value *IsInf =
4283 Builder.CreateFCmpOEQ(LHS: VAbs, RHS: ConstantFP::getInfinity(Ty: V->getType()),
4284 Name: "isinf");
4285 Value *InfLiteral = EmitScalarExpr(E: E->getArg(Arg: 1));
4286 BasicBlock *NotInf = createBasicBlock(name: "fpclassify_not_inf", parent: this->CurFn);
4287 Builder.CreateCondBr(Cond: IsInf, True: End, False: NotInf);
4288 Result->addIncoming(V: InfLiteral, BB: NotNan);
4289
4290 // if (fabs(V) >= MIN_NORMAL) return FP_NORMAL else FP_SUBNORMAL
4291 Builder.SetInsertPoint(NotInf);
4292 APFloat Smallest = APFloat::getSmallestNormalized(
4293 Sem: getContext().getFloatTypeSemantics(T: E->getArg(Arg: 5)->getType()));
4294 Value *IsNormal =
4295 Builder.CreateFCmpUGE(LHS: VAbs, RHS: ConstantFP::get(Context&: V->getContext(), V: Smallest),
4296 Name: "isnormal");
4297 Value *NormalResult =
4298 Builder.CreateSelect(C: IsNormal, True: EmitScalarExpr(E: E->getArg(Arg: 2)),
4299 False: EmitScalarExpr(E: E->getArg(Arg: 3)));
4300 Builder.CreateBr(Dest: End);
4301 Result->addIncoming(V: NormalResult, BB: NotInf);
4302
4303 // return Result
4304 Builder.SetInsertPoint(End);
4305 return RValue::get(V: Result);
4306 }
4307
4308 // An alloca will always return a pointer to the alloca (stack) address
4309 // space. This address space need not be the same as the AST / Language
4310 // default (e.g. in C / C++ auto vars are in the generic address space). At
4311 // the AST level this is handled within CreateTempAlloca et al., but for the
4312 // builtin / dynamic alloca we have to handle it here. We use an explicit cast
4313 // instead of passing an AS to CreateAlloca so as to not inhibit optimisation.
4314 case Builtin::BIalloca:
4315 case Builtin::BI_alloca:
4316 case Builtin::BI__builtin_alloca_uninitialized:
4317 case Builtin::BI__builtin_alloca: {
4318 Value *Size = EmitScalarExpr(E: E->getArg(Arg: 0));
4319 const TargetInfo &TI = getContext().getTargetInfo();
4320 // The alignment of the alloca should correspond to __BIGGEST_ALIGNMENT__.
4321 const Align SuitableAlignmentInBytes =
4322 CGM.getContext()
4323 .toCharUnitsFromBits(BitSize: TI.getSuitableAlign())
4324 .getAsAlign();
4325 AllocaInst *AI = Builder.CreateAlloca(Ty: Builder.getInt8Ty(), ArraySize: Size);
4326 AI->setAlignment(SuitableAlignmentInBytes);
4327 if (BuiltinID != Builtin::BI__builtin_alloca_uninitialized)
4328 initializeAlloca(CGF&: *this, AI, Size, AlignmentInBytes: SuitableAlignmentInBytes);
4329 LangAS AAS = getASTAllocaAddressSpace();
4330 LangAS EAS = E->getType()->getPointeeType().getAddressSpace();
4331 if (AAS != EAS) {
4332 llvm::Type *Ty = CGM.getTypes().ConvertType(T: E->getType());
4333 return RValue::get(
4334 V: getTargetHooks().performAddrSpaceCast(CGF&: *this, V: AI, SrcAddr: AAS, DestTy: Ty));
4335 }
4336 return RValue::get(V: AI);
4337 }
4338
4339 case Builtin::BI__builtin_alloca_with_align_uninitialized:
4340 case Builtin::BI__builtin_alloca_with_align: {
4341 Value *Size = EmitScalarExpr(E: E->getArg(Arg: 0));
4342 Value *AlignmentInBitsValue = EmitScalarExpr(E: E->getArg(Arg: 1));
4343 auto *AlignmentInBitsCI = cast<ConstantInt>(Val: AlignmentInBitsValue);
4344 unsigned AlignmentInBits = AlignmentInBitsCI->getZExtValue();
4345 const Align AlignmentInBytes =
4346 CGM.getContext().toCharUnitsFromBits(BitSize: AlignmentInBits).getAsAlign();
4347 AllocaInst *AI = Builder.CreateAlloca(Ty: Builder.getInt8Ty(), ArraySize: Size);
4348 AI->setAlignment(AlignmentInBytes);
4349 if (BuiltinID != Builtin::BI__builtin_alloca_with_align_uninitialized)
4350 initializeAlloca(CGF&: *this, AI, Size, AlignmentInBytes);
4351 LangAS AAS = getASTAllocaAddressSpace();
4352 LangAS EAS = E->getType()->getPointeeType().getAddressSpace();
4353 if (AAS != EAS) {
4354 llvm::Type *Ty = CGM.getTypes().ConvertType(T: E->getType());
4355 return RValue::get(
4356 V: getTargetHooks().performAddrSpaceCast(CGF&: *this, V: AI, SrcAddr: AAS, DestTy: Ty));
4357 }
4358 return RValue::get(V: AI);
4359 }
4360
4361 case Builtin::BIbzero:
4362 case Builtin::BI__builtin_bzero: {
4363 Address Dest = EmitPointerWithAlignment(Addr: E->getArg(Arg: 0));
4364 Value *SizeVal = EmitScalarExpr(E: E->getArg(Arg: 1));
4365 EmitNonNullArgCheck(Addr: Dest, ArgType: E->getArg(Arg: 0)->getType(),
4366 ArgLoc: E->getArg(Arg: 0)->getExprLoc(), AC: FD, ParmNum: 0);
4367 auto *I = Builder.CreateMemSet(Dest, Value: Builder.getInt8(C: 0), Size: SizeVal, IsVolatile: false);
4368 addInstToNewSourceAtom(KeyInstruction: I, Backup: nullptr);
4369 return RValue::get(V: nullptr);
4370 }
4371
4372 case Builtin::BIbcopy:
4373 case Builtin::BI__builtin_bcopy: {
4374 Address Src = EmitPointerWithAlignment(Addr: E->getArg(Arg: 0));
4375 Address Dest = EmitPointerWithAlignment(Addr: E->getArg(Arg: 1));
4376 Value *SizeVal = EmitScalarExpr(E: E->getArg(Arg: 2));
4377 EmitNonNullArgCheck(RV: RValue::get(V: Src.emitRawPointer(CGF&: *this)),
4378 ArgType: E->getArg(Arg: 0)->getType(), ArgLoc: E->getArg(Arg: 0)->getExprLoc(), AC: FD,
4379 ParmNum: 0);
4380 EmitNonNullArgCheck(RV: RValue::get(V: Dest.emitRawPointer(CGF&: *this)),
4381 ArgType: E->getArg(Arg: 1)->getType(), ArgLoc: E->getArg(Arg: 1)->getExprLoc(), AC: FD,
4382 ParmNum: 0);
4383 auto *I = Builder.CreateMemMove(Dest, Src, Size: SizeVal, IsVolatile: false);
4384 addInstToNewSourceAtom(KeyInstruction: I, Backup: nullptr);
4385 return RValue::get(V: nullptr);
4386 }
4387
4388 case Builtin::BImemcpy:
4389 case Builtin::BI__builtin_memcpy:
4390 case Builtin::BImempcpy:
4391 case Builtin::BI__builtin_mempcpy: {
4392 Address Dest = EmitPointerWithAlignment(Addr: E->getArg(Arg: 0));
4393 Address Src = EmitPointerWithAlignment(Addr: E->getArg(Arg: 1));
4394 Value *SizeVal = EmitScalarExpr(E: E->getArg(Arg: 2));
4395 EmitArgCheck(TCK_Store, Dest, E->getArg(Arg: 0), 0);
4396 EmitArgCheck(TCK_Load, Src, E->getArg(Arg: 1), 1);
4397 auto *I = Builder.CreateMemCpy(Dest, Src, Size: SizeVal, IsVolatile: false);
4398 addInstToNewSourceAtom(KeyInstruction: I, Backup: nullptr);
4399 if (BuiltinID == Builtin::BImempcpy ||
4400 BuiltinID == Builtin::BI__builtin_mempcpy)
4401 return RValue::get(V: Builder.CreateInBoundsGEP(
4402 Ty: Dest.getElementType(), Ptr: Dest.emitRawPointer(CGF&: *this), IdxList: SizeVal));
4403 else
4404 return RValue::get(Addr: Dest, CGF&: *this);
4405 }
4406
4407 case Builtin::BI__builtin_memcpy_inline: {
4408 Address Dest = EmitPointerWithAlignment(Addr: E->getArg(Arg: 0));
4409 Address Src = EmitPointerWithAlignment(Addr: E->getArg(Arg: 1));
4410 uint64_t Size =
4411 E->getArg(Arg: 2)->EvaluateKnownConstInt(Ctx: getContext()).getZExtValue();
4412 EmitArgCheck(TCK_Store, Dest, E->getArg(Arg: 0), 0);
4413 EmitArgCheck(TCK_Load, Src, E->getArg(Arg: 1), 1);
4414 auto *I = Builder.CreateMemCpyInline(Dest, Src, Size);
4415 addInstToNewSourceAtom(KeyInstruction: I, Backup: nullptr);
4416 return RValue::get(V: nullptr);
4417 }
4418
4419 case Builtin::BI__builtin_char_memchr:
4420 BuiltinID = Builtin::BI__builtin_memchr;
4421 break;
4422
4423 case Builtin::BI__builtin___memcpy_chk: {
4424 // fold __builtin_memcpy_chk(x, y, cst1, cst2) to memcpy iff cst1<=cst2.
4425 Expr::EvalResult SizeResult, DstSizeResult;
4426 if (!E->getArg(Arg: 2)->EvaluateAsInt(Result&: SizeResult, Ctx: CGM.getContext()) ||
4427 !E->getArg(Arg: 3)->EvaluateAsInt(Result&: DstSizeResult, Ctx: CGM.getContext()))
4428 break;
4429 llvm::APSInt Size = SizeResult.Val.getInt();
4430 llvm::APSInt DstSize = DstSizeResult.Val.getInt();
4431 if (Size.ugt(RHS: DstSize))
4432 break;
4433 Address Dest = EmitPointerWithAlignment(Addr: E->getArg(Arg: 0));
4434 Address Src = EmitPointerWithAlignment(Addr: E->getArg(Arg: 1));
4435 Value *SizeVal = llvm::ConstantInt::get(Context&: Builder.getContext(), V: Size);
4436 auto *I = Builder.CreateMemCpy(Dest, Src, Size: SizeVal, IsVolatile: false);
4437 addInstToNewSourceAtom(KeyInstruction: I, Backup: nullptr);
4438 return RValue::get(Addr: Dest, CGF&: *this);
4439 }
4440
4441 case Builtin::BI__builtin_objc_memmove_collectable: {
4442 Address DestAddr = EmitPointerWithAlignment(Addr: E->getArg(Arg: 0));
4443 Address SrcAddr = EmitPointerWithAlignment(Addr: E->getArg(Arg: 1));
4444 Value *SizeVal = EmitScalarExpr(E: E->getArg(Arg: 2));
4445 CGM.getObjCRuntime().EmitGCMemmoveCollectable(CGF&: *this,
4446 DestPtr: DestAddr, SrcPtr: SrcAddr, Size: SizeVal);
4447 return RValue::get(Addr: DestAddr, CGF&: *this);
4448 }
4449
4450 case Builtin::BI__builtin___memmove_chk: {
4451 // fold __builtin_memmove_chk(x, y, cst1, cst2) to memmove iff cst1<=cst2.
4452 Expr::EvalResult SizeResult, DstSizeResult;
4453 if (!E->getArg(Arg: 2)->EvaluateAsInt(Result&: SizeResult, Ctx: CGM.getContext()) ||
4454 !E->getArg(Arg: 3)->EvaluateAsInt(Result&: DstSizeResult, Ctx: CGM.getContext()))
4455 break;
4456 llvm::APSInt Size = SizeResult.Val.getInt();
4457 llvm::APSInt DstSize = DstSizeResult.Val.getInt();
4458 if (Size.ugt(RHS: DstSize))
4459 break;
4460 Address Dest = EmitPointerWithAlignment(Addr: E->getArg(Arg: 0));
4461 Address Src = EmitPointerWithAlignment(Addr: E->getArg(Arg: 1));
4462 Value *SizeVal = llvm::ConstantInt::get(Context&: Builder.getContext(), V: Size);
4463 auto *I = Builder.CreateMemMove(Dest, Src, Size: SizeVal, IsVolatile: false);
4464 addInstToNewSourceAtom(KeyInstruction: I, Backup: nullptr);
4465 return RValue::get(Addr: Dest, CGF&: *this);
4466 }
4467
4468 case Builtin::BI__builtin_trivially_relocate:
4469 case Builtin::BImemmove:
4470 case Builtin::BI__builtin_memmove: {
4471 Address Dest = EmitPointerWithAlignment(Addr: E->getArg(Arg: 0));
4472 Address Src = EmitPointerWithAlignment(Addr: E->getArg(Arg: 1));
4473 Value *SizeVal = EmitScalarExpr(E: E->getArg(Arg: 2));
4474 if (BuiltinIDIfNoAsmLabel == Builtin::BI__builtin_trivially_relocate)
4475 SizeVal = Builder.CreateMul(
4476 LHS: SizeVal,
4477 RHS: ConstantInt::get(
4478 Ty: SizeVal->getType(),
4479 V: getContext()
4480 .getTypeSizeInChars(T: E->getArg(Arg: 0)->getType()->getPointeeType())
4481 .getQuantity()));
4482 EmitArgCheck(TCK_Store, Dest, E->getArg(Arg: 0), 0);
4483 EmitArgCheck(TCK_Load, Src, E->getArg(Arg: 1), 1);
4484 auto *I = Builder.CreateMemMove(Dest, Src, Size: SizeVal, IsVolatile: false);
4485 addInstToNewSourceAtom(KeyInstruction: I, Backup: nullptr);
4486 return RValue::get(Addr: Dest, CGF&: *this);
4487 }
4488 case Builtin::BImemset:
4489 case Builtin::BI__builtin_memset: {
4490 Address Dest = EmitPointerWithAlignment(Addr: E->getArg(Arg: 0));
4491 Value *ByteVal = Builder.CreateTrunc(V: EmitScalarExpr(E: E->getArg(Arg: 1)),
4492 DestTy: Builder.getInt8Ty());
4493 Value *SizeVal = EmitScalarExpr(E: E->getArg(Arg: 2));
4494 EmitNonNullArgCheck(Addr: Dest, ArgType: E->getArg(Arg: 0)->getType(),
4495 ArgLoc: E->getArg(Arg: 0)->getExprLoc(), AC: FD, ParmNum: 0);
4496 auto *I = Builder.CreateMemSet(Dest, Value: ByteVal, Size: SizeVal, IsVolatile: false);
4497 addInstToNewSourceAtom(KeyInstruction: I, Backup: ByteVal);
4498 return RValue::get(Addr: Dest, CGF&: *this);
4499 }
4500 case Builtin::BI__builtin_memset_inline: {
4501 Address Dest = EmitPointerWithAlignment(Addr: E->getArg(Arg: 0));
4502 Value *ByteVal =
4503 Builder.CreateTrunc(V: EmitScalarExpr(E: E->getArg(Arg: 1)), DestTy: Builder.getInt8Ty());
4504 uint64_t Size =
4505 E->getArg(Arg: 2)->EvaluateKnownConstInt(Ctx: getContext()).getZExtValue();
4506 EmitNonNullArgCheck(RV: RValue::get(V: Dest.emitRawPointer(CGF&: *this)),
4507 ArgType: E->getArg(Arg: 0)->getType(), ArgLoc: E->getArg(Arg: 0)->getExprLoc(), AC: FD,
4508 ParmNum: 0);
4509 auto *I = Builder.CreateMemSetInline(Dest, Value: ByteVal, Size);
4510 addInstToNewSourceAtom(KeyInstruction: I, Backup: nullptr);
4511 return RValue::get(V: nullptr);
4512 }
4513 case Builtin::BI__builtin___memset_chk: {
4514 // fold __builtin_memset_chk(x, y, cst1, cst2) to memset iff cst1<=cst2.
4515 Expr::EvalResult SizeResult, DstSizeResult;
4516 if (!E->getArg(Arg: 2)->EvaluateAsInt(Result&: SizeResult, Ctx: CGM.getContext()) ||
4517 !E->getArg(Arg: 3)->EvaluateAsInt(Result&: DstSizeResult, Ctx: CGM.getContext()))
4518 break;
4519 llvm::APSInt Size = SizeResult.Val.getInt();
4520 llvm::APSInt DstSize = DstSizeResult.Val.getInt();
4521 if (Size.ugt(RHS: DstSize))
4522 break;
4523 Address Dest = EmitPointerWithAlignment(Addr: E->getArg(Arg: 0));
4524 Value *ByteVal = Builder.CreateTrunc(V: EmitScalarExpr(E: E->getArg(Arg: 1)),
4525 DestTy: Builder.getInt8Ty());
4526 Value *SizeVal = llvm::ConstantInt::get(Context&: Builder.getContext(), V: Size);
4527 auto *I = Builder.CreateMemSet(Dest, Value: ByteVal, Size: SizeVal, IsVolatile: false);
4528 addInstToNewSourceAtom(KeyInstruction: I, Backup: nullptr);
4529 return RValue::get(Addr: Dest, CGF&: *this);
4530 }
4531 case Builtin::BI__builtin_wmemchr: {
4532 // The MSVC runtime library does not provide a definition of wmemchr, so we
4533 // need an inline implementation.
4534 if (!getTarget().getTriple().isOSMSVCRT())
4535 break;
4536
4537 llvm::Type *WCharTy = ConvertType(getContext().WCharTy);
4538 Value *Str = EmitScalarExpr(E: E->getArg(Arg: 0));
4539 Value *Chr = EmitScalarExpr(E: E->getArg(Arg: 1));
4540 Value *Size = EmitScalarExpr(E: E->getArg(Arg: 2));
4541
4542 BasicBlock *Entry = Builder.GetInsertBlock();
4543 BasicBlock *CmpEq = createBasicBlock(name: "wmemchr.eq");
4544 BasicBlock *Next = createBasicBlock(name: "wmemchr.next");
4545 BasicBlock *Exit = createBasicBlock(name: "wmemchr.exit");
4546 Value *SizeEq0 = Builder.CreateICmpEQ(LHS: Size, RHS: ConstantInt::get(Ty: SizeTy, V: 0));
4547 Builder.CreateCondBr(Cond: SizeEq0, True: Exit, False: CmpEq);
4548
4549 EmitBlock(BB: CmpEq);
4550 PHINode *StrPhi = Builder.CreatePHI(Ty: Str->getType(), NumReservedValues: 2);
4551 StrPhi->addIncoming(V: Str, BB: Entry);
4552 PHINode *SizePhi = Builder.CreatePHI(Ty: SizeTy, NumReservedValues: 2);
4553 SizePhi->addIncoming(V: Size, BB: Entry);
4554 CharUnits WCharAlign =
4555 getContext().getTypeAlignInChars(getContext().WCharTy);
4556 Value *StrCh = Builder.CreateAlignedLoad(Ty: WCharTy, Addr: StrPhi, Align: WCharAlign);
4557 Value *FoundChr = Builder.CreateConstInBoundsGEP1_32(Ty: WCharTy, Ptr: StrPhi, Idx0: 0);
4558 Value *StrEqChr = Builder.CreateICmpEQ(LHS: StrCh, RHS: Chr);
4559 Builder.CreateCondBr(Cond: StrEqChr, True: Exit, False: Next);
4560
4561 EmitBlock(BB: Next);
4562 Value *NextStr = Builder.CreateConstInBoundsGEP1_32(Ty: WCharTy, Ptr: StrPhi, Idx0: 1);
4563 Value *NextSize = Builder.CreateSub(LHS: SizePhi, RHS: ConstantInt::get(Ty: SizeTy, V: 1));
4564 Value *NextSizeEq0 =
4565 Builder.CreateICmpEQ(LHS: NextSize, RHS: ConstantInt::get(Ty: SizeTy, V: 0));
4566 Builder.CreateCondBr(Cond: NextSizeEq0, True: Exit, False: CmpEq);
4567 StrPhi->addIncoming(V: NextStr, BB: Next);
4568 SizePhi->addIncoming(V: NextSize, BB: Next);
4569
4570 EmitBlock(BB: Exit);
4571 PHINode *Ret = Builder.CreatePHI(Ty: Str->getType(), NumReservedValues: 3);
4572 Ret->addIncoming(V: llvm::Constant::getNullValue(Ty: Str->getType()), BB: Entry);
4573 Ret->addIncoming(V: llvm::Constant::getNullValue(Ty: Str->getType()), BB: Next);
4574 Ret->addIncoming(V: FoundChr, BB: CmpEq);
4575 return RValue::get(V: Ret);
4576 }
4577 case Builtin::BI__builtin_wmemcmp: {
4578 // The MSVC runtime library does not provide a definition of wmemcmp, so we
4579 // need an inline implementation.
4580 if (!getTarget().getTriple().isOSMSVCRT())
4581 break;
4582
4583 llvm::Type *WCharTy = ConvertType(getContext().WCharTy);
4584
4585 Value *Dst = EmitScalarExpr(E: E->getArg(Arg: 0));
4586 Value *Src = EmitScalarExpr(E: E->getArg(Arg: 1));
4587 Value *Size = EmitScalarExpr(E: E->getArg(Arg: 2));
4588
4589 BasicBlock *Entry = Builder.GetInsertBlock();
4590 BasicBlock *CmpGT = createBasicBlock(name: "wmemcmp.gt");
4591 BasicBlock *CmpLT = createBasicBlock(name: "wmemcmp.lt");
4592 BasicBlock *Next = createBasicBlock(name: "wmemcmp.next");
4593 BasicBlock *Exit = createBasicBlock(name: "wmemcmp.exit");
4594 Value *SizeEq0 = Builder.CreateICmpEQ(LHS: Size, RHS: ConstantInt::get(Ty: SizeTy, V: 0));
4595 Builder.CreateCondBr(Cond: SizeEq0, True: Exit, False: CmpGT);
4596
4597 EmitBlock(BB: CmpGT);
4598 PHINode *DstPhi = Builder.CreatePHI(Ty: Dst->getType(), NumReservedValues: 2);
4599 DstPhi->addIncoming(V: Dst, BB: Entry);
4600 PHINode *SrcPhi = Builder.CreatePHI(Ty: Src->getType(), NumReservedValues: 2);
4601 SrcPhi->addIncoming(V: Src, BB: Entry);
4602 PHINode *SizePhi = Builder.CreatePHI(Ty: SizeTy, NumReservedValues: 2);
4603 SizePhi->addIncoming(V: Size, BB: Entry);
4604 CharUnits WCharAlign =
4605 getContext().getTypeAlignInChars(getContext().WCharTy);
4606 Value *DstCh = Builder.CreateAlignedLoad(Ty: WCharTy, Addr: DstPhi, Align: WCharAlign);
4607 Value *SrcCh = Builder.CreateAlignedLoad(Ty: WCharTy, Addr: SrcPhi, Align: WCharAlign);
4608 Value *DstGtSrc = Builder.CreateICmpUGT(LHS: DstCh, RHS: SrcCh);
4609 Builder.CreateCondBr(Cond: DstGtSrc, True: Exit, False: CmpLT);
4610
4611 EmitBlock(BB: CmpLT);
4612 Value *DstLtSrc = Builder.CreateICmpULT(LHS: DstCh, RHS: SrcCh);
4613 Builder.CreateCondBr(Cond: DstLtSrc, True: Exit, False: Next);
4614
4615 EmitBlock(BB: Next);
4616 Value *NextDst = Builder.CreateConstInBoundsGEP1_32(Ty: WCharTy, Ptr: DstPhi, Idx0: 1);
4617 Value *NextSrc = Builder.CreateConstInBoundsGEP1_32(Ty: WCharTy, Ptr: SrcPhi, Idx0: 1);
4618 Value *NextSize = Builder.CreateSub(LHS: SizePhi, RHS: ConstantInt::get(Ty: SizeTy, V: 1));
4619 Value *NextSizeEq0 =
4620 Builder.CreateICmpEQ(LHS: NextSize, RHS: ConstantInt::get(Ty: SizeTy, V: 0));
4621 Builder.CreateCondBr(Cond: NextSizeEq0, True: Exit, False: CmpGT);
4622 DstPhi->addIncoming(V: NextDst, BB: Next);
4623 SrcPhi->addIncoming(V: NextSrc, BB: Next);
4624 SizePhi->addIncoming(V: NextSize, BB: Next);
4625
4626 EmitBlock(BB: Exit);
4627 PHINode *Ret = Builder.CreatePHI(Ty: IntTy, NumReservedValues: 4);
4628 Ret->addIncoming(V: ConstantInt::get(Ty: IntTy, V: 0), BB: Entry);
4629 Ret->addIncoming(V: ConstantInt::get(Ty: IntTy, V: 1), BB: CmpGT);
4630 Ret->addIncoming(V: ConstantInt::get(Ty: IntTy, V: -1), BB: CmpLT);
4631 Ret->addIncoming(V: ConstantInt::get(Ty: IntTy, V: 0), BB: Next);
4632 return RValue::get(V: Ret);
4633 }
4634 case Builtin::BI__builtin_dwarf_cfa: {
4635 // The offset in bytes from the first argument to the CFA.
4636 //
4637 // Why on earth is this in the frontend? Is there any reason at
4638 // all that the backend can't reasonably determine this while
4639 // lowering llvm.eh.dwarf.cfa()?
4640 //
4641 // TODO: If there's a satisfactory reason, add a target hook for
4642 // this instead of hard-coding 0, which is correct for most targets.
4643 int32_t Offset = 0;
4644
4645 Function *F = CGM.getIntrinsic(Intrinsic::eh_dwarf_cfa);
4646 return RValue::get(V: Builder.CreateCall(Callee: F,
4647 Args: llvm::ConstantInt::get(Ty: Int32Ty, V: Offset)));
4648 }
4649 case Builtin::BI__builtin_return_address: {
4650 Value *Depth = ConstantEmitter(*this).emitAbstract(E->getArg(Arg: 0),
4651 getContext().UnsignedIntTy);
4652 Function *F = CGM.getIntrinsic(Intrinsic::returnaddress);
4653 return RValue::get(V: Builder.CreateCall(Callee: F, Args: Depth));
4654 }
4655 case Builtin::BI_ReturnAddress: {
4656 Function *F = CGM.getIntrinsic(Intrinsic::returnaddress);
4657 return RValue::get(V: Builder.CreateCall(Callee: F, Args: Builder.getInt32(C: 0)));
4658 }
4659 case Builtin::BI__builtin_frame_address: {
4660 Value *Depth = ConstantEmitter(*this).emitAbstract(E->getArg(Arg: 0),
4661 getContext().UnsignedIntTy);
4662 Function *F = CGM.getIntrinsic(Intrinsic::frameaddress, AllocaInt8PtrTy);
4663 return RValue::get(V: Builder.CreateCall(Callee: F, Args: Depth));
4664 }
4665 case Builtin::BI__builtin_extract_return_addr: {
4666 Value *Address = EmitScalarExpr(E: E->getArg(Arg: 0));
4667 Value *Result = getTargetHooks().decodeReturnAddress(CGF&: *this, Address);
4668 return RValue::get(V: Result);
4669 }
4670 case Builtin::BI__builtin_frob_return_addr: {
4671 Value *Address = EmitScalarExpr(E: E->getArg(Arg: 0));
4672 Value *Result = getTargetHooks().encodeReturnAddress(CGF&: *this, Address);
4673 return RValue::get(V: Result);
4674 }
4675 case Builtin::BI__builtin_dwarf_sp_column: {
4676 llvm::IntegerType *Ty
4677 = cast<llvm::IntegerType>(ConvertType(E->getType()));
4678 int Column = getTargetHooks().getDwarfEHStackPointer(M&: CGM);
4679 if (Column == -1) {
4680 CGM.ErrorUnsupported(E, "__builtin_dwarf_sp_column");
4681 return RValue::get(V: llvm::UndefValue::get(T: Ty));
4682 }
4683 return RValue::get(V: llvm::ConstantInt::get(Ty, V: Column, IsSigned: true));
4684 }
4685 case Builtin::BI__builtin_init_dwarf_reg_size_table: {
4686 Value *Address = EmitScalarExpr(E: E->getArg(Arg: 0));
4687 if (getTargetHooks().initDwarfEHRegSizeTable(CGF&: *this, Address))
4688 CGM.ErrorUnsupported(E, "__builtin_init_dwarf_reg_size_table");
4689 return RValue::get(llvm::UndefValue::get(T: ConvertType(E->getType())));
4690 }
4691 case Builtin::BI__builtin_eh_return: {
4692 Value *Int = EmitScalarExpr(E: E->getArg(Arg: 0));
4693 Value *Ptr = EmitScalarExpr(E: E->getArg(Arg: 1));
4694
4695 llvm::IntegerType *IntTy = cast<llvm::IntegerType>(Val: Int->getType());
4696 assert((IntTy->getBitWidth() == 32 || IntTy->getBitWidth() == 64) &&
4697 "LLVM's __builtin_eh_return only supports 32- and 64-bit variants");
4698 Function *F =
4699 CGM.getIntrinsic(IntTy->getBitWidth() == 32 ? Intrinsic::eh_return_i32
4700 : Intrinsic::eh_return_i64);
4701 Builder.CreateCall(Callee: F, Args: {Int, Ptr});
4702 Builder.CreateUnreachable();
4703
4704 // We do need to preserve an insertion point.
4705 EmitBlock(BB: createBasicBlock(name: "builtin_eh_return.cont"));
4706
4707 return RValue::get(V: nullptr);
4708 }
4709 case Builtin::BI__builtin_unwind_init: {
4710 Function *F = CGM.getIntrinsic(Intrinsic::eh_unwind_init);
4711 Builder.CreateCall(Callee: F);
4712 return RValue::get(V: nullptr);
4713 }
4714 case Builtin::BI__builtin_extend_pointer: {
4715 // Extends a pointer to the size of an _Unwind_Word, which is
4716 // uint64_t on all platforms. Generally this gets poked into a
4717 // register and eventually used as an address, so if the
4718 // addressing registers are wider than pointers and the platform
4719 // doesn't implicitly ignore high-order bits when doing
4720 // addressing, we need to make sure we zext / sext based on
4721 // the platform's expectations.
4722 //
4723 // See: http://gcc.gnu.org/ml/gcc-bugs/2002-02/msg00237.html
4724
4725 // Cast the pointer to intptr_t.
4726 Value *Ptr = EmitScalarExpr(E: E->getArg(Arg: 0));
4727 Value *Result = Builder.CreatePtrToInt(V: Ptr, DestTy: IntPtrTy, Name: "extend.cast");
4728
4729 // If that's 64 bits, we're done.
4730 if (IntPtrTy->getBitWidth() == 64)
4731 return RValue::get(V: Result);
4732
4733 // Otherwise, ask the codegen data what to do.
4734 if (getTargetHooks().extendPointerWithSExt())
4735 return RValue::get(V: Builder.CreateSExt(V: Result, DestTy: Int64Ty, Name: "extend.sext"));
4736 else
4737 return RValue::get(V: Builder.CreateZExt(V: Result, DestTy: Int64Ty, Name: "extend.zext"));
4738 }
4739 case Builtin::BI__builtin_setjmp: {
4740 // Buffer is a void**.
4741 Address Buf = EmitPointerWithAlignment(Addr: E->getArg(Arg: 0));
4742
4743 if (getTarget().getTriple().getArch() == llvm::Triple::systemz) {
4744 // On this target, the back end fills in the context buffer completely.
4745 // It doesn't really matter if the frontend stores to the buffer before
4746 // calling setjmp, the back-end is going to overwrite them anyway.
4747 Function *F = CGM.getIntrinsic(Intrinsic::eh_sjlj_setjmp);
4748 return RValue::get(V: Builder.CreateCall(Callee: F, Args: Buf.emitRawPointer(CGF&: *this)));
4749 }
4750
4751 // Store the frame pointer to the setjmp buffer.
4752 Value *FrameAddr = Builder.CreateCall(
4753 CGM.getIntrinsic(Intrinsic::frameaddress, AllocaInt8PtrTy),
4754 ConstantInt::get(Int32Ty, 0));
4755 Builder.CreateStore(Val: FrameAddr, Addr: Buf);
4756
4757 // Store the stack pointer to the setjmp buffer.
4758 Value *StackAddr = Builder.CreateStackSave();
4759 assert(Buf.emitRawPointer(*this)->getType() == StackAddr->getType());
4760
4761 Address StackSaveSlot = Builder.CreateConstInBoundsGEP(Addr: Buf, Index: 2);
4762 Builder.CreateStore(Val: StackAddr, Addr: StackSaveSlot);
4763
4764 // Call LLVM's EH setjmp, which is lightweight.
4765 Function *F = CGM.getIntrinsic(Intrinsic::eh_sjlj_setjmp);
4766 return RValue::get(V: Builder.CreateCall(Callee: F, Args: Buf.emitRawPointer(CGF&: *this)));
4767 }
4768 case Builtin::BI__builtin_longjmp: {
4769 Value *Buf = EmitScalarExpr(E: E->getArg(Arg: 0));
4770
4771 // Call LLVM's EH longjmp, which is lightweight.
4772 Builder.CreateCall(CGM.getIntrinsic(Intrinsic::eh_sjlj_longjmp), Buf);
4773
4774 // longjmp doesn't return; mark this as unreachable.
4775 Builder.CreateUnreachable();
4776
4777 // We do need to preserve an insertion point.
4778 EmitBlock(BB: createBasicBlock(name: "longjmp.cont"));
4779
4780 return RValue::get(V: nullptr);
4781 }
4782 case Builtin::BI__builtin_launder: {
4783 const Expr *Arg = E->getArg(Arg: 0);
4784 QualType ArgTy = Arg->getType()->getPointeeType();
4785 Value *Ptr = EmitScalarExpr(E: Arg);
4786 if (TypeRequiresBuiltinLaunder(CGM, Ty: ArgTy))
4787 Ptr = Builder.CreateLaunderInvariantGroup(Ptr);
4788
4789 return RValue::get(V: Ptr);
4790 }
4791 case Builtin::BI__sync_fetch_and_add:
4792 case Builtin::BI__sync_fetch_and_sub:
4793 case Builtin::BI__sync_fetch_and_or:
4794 case Builtin::BI__sync_fetch_and_and:
4795 case Builtin::BI__sync_fetch_and_xor:
4796 case Builtin::BI__sync_fetch_and_nand:
4797 case Builtin::BI__sync_add_and_fetch:
4798 case Builtin::BI__sync_sub_and_fetch:
4799 case Builtin::BI__sync_and_and_fetch:
4800 case Builtin::BI__sync_or_and_fetch:
4801 case Builtin::BI__sync_xor_and_fetch:
4802 case Builtin::BI__sync_nand_and_fetch:
4803 case Builtin::BI__sync_val_compare_and_swap:
4804 case Builtin::BI__sync_bool_compare_and_swap:
4805 case Builtin::BI__sync_lock_test_and_set:
4806 case Builtin::BI__sync_lock_release:
4807 case Builtin::BI__sync_swap:
4808 llvm_unreachable("Shouldn't make it through sema");
4809 case Builtin::BI__sync_fetch_and_add_1:
4810 case Builtin::BI__sync_fetch_and_add_2:
4811 case Builtin::BI__sync_fetch_and_add_4:
4812 case Builtin::BI__sync_fetch_and_add_8:
4813 case Builtin::BI__sync_fetch_and_add_16:
4814 return EmitBinaryAtomic(CGF&: *this, Kind: llvm::AtomicRMWInst::Add, E);
4815 case Builtin::BI__sync_fetch_and_sub_1:
4816 case Builtin::BI__sync_fetch_and_sub_2:
4817 case Builtin::BI__sync_fetch_and_sub_4:
4818 case Builtin::BI__sync_fetch_and_sub_8:
4819 case Builtin::BI__sync_fetch_and_sub_16:
4820 return EmitBinaryAtomic(CGF&: *this, Kind: llvm::AtomicRMWInst::Sub, E);
4821 case Builtin::BI__sync_fetch_and_or_1:
4822 case Builtin::BI__sync_fetch_and_or_2:
4823 case Builtin::BI__sync_fetch_and_or_4:
4824 case Builtin::BI__sync_fetch_and_or_8:
4825 case Builtin::BI__sync_fetch_and_or_16:
4826 return EmitBinaryAtomic(CGF&: *this, Kind: llvm::AtomicRMWInst::Or, E);
4827 case Builtin::BI__sync_fetch_and_and_1:
4828 case Builtin::BI__sync_fetch_and_and_2:
4829 case Builtin::BI__sync_fetch_and_and_4:
4830 case Builtin::BI__sync_fetch_and_and_8:
4831 case Builtin::BI__sync_fetch_and_and_16:
4832 return EmitBinaryAtomic(CGF&: *this, Kind: llvm::AtomicRMWInst::And, E);
4833 case Builtin::BI__sync_fetch_and_xor_1:
4834 case Builtin::BI__sync_fetch_and_xor_2:
4835 case Builtin::BI__sync_fetch_and_xor_4:
4836 case Builtin::BI__sync_fetch_and_xor_8:
4837 case Builtin::BI__sync_fetch_and_xor_16:
4838 return EmitBinaryAtomic(CGF&: *this, Kind: llvm::AtomicRMWInst::Xor, E);
4839 case Builtin::BI__sync_fetch_and_nand_1:
4840 case Builtin::BI__sync_fetch_and_nand_2:
4841 case Builtin::BI__sync_fetch_and_nand_4:
4842 case Builtin::BI__sync_fetch_and_nand_8:
4843 case Builtin::BI__sync_fetch_and_nand_16:
4844 return EmitBinaryAtomic(CGF&: *this, Kind: llvm::AtomicRMWInst::Nand, E);
4845
4846 // Clang extensions: not overloaded yet.
4847 case Builtin::BI__sync_fetch_and_min:
4848 return EmitBinaryAtomic(CGF&: *this, Kind: llvm::AtomicRMWInst::Min, E);
4849 case Builtin::BI__sync_fetch_and_max:
4850 return EmitBinaryAtomic(CGF&: *this, Kind: llvm::AtomicRMWInst::Max, E);
4851 case Builtin::BI__sync_fetch_and_umin:
4852 return EmitBinaryAtomic(CGF&: *this, Kind: llvm::AtomicRMWInst::UMin, E);
4853 case Builtin::BI__sync_fetch_and_umax:
4854 return EmitBinaryAtomic(CGF&: *this, Kind: llvm::AtomicRMWInst::UMax, E);
4855
4856 case Builtin::BI__sync_add_and_fetch_1:
4857 case Builtin::BI__sync_add_and_fetch_2:
4858 case Builtin::BI__sync_add_and_fetch_4:
4859 case Builtin::BI__sync_add_and_fetch_8:
4860 case Builtin::BI__sync_add_and_fetch_16:
4861 return EmitBinaryAtomicPost(CGF&: *this, Kind: llvm::AtomicRMWInst::Add, E,
4862 Op: llvm::Instruction::Add);
4863 case Builtin::BI__sync_sub_and_fetch_1:
4864 case Builtin::BI__sync_sub_and_fetch_2:
4865 case Builtin::BI__sync_sub_and_fetch_4:
4866 case Builtin::BI__sync_sub_and_fetch_8:
4867 case Builtin::BI__sync_sub_and_fetch_16:
4868 return EmitBinaryAtomicPost(CGF&: *this, Kind: llvm::AtomicRMWInst::Sub, E,
4869 Op: llvm::Instruction::Sub);
4870 case Builtin::BI__sync_and_and_fetch_1:
4871 case Builtin::BI__sync_and_and_fetch_2:
4872 case Builtin::BI__sync_and_and_fetch_4:
4873 case Builtin::BI__sync_and_and_fetch_8:
4874 case Builtin::BI__sync_and_and_fetch_16:
4875 return EmitBinaryAtomicPost(CGF&: *this, Kind: llvm::AtomicRMWInst::And, E,
4876 Op: llvm::Instruction::And);
4877 case Builtin::BI__sync_or_and_fetch_1:
4878 case Builtin::BI__sync_or_and_fetch_2:
4879 case Builtin::BI__sync_or_and_fetch_4:
4880 case Builtin::BI__sync_or_and_fetch_8:
4881 case Builtin::BI__sync_or_and_fetch_16:
4882 return EmitBinaryAtomicPost(CGF&: *this, Kind: llvm::AtomicRMWInst::Or, E,
4883 Op: llvm::Instruction::Or);
4884 case Builtin::BI__sync_xor_and_fetch_1:
4885 case Builtin::BI__sync_xor_and_fetch_2:
4886 case Builtin::BI__sync_xor_and_fetch_4:
4887 case Builtin::BI__sync_xor_and_fetch_8:
4888 case Builtin::BI__sync_xor_and_fetch_16:
4889 return EmitBinaryAtomicPost(CGF&: *this, Kind: llvm::AtomicRMWInst::Xor, E,
4890 Op: llvm::Instruction::Xor);
4891 case Builtin::BI__sync_nand_and_fetch_1:
4892 case Builtin::BI__sync_nand_and_fetch_2:
4893 case Builtin::BI__sync_nand_and_fetch_4:
4894 case Builtin::BI__sync_nand_and_fetch_8:
4895 case Builtin::BI__sync_nand_and_fetch_16:
4896 return EmitBinaryAtomicPost(CGF&: *this, Kind: llvm::AtomicRMWInst::Nand, E,
4897 Op: llvm::Instruction::And, Invert: true);
4898
4899 case Builtin::BI__sync_val_compare_and_swap_1:
4900 case Builtin::BI__sync_val_compare_and_swap_2:
4901 case Builtin::BI__sync_val_compare_and_swap_4:
4902 case Builtin::BI__sync_val_compare_and_swap_8:
4903 case Builtin::BI__sync_val_compare_and_swap_16:
4904 return RValue::get(V: MakeAtomicCmpXchgValue(CGF&: *this, E, ReturnBool: false));
4905
4906 case Builtin::BI__sync_bool_compare_and_swap_1:
4907 case Builtin::BI__sync_bool_compare_and_swap_2:
4908 case Builtin::BI__sync_bool_compare_and_swap_4:
4909 case Builtin::BI__sync_bool_compare_and_swap_8:
4910 case Builtin::BI__sync_bool_compare_and_swap_16:
4911 return RValue::get(V: MakeAtomicCmpXchgValue(CGF&: *this, E, ReturnBool: true));
4912
4913 case Builtin::BI__sync_swap_1:
4914 case Builtin::BI__sync_swap_2:
4915 case Builtin::BI__sync_swap_4:
4916 case Builtin::BI__sync_swap_8:
4917 case Builtin::BI__sync_swap_16:
4918 return EmitBinaryAtomic(CGF&: *this, Kind: llvm::AtomicRMWInst::Xchg, E);
4919
4920 case Builtin::BI__sync_lock_test_and_set_1:
4921 case Builtin::BI__sync_lock_test_and_set_2:
4922 case Builtin::BI__sync_lock_test_and_set_4:
4923 case Builtin::BI__sync_lock_test_and_set_8:
4924 case Builtin::BI__sync_lock_test_and_set_16:
4925 return EmitBinaryAtomic(CGF&: *this, Kind: llvm::AtomicRMWInst::Xchg, E);
4926
4927 case Builtin::BI__sync_lock_release_1:
4928 case Builtin::BI__sync_lock_release_2:
4929 case Builtin::BI__sync_lock_release_4:
4930 case Builtin::BI__sync_lock_release_8:
4931 case Builtin::BI__sync_lock_release_16: {
4932 Address Ptr = CheckAtomicAlignment(CGF&: *this, E);
4933 QualType ElTy = E->getArg(Arg: 0)->getType()->getPointeeType();
4934
4935 llvm::Type *ITy = llvm::IntegerType::get(C&: getLLVMContext(),
4936 NumBits: getContext().getTypeSize(T: ElTy));
4937 llvm::StoreInst *Store =
4938 Builder.CreateStore(Val: llvm::Constant::getNullValue(Ty: ITy), Addr: Ptr);
4939 Store->setAtomic(Ordering: llvm::AtomicOrdering::Release);
4940 return RValue::get(V: nullptr);
4941 }
4942
4943 case Builtin::BI__sync_synchronize: {
4944 // We assume this is supposed to correspond to a C++0x-style
4945 // sequentially-consistent fence (i.e. this is only usable for
4946 // synchronization, not device I/O or anything like that). This intrinsic
4947 // is really badly designed in the sense that in theory, there isn't
4948 // any way to safely use it... but in practice, it mostly works
4949 // to use it with non-atomic loads and stores to get acquire/release
4950 // semantics.
4951 Builder.CreateFence(Ordering: llvm::AtomicOrdering::SequentiallyConsistent);
4952 return RValue::get(V: nullptr);
4953 }
4954
4955 case Builtin::BI__builtin_nontemporal_load:
4956 return RValue::get(V: EmitNontemporalLoad(CGF&: *this, E));
4957 case Builtin::BI__builtin_nontemporal_store:
4958 return RValue::get(V: EmitNontemporalStore(CGF&: *this, E));
4959 case Builtin::BI__c11_atomic_is_lock_free:
4960 case Builtin::BI__atomic_is_lock_free: {
4961 // Call "bool __atomic_is_lock_free(size_t size, void *ptr)". For the
4962 // __c11 builtin, ptr is 0 (indicating a properly-aligned object), since
4963 // _Atomic(T) is always properly-aligned.
4964 const char *LibCallName = "__atomic_is_lock_free";
4965 CallArgList Args;
4966 Args.add(rvalue: RValue::get(V: EmitScalarExpr(E: E->getArg(Arg: 0))),
4967 type: getContext().getSizeType());
4968 if (BuiltinID == Builtin::BI__atomic_is_lock_free)
4969 Args.add(rvalue: RValue::get(V: EmitScalarExpr(E: E->getArg(Arg: 1))),
4970 type: getContext().VoidPtrTy);
4971 else
4972 Args.add(rvalue: RValue::get(V: llvm::Constant::getNullValue(Ty: VoidPtrTy)),
4973 type: getContext().VoidPtrTy);
4974 const CGFunctionInfo &FuncInfo =
4975 CGM.getTypes().arrangeBuiltinFunctionCall(resultType: E->getType(), args: Args);
4976 llvm::FunctionType *FTy = CGM.getTypes().GetFunctionType(Info: FuncInfo);
4977 llvm::FunctionCallee Func = CGM.CreateRuntimeFunction(Ty: FTy, Name: LibCallName);
4978 return EmitCall(CallInfo: FuncInfo, Callee: CGCallee::forDirect(functionPtr: Func),
4979 ReturnValue: ReturnValueSlot(), Args);
4980 }
4981
4982 case Builtin::BI__atomic_thread_fence:
4983 case Builtin::BI__atomic_signal_fence:
4984 case Builtin::BI__c11_atomic_thread_fence:
4985 case Builtin::BI__c11_atomic_signal_fence: {
4986 llvm::SyncScope::ID SSID;
4987 if (BuiltinID == Builtin::BI__atomic_signal_fence ||
4988 BuiltinID == Builtin::BI__c11_atomic_signal_fence)
4989 SSID = llvm::SyncScope::SingleThread;
4990 else
4991 SSID = llvm::SyncScope::System;
4992 Value *Order = EmitScalarExpr(E: E->getArg(Arg: 0));
4993 if (isa<llvm::ConstantInt>(Val: Order)) {
4994 int ord = cast<llvm::ConstantInt>(Val: Order)->getZExtValue();
4995 switch (ord) {
4996 case 0: // memory_order_relaxed
4997 default: // invalid order
4998 break;
4999 case 1: // memory_order_consume
5000 case 2: // memory_order_acquire
5001 Builder.CreateFence(Ordering: llvm::AtomicOrdering::Acquire, SSID);
5002 break;
5003 case 3: // memory_order_release
5004 Builder.CreateFence(Ordering: llvm::AtomicOrdering::Release, SSID);
5005 break;
5006 case 4: // memory_order_acq_rel
5007 Builder.CreateFence(Ordering: llvm::AtomicOrdering::AcquireRelease, SSID);
5008 break;
5009 case 5: // memory_order_seq_cst
5010 Builder.CreateFence(Ordering: llvm::AtomicOrdering::SequentiallyConsistent, SSID);
5011 break;
5012 }
5013 return RValue::get(V: nullptr);
5014 }
5015
5016 llvm::BasicBlock *AcquireBB, *ReleaseBB, *AcqRelBB, *SeqCstBB;
5017 AcquireBB = createBasicBlock(name: "acquire", parent: CurFn);
5018 ReleaseBB = createBasicBlock(name: "release", parent: CurFn);
5019 AcqRelBB = createBasicBlock(name: "acqrel", parent: CurFn);
5020 SeqCstBB = createBasicBlock(name: "seqcst", parent: CurFn);
5021 llvm::BasicBlock *ContBB = createBasicBlock(name: "atomic.continue", parent: CurFn);
5022
5023 Order = Builder.CreateIntCast(V: Order, DestTy: Builder.getInt32Ty(), isSigned: false);
5024 llvm::SwitchInst *SI = Builder.CreateSwitch(V: Order, Dest: ContBB);
5025
5026 Builder.SetInsertPoint(AcquireBB);
5027 Builder.CreateFence(Ordering: llvm::AtomicOrdering::Acquire, SSID);
5028 Builder.CreateBr(Dest: ContBB);
5029 SI->addCase(OnVal: Builder.getInt32(C: 1), Dest: AcquireBB);
5030 SI->addCase(OnVal: Builder.getInt32(C: 2), Dest: AcquireBB);
5031
5032 Builder.SetInsertPoint(ReleaseBB);
5033 Builder.CreateFence(Ordering: llvm::AtomicOrdering::Release, SSID);
5034 Builder.CreateBr(Dest: ContBB);
5035 SI->addCase(OnVal: Builder.getInt32(C: 3), Dest: ReleaseBB);
5036
5037 Builder.SetInsertPoint(AcqRelBB);
5038 Builder.CreateFence(Ordering: llvm::AtomicOrdering::AcquireRelease, SSID);
5039 Builder.CreateBr(Dest: ContBB);
5040 SI->addCase(OnVal: Builder.getInt32(C: 4), Dest: AcqRelBB);
5041
5042 Builder.SetInsertPoint(SeqCstBB);
5043 Builder.CreateFence(Ordering: llvm::AtomicOrdering::SequentiallyConsistent, SSID);
5044 Builder.CreateBr(Dest: ContBB);
5045 SI->addCase(OnVal: Builder.getInt32(C: 5), Dest: SeqCstBB);
5046
5047 Builder.SetInsertPoint(ContBB);
5048 return RValue::get(V: nullptr);
5049 }
5050 case Builtin::BI__scoped_atomic_thread_fence: {
5051 auto ScopeModel = AtomicScopeModel::create(K: AtomicScopeModelKind::Generic);
5052
5053 Value *Order = EmitScalarExpr(E: E->getArg(Arg: 0));
5054 Value *Scope = EmitScalarExpr(E: E->getArg(Arg: 1));
5055 auto Ord = dyn_cast<llvm::ConstantInt>(Val: Order);
5056 auto Scp = dyn_cast<llvm::ConstantInt>(Val: Scope);
5057 if (Ord && Scp) {
5058 SyncScope SS = ScopeModel->isValid(S: Scp->getZExtValue())
5059 ? ScopeModel->map(S: Scp->getZExtValue())
5060 : ScopeModel->map(S: ScopeModel->getFallBackValue());
5061 switch (Ord->getZExtValue()) {
5062 case 0: // memory_order_relaxed
5063 default: // invalid order
5064 break;
5065 case 1: // memory_order_consume
5066 case 2: // memory_order_acquire
5067 Builder.CreateFence(
5068 Ordering: llvm::AtomicOrdering::Acquire,
5069 SSID: getTargetHooks().getLLVMSyncScopeID(LangOpts: getLangOpts(), Scope: SS,
5070 Ordering: llvm::AtomicOrdering::Acquire,
5071 Ctx&: getLLVMContext()));
5072 break;
5073 case 3: // memory_order_release
5074 Builder.CreateFence(
5075 Ordering: llvm::AtomicOrdering::Release,
5076 SSID: getTargetHooks().getLLVMSyncScopeID(LangOpts: getLangOpts(), Scope: SS,
5077 Ordering: llvm::AtomicOrdering::Release,
5078 Ctx&: getLLVMContext()));
5079 break;
5080 case 4: // memory_order_acq_rel
5081 Builder.CreateFence(Ordering: llvm::AtomicOrdering::AcquireRelease,
5082 SSID: getTargetHooks().getLLVMSyncScopeID(
5083 LangOpts: getLangOpts(), Scope: SS,
5084 Ordering: llvm::AtomicOrdering::AcquireRelease,
5085 Ctx&: getLLVMContext()));
5086 break;
5087 case 5: // memory_order_seq_cst
5088 Builder.CreateFence(Ordering: llvm::AtomicOrdering::SequentiallyConsistent,
5089 SSID: getTargetHooks().getLLVMSyncScopeID(
5090 LangOpts: getLangOpts(), Scope: SS,
5091 Ordering: llvm::AtomicOrdering::SequentiallyConsistent,
5092 Ctx&: getLLVMContext()));
5093 break;
5094 }
5095 return RValue::get(V: nullptr);
5096 }
5097
5098 llvm::BasicBlock *ContBB = createBasicBlock(name: "atomic.scope.continue", parent: CurFn);
5099
5100 llvm::SmallVector<std::pair<llvm::BasicBlock *, llvm::AtomicOrdering>>
5101 OrderBBs;
5102 if (Ord) {
5103 switch (Ord->getZExtValue()) {
5104 case 0: // memory_order_relaxed
5105 default: // invalid order
5106 ContBB->eraseFromParent();
5107 return RValue::get(V: nullptr);
5108 case 1: // memory_order_consume
5109 case 2: // memory_order_acquire
5110 OrderBBs.emplace_back(Args: Builder.GetInsertBlock(),
5111 Args: llvm::AtomicOrdering::Acquire);
5112 break;
5113 case 3: // memory_order_release
5114 OrderBBs.emplace_back(Args: Builder.GetInsertBlock(),
5115 Args: llvm::AtomicOrdering::Release);
5116 break;
5117 case 4: // memory_order_acq_rel
5118 OrderBBs.emplace_back(Args: Builder.GetInsertBlock(),
5119 Args: llvm::AtomicOrdering::AcquireRelease);
5120 break;
5121 case 5: // memory_order_seq_cst
5122 OrderBBs.emplace_back(Args: Builder.GetInsertBlock(),
5123 Args: llvm::AtomicOrdering::SequentiallyConsistent);
5124 break;
5125 }
5126 } else {
5127 llvm::BasicBlock *AcquireBB = createBasicBlock(name: "acquire", parent: CurFn);
5128 llvm::BasicBlock *ReleaseBB = createBasicBlock(name: "release", parent: CurFn);
5129 llvm::BasicBlock *AcqRelBB = createBasicBlock(name: "acqrel", parent: CurFn);
5130 llvm::BasicBlock *SeqCstBB = createBasicBlock(name: "seqcst", parent: CurFn);
5131
5132 Order = Builder.CreateIntCast(V: Order, DestTy: Builder.getInt32Ty(), isSigned: false);
5133 llvm::SwitchInst *SI = Builder.CreateSwitch(V: Order, Dest: ContBB);
5134 SI->addCase(OnVal: Builder.getInt32(C: 1), Dest: AcquireBB);
5135 SI->addCase(OnVal: Builder.getInt32(C: 2), Dest: AcquireBB);
5136 SI->addCase(OnVal: Builder.getInt32(C: 3), Dest: ReleaseBB);
5137 SI->addCase(OnVal: Builder.getInt32(C: 4), Dest: AcqRelBB);
5138 SI->addCase(OnVal: Builder.getInt32(C: 5), Dest: SeqCstBB);
5139
5140 OrderBBs.emplace_back(Args&: AcquireBB, Args: llvm::AtomicOrdering::Acquire);
5141 OrderBBs.emplace_back(Args&: ReleaseBB, Args: llvm::AtomicOrdering::Release);
5142 OrderBBs.emplace_back(Args&: AcqRelBB, Args: llvm::AtomicOrdering::AcquireRelease);
5143 OrderBBs.emplace_back(Args&: SeqCstBB,
5144 Args: llvm::AtomicOrdering::SequentiallyConsistent);
5145 }
5146
5147 for (auto &[OrderBB, Ordering] : OrderBBs) {
5148 Builder.SetInsertPoint(OrderBB);
5149 if (Scp) {
5150 SyncScope SS = ScopeModel->isValid(S: Scp->getZExtValue())
5151 ? ScopeModel->map(S: Scp->getZExtValue())
5152 : ScopeModel->map(S: ScopeModel->getFallBackValue());
5153 Builder.CreateFence(Ordering,
5154 SSID: getTargetHooks().getLLVMSyncScopeID(
5155 LangOpts: getLangOpts(), Scope: SS, Ordering, Ctx&: getLLVMContext()));
5156 Builder.CreateBr(Dest: ContBB);
5157 } else {
5158 llvm::DenseMap<unsigned, llvm::BasicBlock *> BBs;
5159 for (unsigned Scp : ScopeModel->getRuntimeValues())
5160 BBs[Scp] = createBasicBlock(name: getAsString(S: ScopeModel->map(S: Scp)), parent: CurFn);
5161
5162 auto *SC = Builder.CreateIntCast(V: Scope, DestTy: Builder.getInt32Ty(), isSigned: false);
5163 llvm::SwitchInst *SI = Builder.CreateSwitch(V: SC, Dest: ContBB);
5164 for (unsigned Scp : ScopeModel->getRuntimeValues()) {
5165 auto *B = BBs[Scp];
5166 SI->addCase(OnVal: Builder.getInt32(C: Scp), Dest: B);
5167
5168 Builder.SetInsertPoint(B);
5169 Builder.CreateFence(Ordering, SSID: getTargetHooks().getLLVMSyncScopeID(
5170 LangOpts: getLangOpts(), Scope: ScopeModel->map(S: Scp),
5171 Ordering, Ctx&: getLLVMContext()));
5172 Builder.CreateBr(Dest: ContBB);
5173 }
5174 }
5175 }
5176
5177 Builder.SetInsertPoint(ContBB);
5178 return RValue::get(V: nullptr);
5179 }
5180
5181 case Builtin::BI__builtin_signbit:
5182 case Builtin::BI__builtin_signbitf:
5183 case Builtin::BI__builtin_signbitl: {
5184 return RValue::get(
5185 Builder.CreateZExt(V: EmitSignBit(CGF&: *this, V: EmitScalarExpr(E: E->getArg(Arg: 0))),
5186 DestTy: ConvertType(E->getType())));
5187 }
5188 case Builtin::BI__warn_memset_zero_len:
5189 return RValue::getIgnored();
5190 case Builtin::BI__annotation: {
5191 // Re-encode each wide string to UTF8 and make an MDString.
5192 SmallVector<Metadata *, 1> Strings;
5193 for (const Expr *Arg : E->arguments()) {
5194 const auto *Str = cast<StringLiteral>(Val: Arg->IgnoreParenCasts());
5195 assert(Str->getCharByteWidth() == 2);
5196 StringRef WideBytes = Str->getBytes();
5197 std::string StrUtf8;
5198 if (!convertUTF16ToUTF8String(
5199 SrcBytes: ArrayRef(WideBytes.data(), WideBytes.size()), Out&: StrUtf8)) {
5200 CGM.ErrorUnsupported(E, "non-UTF16 __annotation argument");
5201 continue;
5202 }
5203 Strings.push_back(Elt: llvm::MDString::get(Context&: getLLVMContext(), Str: StrUtf8));
5204 }
5205
5206 // Build and MDTuple of MDStrings and emit the intrinsic call.
5207 llvm::Function *F = CGM.getIntrinsic(Intrinsic::codeview_annotation, {});
5208 MDTuple *StrTuple = MDTuple::get(Context&: getLLVMContext(), MDs: Strings);
5209 Builder.CreateCall(Callee: F, Args: MetadataAsValue::get(Context&: getLLVMContext(), MD: StrTuple));
5210 return RValue::getIgnored();
5211 }
5212 case Builtin::BI__builtin_annotation: {
5213 llvm::Value *AnnVal = EmitScalarExpr(E: E->getArg(Arg: 0));
5214 llvm::Function *F = CGM.getIntrinsic(
5215 Intrinsic::annotation, {AnnVal->getType(), CGM.ConstGlobalsPtrTy});
5216
5217 // Get the annotation string, go through casts. Sema requires this to be a
5218 // non-wide string literal, potentially casted, so the cast<> is safe.
5219 const Expr *AnnotationStrExpr = E->getArg(Arg: 1)->IgnoreParenCasts();
5220 StringRef Str = cast<StringLiteral>(Val: AnnotationStrExpr)->getString();
5221 return RValue::get(
5222 EmitAnnotationCall(AnnotationFn: F, AnnotatedVal: AnnVal, AnnotationStr: Str, Location: E->getExprLoc(), Attr: nullptr));
5223 }
5224 case Builtin::BI__builtin_addcb:
5225 case Builtin::BI__builtin_addcs:
5226 case Builtin::BI__builtin_addc:
5227 case Builtin::BI__builtin_addcl:
5228 case Builtin::BI__builtin_addcll:
5229 case Builtin::BI__builtin_subcb:
5230 case Builtin::BI__builtin_subcs:
5231 case Builtin::BI__builtin_subc:
5232 case Builtin::BI__builtin_subcl:
5233 case Builtin::BI__builtin_subcll: {
5234
5235 // We translate all of these builtins from expressions of the form:
5236 // int x = ..., y = ..., carryin = ..., carryout, result;
5237 // result = __builtin_addc(x, y, carryin, &carryout);
5238 //
5239 // to LLVM IR of the form:
5240 //
5241 // %tmp1 = call {i32, i1} @llvm.uadd.with.overflow.i32(i32 %x, i32 %y)
5242 // %tmpsum1 = extractvalue {i32, i1} %tmp1, 0
5243 // %carry1 = extractvalue {i32, i1} %tmp1, 1
5244 // %tmp2 = call {i32, i1} @llvm.uadd.with.overflow.i32(i32 %tmpsum1,
5245 // i32 %carryin)
5246 // %result = extractvalue {i32, i1} %tmp2, 0
5247 // %carry2 = extractvalue {i32, i1} %tmp2, 1
5248 // %tmp3 = or i1 %carry1, %carry2
5249 // %tmp4 = zext i1 %tmp3 to i32
5250 // store i32 %tmp4, i32* %carryout
5251
5252 // Scalarize our inputs.
5253 llvm::Value *X = EmitScalarExpr(E: E->getArg(Arg: 0));
5254 llvm::Value *Y = EmitScalarExpr(E: E->getArg(Arg: 1));
5255 llvm::Value *Carryin = EmitScalarExpr(E: E->getArg(Arg: 2));
5256 Address CarryOutPtr = EmitPointerWithAlignment(Addr: E->getArg(Arg: 3));
5257
5258 // Decide if we are lowering to a uadd.with.overflow or usub.with.overflow.
5259 Intrinsic::ID IntrinsicId;
5260 switch (BuiltinID) {
5261 default: llvm_unreachable("Unknown multiprecision builtin id.");
5262 case Builtin::BI__builtin_addcb:
5263 case Builtin::BI__builtin_addcs:
5264 case Builtin::BI__builtin_addc:
5265 case Builtin::BI__builtin_addcl:
5266 case Builtin::BI__builtin_addcll:
5267 IntrinsicId = Intrinsic::uadd_with_overflow;
5268 break;
5269 case Builtin::BI__builtin_subcb:
5270 case Builtin::BI__builtin_subcs:
5271 case Builtin::BI__builtin_subc:
5272 case Builtin::BI__builtin_subcl:
5273 case Builtin::BI__builtin_subcll:
5274 IntrinsicId = Intrinsic::usub_with_overflow;
5275 break;
5276 }
5277
5278 // Construct our resulting LLVM IR expression.
5279 llvm::Value *Carry1;
5280 llvm::Value *Sum1 = EmitOverflowIntrinsic(CGF&: *this, IntrinsicID: IntrinsicId,
5281 X, Y, Carry&: Carry1);
5282 llvm::Value *Carry2;
5283 llvm::Value *Sum2 = EmitOverflowIntrinsic(CGF&: *this, IntrinsicID: IntrinsicId,
5284 X: Sum1, Y: Carryin, Carry&: Carry2);
5285 llvm::Value *CarryOut = Builder.CreateZExt(V: Builder.CreateOr(LHS: Carry1, RHS: Carry2),
5286 DestTy: X->getType());
5287 Builder.CreateStore(Val: CarryOut, Addr: CarryOutPtr);
5288 return RValue::get(V: Sum2);
5289 }
5290
5291 case Builtin::BI__builtin_add_overflow:
5292 case Builtin::BI__builtin_sub_overflow:
5293 case Builtin::BI__builtin_mul_overflow: {
5294 const clang::Expr *LeftArg = E->getArg(Arg: 0);
5295 const clang::Expr *RightArg = E->getArg(Arg: 1);
5296 const clang::Expr *ResultArg = E->getArg(Arg: 2);
5297
5298 clang::QualType ResultQTy =
5299 ResultArg->getType()->castAs<PointerType>()->getPointeeType();
5300
5301 WidthAndSignedness LeftInfo =
5302 getIntegerWidthAndSignedness(context: CGM.getContext(), Type: LeftArg->getType());
5303 WidthAndSignedness RightInfo =
5304 getIntegerWidthAndSignedness(context: CGM.getContext(), Type: RightArg->getType());
5305 WidthAndSignedness ResultInfo =
5306 getIntegerWidthAndSignedness(context: CGM.getContext(), Type: ResultQTy);
5307
5308 // Handle mixed-sign multiplication as a special case, because adding
5309 // runtime or backend support for our generic irgen would be too expensive.
5310 if (isSpecialMixedSignMultiply(BuiltinID, Op1Info: LeftInfo, Op2Info: RightInfo, ResultInfo))
5311 return EmitCheckedMixedSignMultiply(CGF&: *this, Op1: LeftArg, Op1Info: LeftInfo, Op2: RightArg,
5312 Op2Info: RightInfo, ResultArg, ResultQTy,
5313 ResultInfo);
5314
5315 if (isSpecialUnsignedMultiplySignedResult(BuiltinID, Op1Info: LeftInfo, Op2Info: RightInfo,
5316 ResultInfo))
5317 return EmitCheckedUnsignedMultiplySignedResult(
5318 CGF&: *this, Op1: LeftArg, Op1Info: LeftInfo, Op2: RightArg, Op2Info: RightInfo, ResultArg, ResultQTy,
5319 ResultInfo);
5320
5321 WidthAndSignedness EncompassingInfo =
5322 EncompassingIntegerType(Types: {LeftInfo, RightInfo, ResultInfo});
5323
5324 llvm::Type *EncompassingLLVMTy =
5325 llvm::IntegerType::get(C&: CGM.getLLVMContext(), NumBits: EncompassingInfo.Width);
5326
5327 llvm::Type *ResultLLVMTy = CGM.getTypes().ConvertType(T: ResultQTy);
5328
5329 Intrinsic::ID IntrinsicId;
5330 switch (BuiltinID) {
5331 default:
5332 llvm_unreachable("Unknown overflow builtin id.");
5333 case Builtin::BI__builtin_add_overflow:
5334 IntrinsicId = EncompassingInfo.Signed ? Intrinsic::sadd_with_overflow
5335 : Intrinsic::uadd_with_overflow;
5336 break;
5337 case Builtin::BI__builtin_sub_overflow:
5338 IntrinsicId = EncompassingInfo.Signed ? Intrinsic::ssub_with_overflow
5339 : Intrinsic::usub_with_overflow;
5340 break;
5341 case Builtin::BI__builtin_mul_overflow:
5342 IntrinsicId = EncompassingInfo.Signed ? Intrinsic::smul_with_overflow
5343 : Intrinsic::umul_with_overflow;
5344 break;
5345 }
5346
5347 llvm::Value *Left = EmitScalarExpr(E: LeftArg);
5348 llvm::Value *Right = EmitScalarExpr(E: RightArg);
5349 Address ResultPtr = EmitPointerWithAlignment(Addr: ResultArg);
5350
5351 // Extend each operand to the encompassing type.
5352 Left = Builder.CreateIntCast(V: Left, DestTy: EncompassingLLVMTy, isSigned: LeftInfo.Signed);
5353 Right = Builder.CreateIntCast(V: Right, DestTy: EncompassingLLVMTy, isSigned: RightInfo.Signed);
5354
5355 // Perform the operation on the extended values.
5356 llvm::Value *Overflow, *Result;
5357 Result = EmitOverflowIntrinsic(CGF&: *this, IntrinsicID: IntrinsicId, X: Left, Y: Right, Carry&: Overflow);
5358
5359 if (EncompassingInfo.Width > ResultInfo.Width) {
5360 // The encompassing type is wider than the result type, so we need to
5361 // truncate it.
5362 llvm::Value *ResultTrunc = Builder.CreateTrunc(V: Result, DestTy: ResultLLVMTy);
5363
5364 // To see if the truncation caused an overflow, we will extend
5365 // the result and then compare it to the original result.
5366 llvm::Value *ResultTruncExt = Builder.CreateIntCast(
5367 V: ResultTrunc, DestTy: EncompassingLLVMTy, isSigned: ResultInfo.Signed);
5368 llvm::Value *TruncationOverflow =
5369 Builder.CreateICmpNE(LHS: Result, RHS: ResultTruncExt);
5370
5371 Overflow = Builder.CreateOr(LHS: Overflow, RHS: TruncationOverflow);
5372 Result = ResultTrunc;
5373 }
5374
5375 // Finally, store the result using the pointer.
5376 bool isVolatile =
5377 ResultArg->getType()->getPointeeType().isVolatileQualified();
5378 Builder.CreateStore(Val: EmitToMemory(Value: Result, Ty: ResultQTy), Addr: ResultPtr, IsVolatile: isVolatile);
5379
5380 return RValue::get(V: Overflow);
5381 }
5382
5383 case Builtin::BI__builtin_uadd_overflow:
5384 case Builtin::BI__builtin_uaddl_overflow:
5385 case Builtin::BI__builtin_uaddll_overflow:
5386 case Builtin::BI__builtin_usub_overflow:
5387 case Builtin::BI__builtin_usubl_overflow:
5388 case Builtin::BI__builtin_usubll_overflow:
5389 case Builtin::BI__builtin_umul_overflow:
5390 case Builtin::BI__builtin_umull_overflow:
5391 case Builtin::BI__builtin_umulll_overflow:
5392 case Builtin::BI__builtin_sadd_overflow:
5393 case Builtin::BI__builtin_saddl_overflow:
5394 case Builtin::BI__builtin_saddll_overflow:
5395 case Builtin::BI__builtin_ssub_overflow:
5396 case Builtin::BI__builtin_ssubl_overflow:
5397 case Builtin::BI__builtin_ssubll_overflow:
5398 case Builtin::BI__builtin_smul_overflow:
5399 case Builtin::BI__builtin_smull_overflow:
5400 case Builtin::BI__builtin_smulll_overflow: {
5401
5402 // We translate all of these builtins directly to the relevant llvm IR node.
5403
5404 // Scalarize our inputs.
5405 llvm::Value *X = EmitScalarExpr(E: E->getArg(Arg: 0));
5406 llvm::Value *Y = EmitScalarExpr(E: E->getArg(Arg: 1));
5407 Address SumOutPtr = EmitPointerWithAlignment(Addr: E->getArg(Arg: 2));
5408
5409 // Decide which of the overflow intrinsics we are lowering to:
5410 Intrinsic::ID IntrinsicId;
5411 switch (BuiltinID) {
5412 default: llvm_unreachable("Unknown overflow builtin id.");
5413 case Builtin::BI__builtin_uadd_overflow:
5414 case Builtin::BI__builtin_uaddl_overflow:
5415 case Builtin::BI__builtin_uaddll_overflow:
5416 IntrinsicId = Intrinsic::uadd_with_overflow;
5417 break;
5418 case Builtin::BI__builtin_usub_overflow:
5419 case Builtin::BI__builtin_usubl_overflow:
5420 case Builtin::BI__builtin_usubll_overflow:
5421 IntrinsicId = Intrinsic::usub_with_overflow;
5422 break;
5423 case Builtin::BI__builtin_umul_overflow:
5424 case Builtin::BI__builtin_umull_overflow:
5425 case Builtin::BI__builtin_umulll_overflow:
5426 IntrinsicId = Intrinsic::umul_with_overflow;
5427 break;
5428 case Builtin::BI__builtin_sadd_overflow:
5429 case Builtin::BI__builtin_saddl_overflow:
5430 case Builtin::BI__builtin_saddll_overflow:
5431 IntrinsicId = Intrinsic::sadd_with_overflow;
5432 break;
5433 case Builtin::BI__builtin_ssub_overflow:
5434 case Builtin::BI__builtin_ssubl_overflow:
5435 case Builtin::BI__builtin_ssubll_overflow:
5436 IntrinsicId = Intrinsic::ssub_with_overflow;
5437 break;
5438 case Builtin::BI__builtin_smul_overflow:
5439 case Builtin::BI__builtin_smull_overflow:
5440 case Builtin::BI__builtin_smulll_overflow:
5441 IntrinsicId = Intrinsic::smul_with_overflow;
5442 break;
5443 }
5444
5445
5446 llvm::Value *Carry;
5447 llvm::Value *Sum = EmitOverflowIntrinsic(CGF&: *this, IntrinsicID: IntrinsicId, X, Y, Carry);
5448 Builder.CreateStore(Val: Sum, Addr: SumOutPtr);
5449
5450 return RValue::get(V: Carry);
5451 }
5452 case Builtin::BIaddressof:
5453 case Builtin::BI__addressof:
5454 case Builtin::BI__builtin_addressof:
5455 return RValue::get(V: EmitLValue(E: E->getArg(Arg: 0)).getPointer(CGF&: *this));
5456 case Builtin::BI__builtin_function_start:
5457 return RValue::get(V: CGM.GetFunctionStart(
5458 Decl: E->getArg(Arg: 0)->getAsBuiltinConstantDeclRef(Context: CGM.getContext())));
5459 case Builtin::BI__builtin_operator_new:
5460 return EmitBuiltinNewDeleteCall(
5461 Type: E->getCallee()->getType()->castAs<FunctionProtoType>(), TheCallExpr: E, IsDelete: false);
5462 case Builtin::BI__builtin_operator_delete:
5463 EmitBuiltinNewDeleteCall(
5464 Type: E->getCallee()->getType()->castAs<FunctionProtoType>(), TheCallExpr: E, IsDelete: true);
5465 return RValue::get(V: nullptr);
5466
5467 case Builtin::BI__builtin_is_aligned:
5468 return EmitBuiltinIsAligned(E);
5469 case Builtin::BI__builtin_align_up:
5470 return EmitBuiltinAlignTo(E, AlignUp: true);
5471 case Builtin::BI__builtin_align_down:
5472 return EmitBuiltinAlignTo(E, AlignUp: false);
5473
5474 case Builtin::BI__noop:
5475 // __noop always evaluates to an integer literal zero.
5476 return RValue::get(V: ConstantInt::get(Ty: IntTy, V: 0));
5477 case Builtin::BI__builtin_call_with_static_chain: {
5478 const CallExpr *Call = cast<CallExpr>(Val: E->getArg(Arg: 0));
5479 const Expr *Chain = E->getArg(Arg: 1);
5480 return EmitCall(FnType: Call->getCallee()->getType(),
5481 Callee: EmitCallee(E: Call->getCallee()), E: Call, ReturnValue,
5482 Chain: EmitScalarExpr(E: Chain));
5483 }
5484 case Builtin::BI_InterlockedExchange8:
5485 case Builtin::BI_InterlockedExchange16:
5486 case Builtin::BI_InterlockedExchange:
5487 case Builtin::BI_InterlockedExchangePointer:
5488 return RValue::get(
5489 V: EmitMSVCBuiltinExpr(BuiltinID: MSVCIntrin::_InterlockedExchange, E));
5490 case Builtin::BI_InterlockedCompareExchangePointer:
5491 return RValue::get(
5492 V: EmitMSVCBuiltinExpr(BuiltinID: MSVCIntrin::_InterlockedCompareExchange, E));
5493 case Builtin::BI_InterlockedCompareExchangePointer_nf:
5494 return RValue::get(
5495 V: EmitMSVCBuiltinExpr(BuiltinID: MSVCIntrin::_InterlockedCompareExchange_nf, E));
5496 case Builtin::BI_InterlockedCompareExchange8:
5497 case Builtin::BI_InterlockedCompareExchange16:
5498 case Builtin::BI_InterlockedCompareExchange:
5499 case Builtin::BI_InterlockedCompareExchange64:
5500 return RValue::get(V: EmitAtomicCmpXchgForMSIntrin(CGF&: *this, E));
5501 case Builtin::BI_InterlockedIncrement16:
5502 case Builtin::BI_InterlockedIncrement:
5503 return RValue::get(
5504 V: EmitMSVCBuiltinExpr(BuiltinID: MSVCIntrin::_InterlockedIncrement, E));
5505 case Builtin::BI_InterlockedDecrement16:
5506 case Builtin::BI_InterlockedDecrement:
5507 return RValue::get(
5508 V: EmitMSVCBuiltinExpr(BuiltinID: MSVCIntrin::_InterlockedDecrement, E));
5509 case Builtin::BI_InterlockedAnd8:
5510 case Builtin::BI_InterlockedAnd16:
5511 case Builtin::BI_InterlockedAnd:
5512 return RValue::get(V: EmitMSVCBuiltinExpr(BuiltinID: MSVCIntrin::_InterlockedAnd, E));
5513 case Builtin::BI_InterlockedExchangeAdd8:
5514 case Builtin::BI_InterlockedExchangeAdd16:
5515 case Builtin::BI_InterlockedExchangeAdd:
5516 return RValue::get(
5517 V: EmitMSVCBuiltinExpr(BuiltinID: MSVCIntrin::_InterlockedExchangeAdd, E));
5518 case Builtin::BI_InterlockedExchangeSub8:
5519 case Builtin::BI_InterlockedExchangeSub16:
5520 case Builtin::BI_InterlockedExchangeSub:
5521 return RValue::get(
5522 V: EmitMSVCBuiltinExpr(BuiltinID: MSVCIntrin::_InterlockedExchangeSub, E));
5523 case Builtin::BI_InterlockedOr8:
5524 case Builtin::BI_InterlockedOr16:
5525 case Builtin::BI_InterlockedOr:
5526 return RValue::get(V: EmitMSVCBuiltinExpr(BuiltinID: MSVCIntrin::_InterlockedOr, E));
5527 case Builtin::BI_InterlockedXor8:
5528 case Builtin::BI_InterlockedXor16:
5529 case Builtin::BI_InterlockedXor:
5530 return RValue::get(V: EmitMSVCBuiltinExpr(BuiltinID: MSVCIntrin::_InterlockedXor, E));
5531
5532 case Builtin::BI_bittest64:
5533 case Builtin::BI_bittest:
5534 case Builtin::BI_bittestandcomplement64:
5535 case Builtin::BI_bittestandcomplement:
5536 case Builtin::BI_bittestandreset64:
5537 case Builtin::BI_bittestandreset:
5538 case Builtin::BI_bittestandset64:
5539 case Builtin::BI_bittestandset:
5540 case Builtin::BI_interlockedbittestandreset:
5541 case Builtin::BI_interlockedbittestandreset64:
5542 case Builtin::BI_interlockedbittestandset64:
5543 case Builtin::BI_interlockedbittestandset:
5544 case Builtin::BI_interlockedbittestandset_acq:
5545 case Builtin::BI_interlockedbittestandset_rel:
5546 case Builtin::BI_interlockedbittestandset_nf:
5547 case Builtin::BI_interlockedbittestandreset_acq:
5548 case Builtin::BI_interlockedbittestandreset_rel:
5549 case Builtin::BI_interlockedbittestandreset_nf:
5550 return RValue::get(V: EmitBitTestIntrinsic(CGF&: *this, BuiltinID, E));
5551
5552 // These builtins exist to emit regular volatile loads and stores not
5553 // affected by the -fms-volatile setting.
5554 case Builtin::BI__iso_volatile_load8:
5555 case Builtin::BI__iso_volatile_load16:
5556 case Builtin::BI__iso_volatile_load32:
5557 case Builtin::BI__iso_volatile_load64:
5558 return RValue::get(V: EmitISOVolatileLoad(CGF&: *this, E));
5559 case Builtin::BI__iso_volatile_store8:
5560 case Builtin::BI__iso_volatile_store16:
5561 case Builtin::BI__iso_volatile_store32:
5562 case Builtin::BI__iso_volatile_store64:
5563 return RValue::get(V: EmitISOVolatileStore(CGF&: *this, E));
5564
5565 case Builtin::BI__builtin_ptrauth_sign_constant:
5566 return RValue::get(ConstantEmitter(*this).emitAbstract(E, E->getType()));
5567
5568 case Builtin::BI__builtin_ptrauth_auth:
5569 case Builtin::BI__builtin_ptrauth_auth_and_resign:
5570 case Builtin::BI__builtin_ptrauth_blend_discriminator:
5571 case Builtin::BI__builtin_ptrauth_sign_generic_data:
5572 case Builtin::BI__builtin_ptrauth_sign_unauthenticated:
5573 case Builtin::BI__builtin_ptrauth_strip: {
5574 // Emit the arguments.
5575 SmallVector<llvm::Value *, 5> Args;
5576 for (auto argExpr : E->arguments())
5577 Args.push_back(Elt: EmitScalarExpr(argExpr));
5578
5579 // Cast the value to intptr_t, saving its original type.
5580 llvm::Type *OrigValueType = Args[0]->getType();
5581 if (OrigValueType->isPointerTy())
5582 Args[0] = Builder.CreatePtrToInt(V: Args[0], DestTy: IntPtrTy);
5583
5584 switch (BuiltinID) {
5585 case Builtin::BI__builtin_ptrauth_auth_and_resign:
5586 if (Args[4]->getType()->isPointerTy())
5587 Args[4] = Builder.CreatePtrToInt(V: Args[4], DestTy: IntPtrTy);
5588 [[fallthrough]];
5589
5590 case Builtin::BI__builtin_ptrauth_auth:
5591 case Builtin::BI__builtin_ptrauth_sign_unauthenticated:
5592 if (Args[2]->getType()->isPointerTy())
5593 Args[2] = Builder.CreatePtrToInt(V: Args[2], DestTy: IntPtrTy);
5594 break;
5595
5596 case Builtin::BI__builtin_ptrauth_sign_generic_data:
5597 if (Args[1]->getType()->isPointerTy())
5598 Args[1] = Builder.CreatePtrToInt(V: Args[1], DestTy: IntPtrTy);
5599 break;
5600
5601 case Builtin::BI__builtin_ptrauth_blend_discriminator:
5602 case Builtin::BI__builtin_ptrauth_strip:
5603 break;
5604 }
5605
5606 // Call the intrinsic.
5607 auto IntrinsicID = [&]() -> unsigned {
5608 switch (BuiltinID) {
5609 case Builtin::BI__builtin_ptrauth_auth:
5610 return Intrinsic::ptrauth_auth;
5611 case Builtin::BI__builtin_ptrauth_auth_and_resign:
5612 return Intrinsic::ptrauth_resign;
5613 case Builtin::BI__builtin_ptrauth_blend_discriminator:
5614 return Intrinsic::ptrauth_blend;
5615 case Builtin::BI__builtin_ptrauth_sign_generic_data:
5616 return Intrinsic::ptrauth_sign_generic;
5617 case Builtin::BI__builtin_ptrauth_sign_unauthenticated:
5618 return Intrinsic::ptrauth_sign;
5619 case Builtin::BI__builtin_ptrauth_strip:
5620 return Intrinsic::ptrauth_strip;
5621 }
5622 llvm_unreachable("bad ptrauth intrinsic");
5623 }();
5624 auto Intrinsic = CGM.getIntrinsic(IID: IntrinsicID);
5625 llvm::Value *Result = EmitRuntimeCall(callee: Intrinsic, args: Args);
5626
5627 if (BuiltinID != Builtin::BI__builtin_ptrauth_sign_generic_data &&
5628 BuiltinID != Builtin::BI__builtin_ptrauth_blend_discriminator &&
5629 OrigValueType->isPointerTy()) {
5630 Result = Builder.CreateIntToPtr(V: Result, DestTy: OrigValueType);
5631 }
5632 return RValue::get(V: Result);
5633 }
5634
5635 case Builtin::BI__builtin_get_vtable_pointer: {
5636 const Expr *Target = E->getArg(Arg: 0);
5637 QualType TargetType = Target->getType();
5638 const CXXRecordDecl *Decl = TargetType->getPointeeCXXRecordDecl();
5639 assert(Decl);
5640 auto ThisAddress = EmitPointerWithAlignment(Addr: Target);
5641 assert(ThisAddress.isValid());
5642 llvm::Value *VTablePointer =
5643 GetVTablePtr(This: ThisAddress, VTableTy: Int8PtrTy, VTableClass: Decl, AuthMode: VTableAuthMode::MustTrap);
5644 return RValue::get(V: VTablePointer);
5645 }
5646
5647 case Builtin::BI__exception_code:
5648 case Builtin::BI_exception_code:
5649 return RValue::get(V: EmitSEHExceptionCode());
5650 case Builtin::BI__exception_info:
5651 case Builtin::BI_exception_info:
5652 return RValue::get(V: EmitSEHExceptionInfo());
5653 case Builtin::BI__abnormal_termination:
5654 case Builtin::BI_abnormal_termination:
5655 return RValue::get(V: EmitSEHAbnormalTermination());
5656 case Builtin::BI_setjmpex:
5657 if (getTarget().getTriple().isOSMSVCRT() && E->getNumArgs() == 1 &&
5658 E->getArg(Arg: 0)->getType()->isPointerType())
5659 return EmitMSVCRTSetJmp(CGF&: *this, SJKind: MSVCSetJmpKind::_setjmpex, E);
5660 break;
5661 case Builtin::BI_setjmp:
5662 if (getTarget().getTriple().isOSMSVCRT() && E->getNumArgs() == 1 &&
5663 E->getArg(Arg: 0)->getType()->isPointerType()) {
5664 if (getTarget().getTriple().getArch() == llvm::Triple::x86)
5665 return EmitMSVCRTSetJmp(CGF&: *this, SJKind: MSVCSetJmpKind::_setjmp3, E);
5666 else if (getTarget().getTriple().getArch() == llvm::Triple::aarch64)
5667 return EmitMSVCRTSetJmp(CGF&: *this, SJKind: MSVCSetJmpKind::_setjmpex, E);
5668 return EmitMSVCRTSetJmp(CGF&: *this, SJKind: MSVCSetJmpKind::_setjmp, E);
5669 }
5670 break;
5671
5672 // C++ std:: builtins.
5673 case Builtin::BImove:
5674 case Builtin::BImove_if_noexcept:
5675 case Builtin::BIforward:
5676 case Builtin::BIforward_like:
5677 case Builtin::BIas_const:
5678 return RValue::get(V: EmitLValue(E: E->getArg(Arg: 0)).getPointer(CGF&: *this));
5679 case Builtin::BI__GetExceptionInfo: {
5680 if (llvm::GlobalVariable *GV =
5681 CGM.getCXXABI().getThrowInfo(T: FD->getParamDecl(i: 0)->getType()))
5682 return RValue::get(V: GV);
5683 break;
5684 }
5685
5686 case Builtin::BI__fastfail:
5687 return RValue::get(V: EmitMSVCBuiltinExpr(BuiltinID: MSVCIntrin::__fastfail, E));
5688
5689 case Builtin::BI__builtin_coro_id:
5690 return EmitCoroutineIntrinsic(E, Intrinsic::coro_id);
5691 case Builtin::BI__builtin_coro_promise:
5692 return EmitCoroutineIntrinsic(E, Intrinsic::coro_promise);
5693 case Builtin::BI__builtin_coro_resume:
5694 EmitCoroutineIntrinsic(E, Intrinsic::coro_resume);
5695 return RValue::get(V: nullptr);
5696 case Builtin::BI__builtin_coro_frame:
5697 return EmitCoroutineIntrinsic(E, Intrinsic::coro_frame);
5698 case Builtin::BI__builtin_coro_noop:
5699 return EmitCoroutineIntrinsic(E, Intrinsic::coro_noop);
5700 case Builtin::BI__builtin_coro_free:
5701 return EmitCoroutineIntrinsic(E, Intrinsic::coro_free);
5702 case Builtin::BI__builtin_coro_destroy:
5703 EmitCoroutineIntrinsic(E, Intrinsic::coro_destroy);
5704 return RValue::get(V: nullptr);
5705 case Builtin::BI__builtin_coro_done:
5706 return EmitCoroutineIntrinsic(E, Intrinsic::coro_done);
5707 case Builtin::BI__builtin_coro_alloc:
5708 return EmitCoroutineIntrinsic(E, Intrinsic::coro_alloc);
5709 case Builtin::BI__builtin_coro_begin:
5710 return EmitCoroutineIntrinsic(E, Intrinsic::coro_begin);
5711 case Builtin::BI__builtin_coro_end:
5712 return EmitCoroutineIntrinsic(E, Intrinsic::coro_end);
5713 case Builtin::BI__builtin_coro_suspend:
5714 return EmitCoroutineIntrinsic(E, Intrinsic::coro_suspend);
5715 case Builtin::BI__builtin_coro_size:
5716 return EmitCoroutineIntrinsic(E, Intrinsic::coro_size);
5717 case Builtin::BI__builtin_coro_align:
5718 return EmitCoroutineIntrinsic(E, Intrinsic::coro_align);
5719
5720 // OpenCL v2.0 s6.13.16.2, Built-in pipe read and write functions
5721 case Builtin::BIread_pipe:
5722 case Builtin::BIwrite_pipe: {
5723 Value *Arg0 = EmitScalarExpr(E: E->getArg(Arg: 0)),
5724 *Arg1 = EmitScalarExpr(E: E->getArg(Arg: 1));
5725 CGOpenCLRuntime OpenCLRT(CGM);
5726 Value *PacketSize = OpenCLRT.getPipeElemSize(PipeArg: E->getArg(Arg: 0));
5727 Value *PacketAlign = OpenCLRT.getPipeElemAlign(PipeArg: E->getArg(Arg: 0));
5728
5729 // Type of the generic packet parameter.
5730 unsigned GenericAS =
5731 getContext().getTargetAddressSpace(AS: LangAS::opencl_generic);
5732 llvm::Type *I8PTy = llvm::PointerType::get(C&: getLLVMContext(), AddressSpace: GenericAS);
5733
5734 // Testing which overloaded version we should generate the call for.
5735 if (2U == E->getNumArgs()) {
5736 const char *Name = (BuiltinID == Builtin::BIread_pipe) ? "__read_pipe_2"
5737 : "__write_pipe_2";
5738 // Creating a generic function type to be able to call with any builtin or
5739 // user defined type.
5740 llvm::Type *ArgTys[] = {Arg0->getType(), I8PTy, Int32Ty, Int32Ty};
5741 llvm::FunctionType *FTy = llvm::FunctionType::get(Result: Int32Ty, Params: ArgTys, isVarArg: false);
5742 Value *ACast = Builder.CreateAddrSpaceCast(V: Arg1, DestTy: I8PTy);
5743 return RValue::get(
5744 V: EmitRuntimeCall(callee: CGM.CreateRuntimeFunction(Ty: FTy, Name),
5745 args: {Arg0, ACast, PacketSize, PacketAlign}));
5746 } else {
5747 assert(4 == E->getNumArgs() &&
5748 "Illegal number of parameters to pipe function");
5749 const char *Name = (BuiltinID == Builtin::BIread_pipe) ? "__read_pipe_4"
5750 : "__write_pipe_4";
5751
5752 llvm::Type *ArgTys[] = {Arg0->getType(), Arg1->getType(), Int32Ty, I8PTy,
5753 Int32Ty, Int32Ty};
5754 Value *Arg2 = EmitScalarExpr(E: E->getArg(Arg: 2)),
5755 *Arg3 = EmitScalarExpr(E: E->getArg(Arg: 3));
5756 llvm::FunctionType *FTy = llvm::FunctionType::get(Result: Int32Ty, Params: ArgTys, isVarArg: false);
5757 Value *ACast = Builder.CreateAddrSpaceCast(V: Arg3, DestTy: I8PTy);
5758 // We know the third argument is an integer type, but we may need to cast
5759 // it to i32.
5760 if (Arg2->getType() != Int32Ty)
5761 Arg2 = Builder.CreateZExtOrTrunc(V: Arg2, DestTy: Int32Ty);
5762 return RValue::get(
5763 V: EmitRuntimeCall(callee: CGM.CreateRuntimeFunction(Ty: FTy, Name),
5764 args: {Arg0, Arg1, Arg2, ACast, PacketSize, PacketAlign}));
5765 }
5766 }
5767 // OpenCL v2.0 s6.13.16 ,s9.17.3.5 - Built-in pipe reserve read and write
5768 // functions
5769 case Builtin::BIreserve_read_pipe:
5770 case Builtin::BIreserve_write_pipe:
5771 case Builtin::BIwork_group_reserve_read_pipe:
5772 case Builtin::BIwork_group_reserve_write_pipe:
5773 case Builtin::BIsub_group_reserve_read_pipe:
5774 case Builtin::BIsub_group_reserve_write_pipe: {
5775 // Composing the mangled name for the function.
5776 const char *Name;
5777 if (BuiltinID == Builtin::BIreserve_read_pipe)
5778 Name = "__reserve_read_pipe";
5779 else if (BuiltinID == Builtin::BIreserve_write_pipe)
5780 Name = "__reserve_write_pipe";
5781 else if (BuiltinID == Builtin::BIwork_group_reserve_read_pipe)
5782 Name = "__work_group_reserve_read_pipe";
5783 else if (BuiltinID == Builtin::BIwork_group_reserve_write_pipe)
5784 Name = "__work_group_reserve_write_pipe";
5785 else if (BuiltinID == Builtin::BIsub_group_reserve_read_pipe)
5786 Name = "__sub_group_reserve_read_pipe";
5787 else
5788 Name = "__sub_group_reserve_write_pipe";
5789
5790 Value *Arg0 = EmitScalarExpr(E: E->getArg(Arg: 0)),
5791 *Arg1 = EmitScalarExpr(E: E->getArg(Arg: 1));
5792 llvm::Type *ReservedIDTy = ConvertType(getContext().OCLReserveIDTy);
5793 CGOpenCLRuntime OpenCLRT(CGM);
5794 Value *PacketSize = OpenCLRT.getPipeElemSize(PipeArg: E->getArg(Arg: 0));
5795 Value *PacketAlign = OpenCLRT.getPipeElemAlign(PipeArg: E->getArg(Arg: 0));
5796
5797 // Building the generic function prototype.
5798 llvm::Type *ArgTys[] = {Arg0->getType(), Int32Ty, Int32Ty, Int32Ty};
5799 llvm::FunctionType *FTy =
5800 llvm::FunctionType::get(Result: ReservedIDTy, Params: ArgTys, isVarArg: false);
5801 // We know the second argument is an integer type, but we may need to cast
5802 // it to i32.
5803 if (Arg1->getType() != Int32Ty)
5804 Arg1 = Builder.CreateZExtOrTrunc(V: Arg1, DestTy: Int32Ty);
5805 return RValue::get(V: EmitRuntimeCall(callee: CGM.CreateRuntimeFunction(Ty: FTy, Name),
5806 args: {Arg0, Arg1, PacketSize, PacketAlign}));
5807 }
5808 // OpenCL v2.0 s6.13.16, s9.17.3.5 - Built-in pipe commit read and write
5809 // functions
5810 case Builtin::BIcommit_read_pipe:
5811 case Builtin::BIcommit_write_pipe:
5812 case Builtin::BIwork_group_commit_read_pipe:
5813 case Builtin::BIwork_group_commit_write_pipe:
5814 case Builtin::BIsub_group_commit_read_pipe:
5815 case Builtin::BIsub_group_commit_write_pipe: {
5816 const char *Name;
5817 if (BuiltinID == Builtin::BIcommit_read_pipe)
5818 Name = "__commit_read_pipe";
5819 else if (BuiltinID == Builtin::BIcommit_write_pipe)
5820 Name = "__commit_write_pipe";
5821 else if (BuiltinID == Builtin::BIwork_group_commit_read_pipe)
5822 Name = "__work_group_commit_read_pipe";
5823 else if (BuiltinID == Builtin::BIwork_group_commit_write_pipe)
5824 Name = "__work_group_commit_write_pipe";
5825 else if (BuiltinID == Builtin::BIsub_group_commit_read_pipe)
5826 Name = "__sub_group_commit_read_pipe";
5827 else
5828 Name = "__sub_group_commit_write_pipe";
5829
5830 Value *Arg0 = EmitScalarExpr(E: E->getArg(Arg: 0)),
5831 *Arg1 = EmitScalarExpr(E: E->getArg(Arg: 1));
5832 CGOpenCLRuntime OpenCLRT(CGM);
5833 Value *PacketSize = OpenCLRT.getPipeElemSize(PipeArg: E->getArg(Arg: 0));
5834 Value *PacketAlign = OpenCLRT.getPipeElemAlign(PipeArg: E->getArg(Arg: 0));
5835
5836 // Building the generic function prototype.
5837 llvm::Type *ArgTys[] = {Arg0->getType(), Arg1->getType(), Int32Ty, Int32Ty};
5838 llvm::FunctionType *FTy = llvm::FunctionType::get(
5839 Result: llvm::Type::getVoidTy(C&: getLLVMContext()), Params: ArgTys, isVarArg: false);
5840
5841 return RValue::get(V: EmitRuntimeCall(callee: CGM.CreateRuntimeFunction(Ty: FTy, Name),
5842 args: {Arg0, Arg1, PacketSize, PacketAlign}));
5843 }
5844 // OpenCL v2.0 s6.13.16.4 Built-in pipe query functions
5845 case Builtin::BIget_pipe_num_packets:
5846 case Builtin::BIget_pipe_max_packets: {
5847 const char *BaseName;
5848 const auto *PipeTy = E->getArg(Arg: 0)->getType()->castAs<PipeType>();
5849 if (BuiltinID == Builtin::BIget_pipe_num_packets)
5850 BaseName = "__get_pipe_num_packets";
5851 else
5852 BaseName = "__get_pipe_max_packets";
5853 std::string Name = std::string(BaseName) +
5854 std::string(PipeTy->isReadOnly() ? "_ro" : "_wo");
5855
5856 // Building the generic function prototype.
5857 Value *Arg0 = EmitScalarExpr(E: E->getArg(Arg: 0));
5858 CGOpenCLRuntime OpenCLRT(CGM);
5859 Value *PacketSize = OpenCLRT.getPipeElemSize(PipeArg: E->getArg(Arg: 0));
5860 Value *PacketAlign = OpenCLRT.getPipeElemAlign(PipeArg: E->getArg(Arg: 0));
5861 llvm::Type *ArgTys[] = {Arg0->getType(), Int32Ty, Int32Ty};
5862 llvm::FunctionType *FTy = llvm::FunctionType::get(Result: Int32Ty, Params: ArgTys, isVarArg: false);
5863
5864 return RValue::get(V: EmitRuntimeCall(callee: CGM.CreateRuntimeFunction(Ty: FTy, Name),
5865 args: {Arg0, PacketSize, PacketAlign}));
5866 }
5867
5868 // OpenCL v2.0 s6.13.9 - Address space qualifier functions.
5869 case Builtin::BIto_global:
5870 case Builtin::BIto_local:
5871 case Builtin::BIto_private: {
5872 auto Arg0 = EmitScalarExpr(E: E->getArg(Arg: 0));
5873 auto NewArgT = llvm::PointerType::get(
5874 C&: getLLVMContext(),
5875 AddressSpace: CGM.getContext().getTargetAddressSpace(AS: LangAS::opencl_generic));
5876 auto NewRetT = llvm::PointerType::get(
5877 getLLVMContext(),
5878 CGM.getContext().getTargetAddressSpace(
5879 AS: E->getType()->getPointeeType().getAddressSpace()));
5880 auto FTy = llvm::FunctionType::get(NewRetT, {NewArgT}, false);
5881 llvm::Value *NewArg;
5882 if (Arg0->getType()->getPointerAddressSpace() !=
5883 NewArgT->getPointerAddressSpace())
5884 NewArg = Builder.CreateAddrSpaceCast(V: Arg0, DestTy: NewArgT);
5885 else
5886 NewArg = Builder.CreateBitOrPointerCast(V: Arg0, DestTy: NewArgT);
5887 auto NewName = std::string("__") + E->getDirectCallee()->getName().str();
5888 auto NewCall =
5889 EmitRuntimeCall(CGM.CreateRuntimeFunction(FTy, NewName), {NewArg});
5890 return RValue::get(Builder.CreateBitOrPointerCast(V: NewCall,
5891 DestTy: ConvertType(E->getType())));
5892 }
5893
5894 // OpenCL v2.0, s6.13.17 - Enqueue kernel function.
5895 // Table 6.13.17.1 specifies four overload forms of enqueue_kernel.
5896 // The code below expands the builtin call to a call to one of the following
5897 // functions that an OpenCL runtime library will have to provide:
5898 // __enqueue_kernel_basic
5899 // __enqueue_kernel_varargs
5900 // __enqueue_kernel_basic_events
5901 // __enqueue_kernel_events_varargs
5902 case Builtin::BIenqueue_kernel: {
5903 StringRef Name; // Generated function call name
5904 unsigned NumArgs = E->getNumArgs();
5905
5906 llvm::Type *QueueTy = ConvertType(getContext().OCLQueueTy);
5907 llvm::Type *GenericVoidPtrTy = Builder.getPtrTy(
5908 AddrSpace: getContext().getTargetAddressSpace(AS: LangAS::opencl_generic));
5909
5910 llvm::Value *Queue = EmitScalarExpr(E: E->getArg(Arg: 0));
5911 llvm::Value *Flags = EmitScalarExpr(E: E->getArg(Arg: 1));
5912 LValue NDRangeL = EmitAggExprToLValue(E: E->getArg(Arg: 2));
5913 llvm::Value *Range = NDRangeL.getAddress().emitRawPointer(CGF&: *this);
5914
5915 // FIXME: Look through the addrspacecast which may exist to the stack
5916 // temporary as a hack.
5917 //
5918 // This is hardcoding the assumed ABI of the target function. This assumes
5919 // direct passing for every argument except NDRange, which is assumed to be
5920 // byval or byref indirect passed.
5921 //
5922 // This should be fixed to query a signature from CGOpenCLRuntime, and go
5923 // through EmitCallArgs to get the correct target ABI.
5924 Range = Range->stripPointerCasts();
5925
5926 llvm::Type *RangePtrTy = Range->getType();
5927
5928 if (NumArgs == 4) {
5929 // The most basic form of the call with parameters:
5930 // queue_t, kernel_enqueue_flags_t, ndrange_t, block(void)
5931 Name = "__enqueue_kernel_basic";
5932 llvm::Type *ArgTys[] = {QueueTy, Int32Ty, RangePtrTy, GenericVoidPtrTy,
5933 GenericVoidPtrTy};
5934 llvm::FunctionType *FTy = llvm::FunctionType::get(Result: Int32Ty, Params: ArgTys, isVarArg: false);
5935
5936 auto Info =
5937 CGM.getOpenCLRuntime().emitOpenCLEnqueuedBlock(CGF&: *this, E: E->getArg(Arg: 3));
5938 llvm::Value *Kernel =
5939 Builder.CreatePointerCast(V: Info.KernelHandle, DestTy: GenericVoidPtrTy);
5940 llvm::Value *Block =
5941 Builder.CreatePointerCast(V: Info.BlockArg, DestTy: GenericVoidPtrTy);
5942
5943 auto RTCall = EmitRuntimeCall(callee: CGM.CreateRuntimeFunction(Ty: FTy, Name),
5944 args: {Queue, Flags, Range, Kernel, Block});
5945 return RValue::get(RTCall);
5946 }
5947 assert(NumArgs >= 5 && "Invalid enqueue_kernel signature");
5948
5949 // Create a temporary array to hold the sizes of local pointer arguments
5950 // for the block. \p First is the position of the first size argument.
5951 auto CreateArrayForSizeVar = [=](unsigned First)
5952 -> std::tuple<llvm::Value *, llvm::Value *, llvm::Value *> {
5953 llvm::APInt ArraySize(32, NumArgs - First);
5954 QualType SizeArrayTy = getContext().getConstantArrayType(
5955 EltTy: getContext().getSizeType(), ArySize: ArraySize, SizeExpr: nullptr,
5956 ASM: ArraySizeModifier::Normal,
5957 /*IndexTypeQuals=*/0);
5958 auto Tmp = CreateMemTemp(T: SizeArrayTy, Name: "block_sizes");
5959 llvm::Value *TmpPtr = Tmp.getPointer();
5960 // The EmitLifetime* pair expect a naked Alloca as their last argument,
5961 // however for cases where the default AS is not the Alloca AS, Tmp is
5962 // actually the Alloca ascasted to the default AS, hence the
5963 // stripPointerCasts()
5964 llvm::Value *Alloca = TmpPtr->stripPointerCasts();
5965 llvm::Value *TmpSize = EmitLifetimeStart(
5966 Size: CGM.getDataLayout().getTypeAllocSize(Ty: Tmp.getElementType()), Addr: Alloca);
5967 llvm::Value *ElemPtr;
5968 // Each of the following arguments specifies the size of the corresponding
5969 // argument passed to the enqueued block.
5970 auto *Zero = llvm::ConstantInt::get(Ty: IntTy, V: 0);
5971 for (unsigned I = First; I < NumArgs; ++I) {
5972 auto *Index = llvm::ConstantInt::get(Ty: IntTy, V: I - First);
5973 auto *GEP = Builder.CreateGEP(Ty: Tmp.getElementType(), Ptr: TmpPtr,
5974 IdxList: {Zero, Index});
5975 if (I == First)
5976 ElemPtr = GEP;
5977 auto *V =
5978 Builder.CreateZExtOrTrunc(V: EmitScalarExpr(E: E->getArg(Arg: I)), DestTy: SizeTy);
5979 Builder.CreateAlignedStore(
5980 Val: V, Ptr: GEP, Align: CGM.getDataLayout().getPrefTypeAlign(Ty: SizeTy));
5981 }
5982 // Return the Alloca itself rather than a potential ascast as this is only
5983 // used by the paired EmitLifetimeEnd.
5984 return {ElemPtr, TmpSize, Alloca};
5985 };
5986
5987 // Could have events and/or varargs.
5988 if (E->getArg(Arg: 3)->getType()->isBlockPointerType()) {
5989 // No events passed, but has variadic arguments.
5990 Name = "__enqueue_kernel_varargs";
5991 auto Info =
5992 CGM.getOpenCLRuntime().emitOpenCLEnqueuedBlock(CGF&: *this, E: E->getArg(Arg: 3));
5993 llvm::Value *Kernel =
5994 Builder.CreatePointerCast(V: Info.KernelHandle, DestTy: GenericVoidPtrTy);
5995 auto *Block = Builder.CreatePointerCast(V: Info.BlockArg, DestTy: GenericVoidPtrTy);
5996 auto [ElemPtr, TmpSize, TmpPtr] = CreateArrayForSizeVar(4);
5997
5998 // Create a vector of the arguments, as well as a constant value to
5999 // express to the runtime the number of variadic arguments.
6000 llvm::Value *const Args[] = {Queue, Flags,
6001 Range, Kernel,
6002 Block, ConstantInt::get(Ty: IntTy, V: NumArgs - 4),
6003 ElemPtr};
6004 llvm::Type *const ArgTys[] = {
6005 QueueTy, IntTy, RangePtrTy, GenericVoidPtrTy,
6006 GenericVoidPtrTy, IntTy, ElemPtr->getType()};
6007
6008 llvm::FunctionType *FTy = llvm::FunctionType::get(Result: Int32Ty, Params: ArgTys, isVarArg: false);
6009 auto Call = RValue::get(
6010 V: EmitRuntimeCall(callee: CGM.CreateRuntimeFunction(Ty: FTy, Name), args: Args));
6011 if (TmpSize)
6012 EmitLifetimeEnd(Size: TmpSize, Addr: TmpPtr);
6013 return Call;
6014 }
6015 // Any calls now have event arguments passed.
6016 if (NumArgs >= 7) {
6017 llvm::PointerType *PtrTy = llvm::PointerType::get(
6018 C&: CGM.getLLVMContext(),
6019 AddressSpace: CGM.getContext().getTargetAddressSpace(AS: LangAS::opencl_generic));
6020
6021 llvm::Value *NumEvents =
6022 Builder.CreateZExtOrTrunc(V: EmitScalarExpr(E: E->getArg(Arg: 3)), DestTy: Int32Ty);
6023
6024 // Since SemaOpenCLBuiltinEnqueueKernel allows fifth and sixth arguments
6025 // to be a null pointer constant (including `0` literal), we can take it
6026 // into account and emit null pointer directly.
6027 llvm::Value *EventWaitList = nullptr;
6028 if (E->getArg(Arg: 4)->isNullPointerConstant(
6029 Ctx&: getContext(), NPC: Expr::NPC_ValueDependentIsNotNull)) {
6030 EventWaitList = llvm::ConstantPointerNull::get(T: PtrTy);
6031 } else {
6032 EventWaitList =
6033 E->getArg(Arg: 4)->getType()->isArrayType()
6034 ? EmitArrayToPointerDecay(Array: E->getArg(Arg: 4)).emitRawPointer(CGF&: *this)
6035 : EmitScalarExpr(E: E->getArg(Arg: 4));
6036 // Convert to generic address space.
6037 EventWaitList = Builder.CreatePointerCast(V: EventWaitList, DestTy: PtrTy);
6038 }
6039 llvm::Value *EventRet = nullptr;
6040 if (E->getArg(Arg: 5)->isNullPointerConstant(
6041 Ctx&: getContext(), NPC: Expr::NPC_ValueDependentIsNotNull)) {
6042 EventRet = llvm::ConstantPointerNull::get(T: PtrTy);
6043 } else {
6044 EventRet =
6045 Builder.CreatePointerCast(V: EmitScalarExpr(E: E->getArg(Arg: 5)), DestTy: PtrTy);
6046 }
6047
6048 auto Info =
6049 CGM.getOpenCLRuntime().emitOpenCLEnqueuedBlock(CGF&: *this, E: E->getArg(Arg: 6));
6050 llvm::Value *Kernel =
6051 Builder.CreatePointerCast(V: Info.KernelHandle, DestTy: GenericVoidPtrTy);
6052 llvm::Value *Block =
6053 Builder.CreatePointerCast(V: Info.BlockArg, DestTy: GenericVoidPtrTy);
6054
6055 std::vector<llvm::Type *> ArgTys = {
6056 QueueTy, Int32Ty, RangePtrTy, Int32Ty,
6057 PtrTy, PtrTy, GenericVoidPtrTy, GenericVoidPtrTy};
6058
6059 std::vector<llvm::Value *> Args = {Queue, Flags, Range,
6060 NumEvents, EventWaitList, EventRet,
6061 Kernel, Block};
6062
6063 if (NumArgs == 7) {
6064 // Has events but no variadics.
6065 Name = "__enqueue_kernel_basic_events";
6066 llvm::FunctionType *FTy =
6067 llvm::FunctionType::get(Result: Int32Ty, Params: ArgTys, isVarArg: false);
6068 return RValue::get(
6069 V: EmitRuntimeCall(callee: CGM.CreateRuntimeFunction(Ty: FTy, Name), args: Args));
6070 }
6071 // Has event info and variadics
6072 // Pass the number of variadics to the runtime function too.
6073 Args.push_back(x: ConstantInt::get(Ty: Int32Ty, V: NumArgs - 7));
6074 ArgTys.push_back(x: Int32Ty);
6075 Name = "__enqueue_kernel_events_varargs";
6076
6077 auto [ElemPtr, TmpSize, TmpPtr] = CreateArrayForSizeVar(7);
6078 Args.push_back(x: ElemPtr);
6079 ArgTys.push_back(x: ElemPtr->getType());
6080
6081 llvm::FunctionType *FTy = llvm::FunctionType::get(Result: Int32Ty, Params: ArgTys, isVarArg: false);
6082 auto Call = RValue::get(
6083 V: EmitRuntimeCall(callee: CGM.CreateRuntimeFunction(Ty: FTy, Name), args: Args));
6084 if (TmpSize)
6085 EmitLifetimeEnd(Size: TmpSize, Addr: TmpPtr);
6086 return Call;
6087 }
6088 llvm_unreachable("Unexpected enqueue_kernel signature");
6089 }
6090 // OpenCL v2.0 s6.13.17.6 - Kernel query functions need bitcast of block
6091 // parameter.
6092 case Builtin::BIget_kernel_work_group_size: {
6093 llvm::Type *GenericVoidPtrTy = Builder.getPtrTy(
6094 AddrSpace: getContext().getTargetAddressSpace(AS: LangAS::opencl_generic));
6095 auto Info =
6096 CGM.getOpenCLRuntime().emitOpenCLEnqueuedBlock(CGF&: *this, E: E->getArg(Arg: 0));
6097 Value *Kernel =
6098 Builder.CreatePointerCast(V: Info.KernelHandle, DestTy: GenericVoidPtrTy);
6099 Value *Arg = Builder.CreatePointerCast(V: Info.BlockArg, DestTy: GenericVoidPtrTy);
6100 return RValue::get(V: EmitRuntimeCall(
6101 callee: CGM.CreateRuntimeFunction(
6102 Ty: llvm::FunctionType::get(Result: IntTy, Params: {GenericVoidPtrTy, GenericVoidPtrTy},
6103 isVarArg: false),
6104 Name: "__get_kernel_work_group_size_impl"),
6105 args: {Kernel, Arg}));
6106 }
6107 case Builtin::BIget_kernel_preferred_work_group_size_multiple: {
6108 llvm::Type *GenericVoidPtrTy = Builder.getPtrTy(
6109 AddrSpace: getContext().getTargetAddressSpace(AS: LangAS::opencl_generic));
6110 auto Info =
6111 CGM.getOpenCLRuntime().emitOpenCLEnqueuedBlock(CGF&: *this, E: E->getArg(Arg: 0));
6112 Value *Kernel =
6113 Builder.CreatePointerCast(V: Info.KernelHandle, DestTy: GenericVoidPtrTy);
6114 Value *Arg = Builder.CreatePointerCast(V: Info.BlockArg, DestTy: GenericVoidPtrTy);
6115 return RValue::get(V: EmitRuntimeCall(
6116 callee: CGM.CreateRuntimeFunction(
6117 Ty: llvm::FunctionType::get(Result: IntTy, Params: {GenericVoidPtrTy, GenericVoidPtrTy},
6118 isVarArg: false),
6119 Name: "__get_kernel_preferred_work_group_size_multiple_impl"),
6120 args: {Kernel, Arg}));
6121 }
6122 case Builtin::BIget_kernel_max_sub_group_size_for_ndrange:
6123 case Builtin::BIget_kernel_sub_group_count_for_ndrange: {
6124 llvm::Type *GenericVoidPtrTy = Builder.getPtrTy(
6125 AddrSpace: getContext().getTargetAddressSpace(AS: LangAS::opencl_generic));
6126 LValue NDRangeL = EmitAggExprToLValue(E: E->getArg(Arg: 0));
6127 llvm::Value *NDRange = NDRangeL.getAddress().emitRawPointer(CGF&: *this);
6128 auto Info =
6129 CGM.getOpenCLRuntime().emitOpenCLEnqueuedBlock(CGF&: *this, E: E->getArg(Arg: 1));
6130 Value *Kernel =
6131 Builder.CreatePointerCast(V: Info.KernelHandle, DestTy: GenericVoidPtrTy);
6132 Value *Block = Builder.CreatePointerCast(V: Info.BlockArg, DestTy: GenericVoidPtrTy);
6133 const char *Name =
6134 BuiltinID == Builtin::BIget_kernel_max_sub_group_size_for_ndrange
6135 ? "__get_kernel_max_sub_group_size_for_ndrange_impl"
6136 : "__get_kernel_sub_group_count_for_ndrange_impl";
6137 return RValue::get(V: EmitRuntimeCall(
6138 callee: CGM.CreateRuntimeFunction(
6139 Ty: llvm::FunctionType::get(
6140 Result: IntTy, Params: {NDRange->getType(), GenericVoidPtrTy, GenericVoidPtrTy},
6141 isVarArg: false),
6142 Name),
6143 args: {NDRange, Kernel, Block}));
6144 }
6145 case Builtin::BI__builtin_store_half:
6146 case Builtin::BI__builtin_store_halff: {
6147 Value *Val = EmitScalarExpr(E: E->getArg(Arg: 0));
6148 Address Address = EmitPointerWithAlignment(Addr: E->getArg(Arg: 1));
6149 Value *HalfVal = Builder.CreateFPTrunc(V: Val, DestTy: Builder.getHalfTy());
6150 Builder.CreateStore(Val: HalfVal, Addr: Address);
6151 return RValue::get(V: nullptr);
6152 }
6153 case Builtin::BI__builtin_load_half: {
6154 Address Address = EmitPointerWithAlignment(Addr: E->getArg(Arg: 0));
6155 Value *HalfVal = Builder.CreateLoad(Addr: Address);
6156 return RValue::get(V: Builder.CreateFPExt(V: HalfVal, DestTy: Builder.getDoubleTy()));
6157 }
6158 case Builtin::BI__builtin_load_halff: {
6159 Address Address = EmitPointerWithAlignment(Addr: E->getArg(Arg: 0));
6160 Value *HalfVal = Builder.CreateLoad(Addr: Address);
6161 return RValue::get(V: Builder.CreateFPExt(V: HalfVal, DestTy: Builder.getFloatTy()));
6162 }
6163 case Builtin::BI__builtin_printf:
6164 case Builtin::BIprintf:
6165 if (getTarget().getTriple().isNVPTX() ||
6166 getTarget().getTriple().isAMDGCN() ||
6167 (getTarget().getTriple().isSPIRV() &&
6168 getTarget().getTriple().getVendor() == Triple::VendorType::AMD)) {
6169 if (getTarget().getTriple().isNVPTX())
6170 return EmitNVPTXDevicePrintfCallExpr(E);
6171 if ((getTarget().getTriple().isAMDGCN() ||
6172 getTarget().getTriple().isSPIRV()) &&
6173 getLangOpts().HIP)
6174 return EmitAMDGPUDevicePrintfCallExpr(E);
6175 }
6176
6177 break;
6178 case Builtin::BI__builtin_canonicalize:
6179 case Builtin::BI__builtin_canonicalizef:
6180 case Builtin::BI__builtin_canonicalizef16:
6181 case Builtin::BI__builtin_canonicalizel:
6182 return RValue::get(
6183 emitBuiltinWithOneOverloadedType<1>(*this, E, Intrinsic::canonicalize));
6184
6185 case Builtin::BI__builtin_thread_pointer: {
6186 if (!getContext().getTargetInfo().isTLSSupported())
6187 CGM.ErrorUnsupported(E, "__builtin_thread_pointer");
6188
6189 return RValue::get(Builder.CreateIntrinsic(llvm::Intrinsic::thread_pointer,
6190 {GlobalsInt8PtrTy}, {}));
6191 }
6192 case Builtin::BI__builtin_os_log_format:
6193 return emitBuiltinOSLogFormat(E: *E);
6194
6195 case Builtin::BI__xray_customevent: {
6196 if (!ShouldXRayInstrumentFunction())
6197 return RValue::getIgnored();
6198
6199 if (!CGM.getCodeGenOpts().XRayInstrumentationBundle.has(
6200 K: XRayInstrKind::Custom))
6201 return RValue::getIgnored();
6202
6203 if (const auto *XRayAttr = CurFuncDecl->getAttr<XRayInstrumentAttr>())
6204 if (XRayAttr->neverXRayInstrument() && !AlwaysEmitXRayCustomEvents())
6205 return RValue::getIgnored();
6206
6207 Function *F = CGM.getIntrinsic(Intrinsic::xray_customevent);
6208 auto FTy = F->getFunctionType();
6209 auto Arg0 = E->getArg(Arg: 0);
6210 auto Arg0Val = EmitScalarExpr(E: Arg0);
6211 auto Arg0Ty = Arg0->getType();
6212 auto PTy0 = FTy->getParamType(i: 0);
6213 if (PTy0 != Arg0Val->getType()) {
6214 if (Arg0Ty->isArrayType())
6215 Arg0Val = EmitArrayToPointerDecay(Array: Arg0).emitRawPointer(CGF&: *this);
6216 else
6217 Arg0Val = Builder.CreatePointerCast(V: Arg0Val, DestTy: PTy0);
6218 }
6219 auto Arg1 = EmitScalarExpr(E: E->getArg(Arg: 1));
6220 auto PTy1 = FTy->getParamType(i: 1);
6221 if (PTy1 != Arg1->getType())
6222 Arg1 = Builder.CreateTruncOrBitCast(V: Arg1, DestTy: PTy1);
6223 return RValue::get(V: Builder.CreateCall(Callee: F, Args: {Arg0Val, Arg1}));
6224 }
6225
6226 case Builtin::BI__xray_typedevent: {
6227 // TODO: There should be a way to always emit events even if the current
6228 // function is not instrumented. Losing events in a stream can cripple
6229 // a trace.
6230 if (!ShouldXRayInstrumentFunction())
6231 return RValue::getIgnored();
6232
6233 if (!CGM.getCodeGenOpts().XRayInstrumentationBundle.has(
6234 K: XRayInstrKind::Typed))
6235 return RValue::getIgnored();
6236
6237 if (const auto *XRayAttr = CurFuncDecl->getAttr<XRayInstrumentAttr>())
6238 if (XRayAttr->neverXRayInstrument() && !AlwaysEmitXRayTypedEvents())
6239 return RValue::getIgnored();
6240
6241 Function *F = CGM.getIntrinsic(Intrinsic::xray_typedevent);
6242 auto FTy = F->getFunctionType();
6243 auto Arg0 = EmitScalarExpr(E: E->getArg(Arg: 0));
6244 auto PTy0 = FTy->getParamType(i: 0);
6245 if (PTy0 != Arg0->getType())
6246 Arg0 = Builder.CreateTruncOrBitCast(V: Arg0, DestTy: PTy0);
6247 auto Arg1 = E->getArg(Arg: 1);
6248 auto Arg1Val = EmitScalarExpr(E: Arg1);
6249 auto Arg1Ty = Arg1->getType();
6250 auto PTy1 = FTy->getParamType(i: 1);
6251 if (PTy1 != Arg1Val->getType()) {
6252 if (Arg1Ty->isArrayType())
6253 Arg1Val = EmitArrayToPointerDecay(Array: Arg1).emitRawPointer(CGF&: *this);
6254 else
6255 Arg1Val = Builder.CreatePointerCast(V: Arg1Val, DestTy: PTy1);
6256 }
6257 auto Arg2 = EmitScalarExpr(E: E->getArg(Arg: 2));
6258 auto PTy2 = FTy->getParamType(i: 2);
6259 if (PTy2 != Arg2->getType())
6260 Arg2 = Builder.CreateTruncOrBitCast(V: Arg2, DestTy: PTy2);
6261 return RValue::get(V: Builder.CreateCall(Callee: F, Args: {Arg0, Arg1Val, Arg2}));
6262 }
6263
6264 case Builtin::BI__builtin_ms_va_start:
6265 case Builtin::BI__builtin_ms_va_end:
6266 return RValue::get(
6267 EmitVAStartEnd(EmitMSVAListRef(E->getArg(0)).emitRawPointer(*this),
6268 BuiltinID == Builtin::BI__builtin_ms_va_start));
6269
6270 case Builtin::BI__builtin_ms_va_copy: {
6271 // Lower this manually. We can't reliably determine whether or not any
6272 // given va_copy() is for a Win64 va_list from the calling convention
6273 // alone, because it's legal to do this from a System V ABI function.
6274 // With opaque pointer types, we won't have enough information in LLVM
6275 // IR to determine this from the argument types, either. Best to do it
6276 // now, while we have enough information.
6277 Address DestAddr = EmitMSVAListRef(E: E->getArg(Arg: 0));
6278 Address SrcAddr = EmitMSVAListRef(E: E->getArg(Arg: 1));
6279
6280 DestAddr = DestAddr.withElementType(ElemTy: Int8PtrTy);
6281 SrcAddr = SrcAddr.withElementType(ElemTy: Int8PtrTy);
6282
6283 Value *ArgPtr = Builder.CreateLoad(Addr: SrcAddr, Name: "ap.val");
6284 return RValue::get(V: Builder.CreateStore(Val: ArgPtr, Addr: DestAddr));
6285 }
6286
6287 case Builtin::BI__builtin_get_device_side_mangled_name: {
6288 auto Name = CGM.getCUDARuntime().getDeviceSideName(
6289 cast<DeclRefExpr>(Val: E->getArg(Arg: 0)->IgnoreImpCasts())->getDecl());
6290 auto Str = CGM.GetAddrOfConstantCString(Str: Name, GlobalName: "");
6291 return RValue::get(Str.getPointer());
6292 }
6293 }
6294
6295 // If this is an alias for a lib function (e.g. __builtin_sin), emit
6296 // the call using the normal call path, but using the unmangled
6297 // version of the function name.
6298 const auto &BI = getContext().BuiltinInfo;
6299 if (!shouldEmitBuiltinAsIR(BuiltinID, BI, CGF: *this) &&
6300 BI.isLibFunction(ID: BuiltinID))
6301 return emitLibraryCall(CGF&: *this, FD, E,
6302 calleeValue: CGM.getBuiltinLibFunction(FD, BuiltinID));
6303
6304 // If this is a predefined lib function (e.g. malloc), emit the call
6305 // using exactly the normal call path.
6306 if (BI.isPredefinedLibFunction(ID: BuiltinID))
6307 return emitLibraryCall(CGF&: *this, FD, E, calleeValue: CGM.getRawFunctionPointer(GD: FD));
6308
6309 // Check that a call to a target specific builtin has the correct target
6310 // features.
6311 // This is down here to avoid non-target specific builtins, however, if
6312 // generic builtins start to require generic target features then we
6313 // can move this up to the beginning of the function.
6314 checkTargetFeatures(E, TargetDecl: FD);
6315
6316 if (unsigned VectorWidth = getContext().BuiltinInfo.getRequiredVectorWidth(ID: BuiltinID))
6317 LargestVectorWidth = std::max(a: LargestVectorWidth, b: VectorWidth);
6318
6319 // See if we have a target specific intrinsic.
6320 std::string Name = getContext().BuiltinInfo.getName(ID: BuiltinID);
6321 Intrinsic::ID IntrinsicID = Intrinsic::not_intrinsic;
6322 StringRef Prefix =
6323 llvm::Triple::getArchTypePrefix(Kind: getTarget().getTriple().getArch());
6324 if (!Prefix.empty()) {
6325 IntrinsicID = Intrinsic::getIntrinsicForClangBuiltin(TargetPrefix: Prefix.data(), BuiltinName: Name);
6326 if (IntrinsicID == Intrinsic::not_intrinsic && Prefix == "spv" &&
6327 getTarget().getTriple().getOS() == llvm::Triple::OSType::AMDHSA)
6328 IntrinsicID = Intrinsic::getIntrinsicForClangBuiltin(TargetPrefix: "amdgcn", BuiltinName: Name);
6329 // NOTE we don't need to perform a compatibility flag check here since the
6330 // intrinsics are declared in Builtins*.def via LANGBUILTIN which filter the
6331 // MS builtins via ALL_MS_LANGUAGES and are filtered earlier.
6332 if (IntrinsicID == Intrinsic::not_intrinsic)
6333 IntrinsicID = Intrinsic::getIntrinsicForMSBuiltin(TargetPrefix: Prefix.data(), BuiltinName: Name);
6334 }
6335
6336 if (IntrinsicID != Intrinsic::not_intrinsic) {
6337 SmallVector<Value*, 16> Args;
6338
6339 // Find out if any arguments are required to be integer constant
6340 // expressions.
6341 unsigned ICEArguments = 0;
6342 ASTContext::GetBuiltinTypeError Error;
6343 getContext().GetBuiltinType(ID: BuiltinID, Error, IntegerConstantArgs: &ICEArguments);
6344 assert(Error == ASTContext::GE_None && "Should not codegen an error");
6345
6346 Function *F = CGM.getIntrinsic(IID: IntrinsicID);
6347 llvm::FunctionType *FTy = F->getFunctionType();
6348
6349 for (unsigned i = 0, e = E->getNumArgs(); i != e; ++i) {
6350 Value *ArgValue = EmitScalarOrConstFoldImmArg(ICEArguments, Idx: i, E);
6351 // If the intrinsic arg type is different from the builtin arg type
6352 // we need to do a bit cast.
6353 llvm::Type *PTy = FTy->getParamType(i);
6354 if (PTy != ArgValue->getType()) {
6355 // XXX - vector of pointers?
6356 if (auto *PtrTy = dyn_cast<llvm::PointerType>(Val: PTy)) {
6357 if (PtrTy->getAddressSpace() !=
6358 ArgValue->getType()->getPointerAddressSpace()) {
6359 ArgValue = Builder.CreateAddrSpaceCast(
6360 V: ArgValue, DestTy: llvm::PointerType::get(C&: getLLVMContext(),
6361 AddressSpace: PtrTy->getAddressSpace()));
6362 }
6363 }
6364
6365 // Cast vector type (e.g., v256i32) to x86_amx, this only happen
6366 // in amx intrinsics.
6367 if (PTy->isX86_AMXTy())
6368 ArgValue = Builder.CreateIntrinsic(Intrinsic::x86_cast_vector_to_tile,
6369 {ArgValue->getType()}, {ArgValue});
6370 else
6371 ArgValue = Builder.CreateBitCast(V: ArgValue, DestTy: PTy);
6372 }
6373
6374 Args.push_back(Elt: ArgValue);
6375 }
6376
6377 Value *V = Builder.CreateCall(Callee: F, Args);
6378 QualType BuiltinRetType = E->getType();
6379
6380 llvm::Type *RetTy = VoidTy;
6381 if (!BuiltinRetType->isVoidType())
6382 RetTy = ConvertType(T: BuiltinRetType);
6383
6384 if (RetTy != V->getType()) {
6385 // XXX - vector of pointers?
6386 if (auto *PtrTy = dyn_cast<llvm::PointerType>(Val: RetTy)) {
6387 if (PtrTy->getAddressSpace() != V->getType()->getPointerAddressSpace()) {
6388 V = Builder.CreateAddrSpaceCast(
6389 V, DestTy: llvm::PointerType::get(C&: getLLVMContext(),
6390 AddressSpace: PtrTy->getAddressSpace()));
6391 }
6392 }
6393
6394 // Cast x86_amx to vector type (e.g., v256i32), this only happen
6395 // in amx intrinsics.
6396 if (V->getType()->isX86_AMXTy())
6397 V = Builder.CreateIntrinsic(Intrinsic::x86_cast_tile_to_vector, {RetTy},
6398 {V});
6399 else
6400 V = Builder.CreateBitCast(V, DestTy: RetTy);
6401 }
6402
6403 if (RetTy->isVoidTy())
6404 return RValue::get(V: nullptr);
6405
6406 return RValue::get(V);
6407 }
6408
6409 // Some target-specific builtins can have aggregate return values, e.g.
6410 // __builtin_arm_mve_vld2q_u32. So if the result is an aggregate, force
6411 // ReturnValue to be non-null, so that the target-specific emission code can
6412 // always just emit into it.
6413 TypeEvaluationKind EvalKind = getEvaluationKind(T: E->getType());
6414 if (EvalKind == TEK_Aggregate && ReturnValue.isNull()) {
6415 Address DestPtr = CreateMemTemp(E->getType(), "agg.tmp");
6416 ReturnValue = ReturnValueSlot(DestPtr, false);
6417 }
6418
6419 // Now see if we can emit a target-specific builtin.
6420 if (Value *V = EmitTargetBuiltinExpr(BuiltinID, E, ReturnValue)) {
6421 switch (EvalKind) {
6422 case TEK_Scalar:
6423 if (V->getType()->isVoidTy())
6424 return RValue::get(V: nullptr);
6425 return RValue::get(V);
6426 case TEK_Aggregate:
6427 return RValue::getAggregate(addr: ReturnValue.getAddress(),
6428 isVolatile: ReturnValue.isVolatile());
6429 case TEK_Complex:
6430 llvm_unreachable("No current target builtin returns complex");
6431 }
6432 llvm_unreachable("Bad evaluation kind in EmitBuiltinExpr");
6433 }
6434
6435 // EmitHLSLBuiltinExpr will check getLangOpts().HLSL
6436 if (Value *V = EmitHLSLBuiltinExpr(BuiltinID, E, ReturnValue)) {
6437 switch (EvalKind) {
6438 case TEK_Scalar:
6439 if (V->getType()->isVoidTy())
6440 return RValue::get(V: nullptr);
6441 return RValue::get(V);
6442 case TEK_Aggregate:
6443 return RValue::getAggregate(addr: ReturnValue.getAddress(),
6444 isVolatile: ReturnValue.isVolatile());
6445 case TEK_Complex:
6446 llvm_unreachable("No current hlsl builtin returns complex");
6447 }
6448 llvm_unreachable("Bad evaluation kind in EmitBuiltinExpr");
6449 }
6450
6451 if (getLangOpts().HIPStdPar && getLangOpts().CUDAIsDevice)
6452 return EmitHipStdParUnsupportedBuiltin(CGF: this, FD);
6453
6454 ErrorUnsupported(E, "builtin function");
6455
6456 // Unknown builtin, for now just dump it out and return undef.
6457 return GetUndefRValue(Ty: E->getType());
6458}
6459
6460namespace {
6461struct BuiltinAlignArgs {
6462 llvm::Value *Src = nullptr;
6463 llvm::Type *SrcType = nullptr;
6464 llvm::Value *Alignment = nullptr;
6465 llvm::Value *Mask = nullptr;
6466 llvm::IntegerType *IntType = nullptr;
6467
6468 BuiltinAlignArgs(const CallExpr *E, CodeGenFunction &CGF) {
6469 QualType AstType = E->getArg(Arg: 0)->getType();
6470 if (AstType->isArrayType())
6471 Src = CGF.EmitArrayToPointerDecay(Array: E->getArg(Arg: 0)).emitRawPointer(CGF);
6472 else
6473 Src = CGF.EmitScalarExpr(E: E->getArg(Arg: 0));
6474 SrcType = Src->getType();
6475 if (SrcType->isPointerTy()) {
6476 IntType = IntegerType::get(
6477 C&: CGF.getLLVMContext(),
6478 NumBits: CGF.CGM.getDataLayout().getIndexTypeSizeInBits(Ty: SrcType));
6479 } else {
6480 assert(SrcType->isIntegerTy());
6481 IntType = cast<llvm::IntegerType>(Val: SrcType);
6482 }
6483 Alignment = CGF.EmitScalarExpr(E: E->getArg(Arg: 1));
6484 Alignment = CGF.Builder.CreateZExtOrTrunc(V: Alignment, DestTy: IntType, Name: "alignment");
6485 auto *One = llvm::ConstantInt::get(Ty: IntType, V: 1);
6486 Mask = CGF.Builder.CreateSub(LHS: Alignment, RHS: One, Name: "mask");
6487 }
6488};
6489} // namespace
6490
6491/// Generate (x & (y-1)) == 0.
6492RValue CodeGenFunction::EmitBuiltinIsAligned(const CallExpr *E) {
6493 BuiltinAlignArgs Args(E, *this);
6494 llvm::Value *SrcAddress = Args.Src;
6495 if (Args.SrcType->isPointerTy())
6496 SrcAddress =
6497 Builder.CreateBitOrPointerCast(V: Args.Src, DestTy: Args.IntType, Name: "src_addr");
6498 return RValue::get(V: Builder.CreateICmpEQ(
6499 LHS: Builder.CreateAnd(LHS: SrcAddress, RHS: Args.Mask, Name: "set_bits"),
6500 RHS: llvm::Constant::getNullValue(Ty: Args.IntType), Name: "is_aligned"));
6501}
6502
6503/// Generate (x & ~(y-1)) to align down or ((x+(y-1)) & ~(y-1)) to align up.
6504/// Note: For pointer types we can avoid ptrtoint/inttoptr pairs by using the
6505/// llvm.ptrmask intrinsic (with a GEP before in the align_up case).
6506RValue CodeGenFunction::EmitBuiltinAlignTo(const CallExpr *E, bool AlignUp) {
6507 BuiltinAlignArgs Args(E, *this);
6508 llvm::Value *SrcForMask = Args.Src;
6509 if (AlignUp) {
6510 // When aligning up we have to first add the mask to ensure we go over the
6511 // next alignment value and then align down to the next valid multiple.
6512 // By adding the mask, we ensure that align_up on an already aligned
6513 // value will not change the value.
6514 if (Args.Src->getType()->isPointerTy()) {
6515 if (getLangOpts().PointerOverflowDefined)
6516 SrcForMask =
6517 Builder.CreateGEP(Ty: Int8Ty, Ptr: SrcForMask, IdxList: Args.Mask, Name: "over_boundary");
6518 else
6519 SrcForMask = EmitCheckedInBoundsGEP(Int8Ty, SrcForMask, Args.Mask,
6520 /*SignedIndices=*/true,
6521 /*isSubtraction=*/false,
6522 E->getExprLoc(), "over_boundary");
6523 } else {
6524 SrcForMask = Builder.CreateAdd(LHS: SrcForMask, RHS: Args.Mask, Name: "over_boundary");
6525 }
6526 }
6527 // Invert the mask to only clear the lower bits.
6528 llvm::Value *InvertedMask = Builder.CreateNot(V: Args.Mask, Name: "inverted_mask");
6529 llvm::Value *Result = nullptr;
6530 if (Args.Src->getType()->isPointerTy()) {
6531 Result = Builder.CreateIntrinsic(
6532 Intrinsic::ptrmask, {Args.SrcType, Args.IntType},
6533 {SrcForMask, InvertedMask}, nullptr, "aligned_result");
6534 } else {
6535 Result = Builder.CreateAnd(LHS: SrcForMask, RHS: InvertedMask, Name: "aligned_result");
6536 }
6537 assert(Result->getType() == Args.SrcType);
6538 return RValue::get(V: Result);
6539}
6540

source code of clang/lib/CodeGen/CGBuiltin.cpp