1	//===------ SimplifyLibCalls.cpp - Library calls simplifier ---------------===//
2	//
3	// Part of the LLVM Project, under the Apache License v2.0 with LLVM Exceptions.
4	// See https://llvm.org/LICENSE.txt for license information.
5	// SPDX-License-Identifier: Apache-2.0 WITH LLVM-exception
6	//
7	//===----------------------------------------------------------------------===//
8	//
9	// This file implements the library calls simplifier. It does not implement
10	// any pass, but can't be used by other passes to do simplifications.
11	//
12	//===----------------------------------------------------------------------===//
13
14	#include "llvm/Transforms/Utils/SimplifyLibCalls.h"
15	#include "llvm/ADT/APSInt.h"
16	#include "llvm/ADT/SmallString.h"
17	#include "llvm/ADT/StringExtras.h"
18	#include "llvm/Analysis/ConstantFolding.h"
19	#include "llvm/Analysis/Loads.h"
20	#include "llvm/Analysis/OptimizationRemarkEmitter.h"
21	#include "llvm/Analysis/ValueTracking.h"
22	#include "llvm/IR/AttributeMask.h"
23	#include "llvm/IR/DataLayout.h"
24	#include "llvm/IR/Function.h"
25	#include "llvm/IR/IRBuilder.h"
26	#include "llvm/IR/IntrinsicInst.h"
27	#include "llvm/IR/Intrinsics.h"
28	#include "llvm/IR/Module.h"
29	#include "llvm/IR/PatternMatch.h"
30	#include "llvm/Support/CommandLine.h"
31	#include "llvm/Support/KnownBits.h"
32	#include "llvm/Support/MathExtras.h"
33	#include "llvm/TargetParser/Triple.h"
34	#include "llvm/Transforms/Utils/BuildLibCalls.h"
35	#include "llvm/Transforms/Utils/Local.h"
36	#include "llvm/Transforms/Utils/SizeOpts.h"
37
38	#include <cmath>
39
40	using namespace llvm;
41	using namespace PatternMatch;
42
43	static cl::opt<bool>
44	EnableUnsafeFPShrink("enable-double-float-shrink", cl::Hidden,
45	cl::init(Val: false),
46	cl::desc("Enable unsafe double to float "
47	"shrinking for math lib calls"));
48
49	// Enable conversion of operator new calls with a MemProf hot or cold hint
50	// to an operator new call that takes a hot/cold hint. Off by default since
51	// not all allocators currently support this extension.
52	static cl::opt<bool>
53	OptimizeHotColdNew("optimize-hot-cold-new", cl::Hidden, cl::init(Val: false),
54	cl::desc("Enable hot/cold operator new library calls"));
55
56	namespace {
57
58	// Specialized parser to ensure the hint is an 8 bit value (we can't specify
59	// uint8_t to opt<> as that is interpreted to mean that we are passing a char
60	// option with a specific set of values.
61	struct HotColdHintParser : public cl::parser<unsigned> {
62	HotColdHintParser(cl::Option &O) : cl::parser<unsigned>(O) {}
63
64	bool parse(cl::Option &O, StringRef ArgName, StringRef Arg, unsigned &Value) {
65	if (Arg.getAsInteger(Radix: 0, Result&: Value))
66	return O.error(Message: "'" + Arg + "' value invalid for uint argument!");
67
68	if (Value > 255)
69	return O.error(Message: "'" + Arg + "' value must be in the range [0, 255]!");
70
71	return false;
72	}
73	};
74
75	} // end anonymous namespace
76
77	// Hot/cold operator new takes an 8 bit hotness hint, where 0 is the coldest
78	// and 255 is the hottest. Default to 1 value away from the coldest and hottest
79	// hints, so that the compiler hinted allocations are slightly less strong than
80	// manually inserted hints at the two extremes.
81	static cl::opt<unsigned, false, HotColdHintParser> ColdNewHintValue(
82	"cold-new-hint-value", cl::Hidden, cl::init(Val: 1),
83	cl::desc("Value to pass to hot/cold operator new for cold allocation"));
84	static cl::opt<unsigned, false, HotColdHintParser> HotNewHintValue(
85	"hot-new-hint-value", cl::Hidden, cl::init(Val: 254),
86	cl::desc("Value to pass to hot/cold operator new for hot allocation"));
87
88	//===----------------------------------------------------------------------===//
89	// Helper Functions
90	//===----------------------------------------------------------------------===//
91
92	static bool ignoreCallingConv(LibFunc Func) {
93	return Func == LibFunc_abs || Func == LibFunc_labs ||
94	Func == LibFunc_llabs || Func == LibFunc_strlen;
95	}
96
97	/// Return true if it is only used in equality comparisons with With.
98	static bool isOnlyUsedInEqualityComparison(Value *V, Value *With) {
99	for (User *U : V->users()) {
100	if (ICmpInst *IC = dyn_cast<ICmpInst>(Val: U))
101	if (IC->isEquality() && IC->getOperand(i_nocapture: 1) == With)
102	continue;
103	// Unknown instruction.
104	return false;
105	}
106	return true;
107	}
108
109	static bool callHasFloatingPointArgument(const CallInst *CI) {
110	return any_of(Range: CI->operands(), P: [](const Use &OI) {
111	return OI->getType()->isFloatingPointTy();
112	});
113	}
114
115	static bool callHasFP128Argument(const CallInst *CI) {
116	return any_of(Range: CI->operands(), P: [](const Use &OI) {
117	return OI->getType()->isFP128Ty();
118	});
119	}
120
121	// Convert the entire string Str representing an integer in Base, up to
122	// the terminating nul if present, to a constant according to the rules
123	// of strtoul[l] or, when AsSigned is set, of strtol[l]. On success
124	// return the result, otherwise null.
125	// The function assumes the string is encoded in ASCII and carefully
126	// avoids converting sequences (including "") that the corresponding
127	// library call might fail and set errno for.
128	static Value *convertStrToInt(CallInst *CI, StringRef &Str, Value *EndPtr,
129	uint64_t Base, bool AsSigned, IRBuilderBase &B) {
130	if (Base < 2 || Base > 36)
131	if (Base != 0)
132	// Fail for an invalid base (required by POSIX).
133	return nullptr;
134
135	// Current offset into the original string to reflect in EndPtr.
136	size_t Offset = 0;
137	// Strip leading whitespace.
138	for ( ; Offset != Str.size(); ++Offset)
139	if (!isSpace(C: (unsigned char)Str[Offset])) {
140	Str = Str.substr(Start: Offset);
141	break;
142	}
143
144	if (Str.empty())
145	// Fail for empty subject sequences (POSIX allows but doesn't require
146	// strtol[l]/strtoul[l] to fail with EINVAL).
147	return nullptr;
148
149	// Strip but remember the sign.
150	bool Negate = Str[0] == '-';
151	if (Str[0] == '-' || Str[0] == '+') {
152	Str = Str.drop_front();
153	if (Str.empty())
154	// Fail for a sign with nothing after it.
155	return nullptr;
156	++Offset;
157	}
158
159	// Set Max to the absolute value of the minimum (for signed), or
160	// to the maximum (for unsigned) value representable in the type.
161	Type *RetTy = CI->getType();
162	unsigned NBits = RetTy->getPrimitiveSizeInBits();
163	uint64_t Max = AsSigned && Negate ? 1 : 0;
164	Max += AsSigned ? maxIntN(N: NBits) : maxUIntN(N: NBits);
165
166	// Autodetect Base if it's zero and consume the "0x" prefix.
167	if (Str.size() > 1) {
168	if (Str[0] == '0') {
169	if (toUpper(x: (unsigned char)Str[1]) == 'X') {
170	if (Str.size() == 2 || (Base && Base != 16))
171	// Fail if Base doesn't allow the "0x" prefix or for the prefix
172	// alone that implementations like BSD set errno to EINVAL for.
173	return nullptr;
174
175	Str = Str.drop_front(N: 2);
176	Offset += 2;
177	Base = 16;
178	}
179	else if (Base == 0)
180	Base = 8;
181	} else if (Base == 0)
182	Base = 10;
183	}
184	else if (Base == 0)
185	Base = 10;
186
187	// Convert the rest of the subject sequence, not including the sign,
188	// to its uint64_t representation (this assumes the source character
189	// set is ASCII).
190	uint64_t Result = 0;
191	for (unsigned i = 0; i != Str.size(); ++i) {
192	unsigned char DigVal = Str[i];
193	if (isDigit(C: DigVal))
194	DigVal = DigVal - '0';
195	else {
196	DigVal = toUpper(x: DigVal);
197	if (isAlpha(C: DigVal))
198	DigVal = DigVal - 'A' + 10;
199	else
200	return nullptr;
201	}
202
203	if (DigVal >= Base)
204	// Fail if the digit is not valid in the Base.
205	return nullptr;
206
207	// Add the digit and fail if the result is not representable in
208	// the (unsigned form of the) destination type.
209	bool VFlow;
210	Result = SaturatingMultiplyAdd(X: Result, Y: Base, A: (uint64_t)DigVal, ResultOverflowed: &VFlow);
211	if (VFlow || Result > Max)
212	return nullptr;
213	}
214
215	if (EndPtr) {
216	// Store the pointer to the end.
217	Value *Off = B.getInt64(C: Offset + Str.size());
218	Value *StrBeg = CI->getArgOperand(i: 0);
219	Value *StrEnd = B.CreateInBoundsGEP(Ty: B.getInt8Ty(), Ptr: StrBeg, IdxList: Off, Name: "endptr");
220	B.CreateStore(Val: StrEnd, Ptr: EndPtr);
221	}
222
223	if (Negate)
224	// Unsigned negation doesn't overflow.
225	Result = -Result;
226
227	return ConstantInt::get(Ty: RetTy, V: Result);
228	}
229
230	static bool isOnlyUsedInComparisonWithZero(Value *V) {
231	for (User *U : V->users()) {
232	if (ICmpInst *IC = dyn_cast<ICmpInst>(Val: U))
233	if (Constant *C = dyn_cast<Constant>(Val: IC->getOperand(i_nocapture: 1)))
234	if (C->isNullValue())
235	continue;
236	// Unknown instruction.
237	return false;
238	}
239	return true;
240	}
241
242	static bool canTransformToMemCmp(CallInst *CI, Value *Str, uint64_t Len,
243	const DataLayout &DL) {
244	if (!isOnlyUsedInComparisonWithZero(V: CI))
245	return false;
246
247	if (!isDereferenceableAndAlignedPointer(V: Str, Alignment: Align(1), Size: APInt(64, Len), DL))
248	return false;
249
250	if (CI->getFunction()->hasFnAttribute(Attribute::SanitizeMemory))
251	return false;
252
253	return true;
254	}
255
256	static void annotateDereferenceableBytes(CallInst *CI,
257	ArrayRef<unsigned> ArgNos,
258	uint64_t DereferenceableBytes) {
259	const Function *F = CI->getCaller();
260	if (!F)
261	return;
262	for (unsigned ArgNo : ArgNos) {
263	uint64_t DerefBytes = DereferenceableBytes;
264	unsigned AS = CI->getArgOperand(i: ArgNo)->getType()->getPointerAddressSpace();
265	if (!llvm::NullPointerIsDefined(F, AS) ||
266	CI->paramHasAttr(ArgNo, Attribute::Kind: NonNull))
267	DerefBytes = std::max(a: CI->getParamDereferenceableOrNullBytes(i: ArgNo),
268	b: DereferenceableBytes);
269
270	if (CI->getParamDereferenceableBytes(i: ArgNo) < DerefBytes) {
271	CI->removeParamAttr(ArgNo, Attribute::Dereferenceable);
272	if (!llvm::NullPointerIsDefined(F, AS) ||
273	CI->paramHasAttr(ArgNo, Attribute::Kind: NonNull))
274	CI->removeParamAttr(ArgNo, Attribute::DereferenceableOrNull);
275	CI->addParamAttr(ArgNo, Attr: Attribute::getWithDereferenceableBytes(
276	Context&: CI->getContext(), Bytes: DerefBytes));
277	}
278	}
279	}
280
281	static void annotateNonNullNoUndefBasedOnAccess(CallInst *CI,
282	ArrayRef<unsigned> ArgNos) {
283	Function *F = CI->getCaller();
284	if (!F)
285	return;
286
287	for (unsigned ArgNo : ArgNos) {
288	if (!CI->paramHasAttr(ArgNo, Attribute::Kind: NoUndef))
289	CI->addParamAttr(ArgNo, Attribute::NoUndef);
290
291	if (!CI->paramHasAttr(ArgNo, Attribute::Kind: NonNull)) {
292	unsigned AS =
293	CI->getArgOperand(i: ArgNo)->getType()->getPointerAddressSpace();
294	if (llvm::NullPointerIsDefined(F, AS))
295	continue;
296	CI->addParamAttr(ArgNo, Attribute::NonNull);
297	}
298
299	annotateDereferenceableBytes(CI, ArgNos: ArgNo, DereferenceableBytes: 1);
300	}
301	}
302
303	static void annotateNonNullAndDereferenceable(CallInst *CI, ArrayRef<unsigned> ArgNos,
304	Value *Size, const DataLayout &DL) {
305	if (ConstantInt *LenC = dyn_cast<ConstantInt>(Val: Size)) {
306	annotateNonNullNoUndefBasedOnAccess(CI, ArgNos);
307	annotateDereferenceableBytes(CI, ArgNos, DereferenceableBytes: LenC->getZExtValue());
308	} else if (isKnownNonZero(V: Size, Q: DL)) {
309	annotateNonNullNoUndefBasedOnAccess(CI, ArgNos);
310	const APInt *X, *Y;
311	uint64_t DerefMin = 1;
312	if (match(V: Size, P: m_Select(C: m_Value(), L: m_APInt(Res&: X), R: m_APInt(Res&: Y)))) {
313	DerefMin = std::min(a: X->getZExtValue(), b: Y->getZExtValue());
314	annotateDereferenceableBytes(CI, ArgNos, DereferenceableBytes: DerefMin);
315	}
316	}
317	}
318
319	// Copy CallInst "flags" like musttail, notail, and tail. Return New param for
320	// easier chaining. Calls to emit* and B.createCall should probably be wrapped
321	// in this function when New is created to replace Old. Callers should take
322	// care to check Old.isMustTailCall() if they aren't replacing Old directly
323	// with New.
324	static Value *copyFlags(const CallInst &Old, Value *New) {
325	assert(!Old.isMustTailCall() && "do not copy musttail call flags");
326	assert(!Old.isNoTailCall() && "do not copy notail call flags");
327	if (auto *NewCI = dyn_cast_or_null<CallInst>(Val: New))
328	NewCI->setTailCallKind(Old.getTailCallKind());
329	return New;
330	}
331
332	static Value *mergeAttributesAndFlags(CallInst *NewCI, const CallInst &Old) {
333	NewCI->setAttributes(AttributeList::get(
334	C&: NewCI->getContext(), Attrs: {NewCI->getAttributes(), Old.getAttributes()}));
335	NewCI->removeRetAttrs(AttrsToRemove: AttributeFuncs::typeIncompatible(Ty: NewCI->getType()));
336	return copyFlags(Old, New: NewCI);
337	}
338
339	// Helper to avoid truncating the length if size_t is 32-bits.
340	static StringRef substr(StringRef Str, uint64_t Len) {
341	return Len >= Str.size() ? Str : Str.substr(Start: 0, N: Len);
342	}
343
344	//===----------------------------------------------------------------------===//
345	// String and Memory Library Call Optimizations
346	//===----------------------------------------------------------------------===//
347
348	Value *LibCallSimplifier::optimizeStrCat(CallInst *CI, IRBuilderBase &B) {
349	// Extract some information from the instruction
350	Value *Dst = CI->getArgOperand(i: 0);
351	Value *Src = CI->getArgOperand(i: 1);
352	annotateNonNullNoUndefBasedOnAccess(CI, ArgNos: {0, 1});
353
354	// See if we can get the length of the input string.
355	uint64_t Len = GetStringLength(V: Src);
356	if (Len)
357	annotateDereferenceableBytes(CI, ArgNos: 1, DereferenceableBytes: Len);
358	else
359	return nullptr;
360	--Len; // Unbias length.
361
362	// Handle the simple, do-nothing case: strcat(x, "") -> x
363	if (Len == 0)
364	return Dst;
365
366	return copyFlags(Old: *CI, New: emitStrLenMemCpy(Src, Dst, Len, B));
367	}
368
369	Value *LibCallSimplifier::emitStrLenMemCpy(Value *Src, Value *Dst, uint64_t Len,
370	IRBuilderBase &B) {
371	// We need to find the end of the destination string. That's where the
372	// memory is to be moved to. We just generate a call to strlen.
373	Value *DstLen = emitStrLen(Ptr: Dst, B, DL, TLI);
374	if (!DstLen)
375	return nullptr;
376
377	// Now that we have the destination's length, we must index into the
378	// destination's pointer to get the actual memcpy destination (end of
379	// the string .. we're concatenating).
380	Value *CpyDst = B.CreateInBoundsGEP(Ty: B.getInt8Ty(), Ptr: Dst, IdxList: DstLen, Name: "endptr");
381
382	// We have enough information to now generate the memcpy call to do the
383	// concatenation for us. Make a memcpy to copy the nul byte with align = 1.
384	B.CreateMemCpy(
385	Dst: CpyDst, DstAlign: Align(1), Src, SrcAlign: Align(1),
386	Size: ConstantInt::get(Ty: DL.getIntPtrType(C&: Src->getContext()), V: Len + 1));
387	return Dst;
388	}
389
390	Value *LibCallSimplifier::optimizeStrNCat(CallInst *CI, IRBuilderBase &B) {
391	// Extract some information from the instruction.
392	Value *Dst = CI->getArgOperand(i: 0);
393	Value *Src = CI->getArgOperand(i: 1);
394	Value *Size = CI->getArgOperand(i: 2);
395	uint64_t Len;
396	annotateNonNullNoUndefBasedOnAccess(CI, ArgNos: 0);
397	if (isKnownNonZero(V: Size, Q: DL))
398	annotateNonNullNoUndefBasedOnAccess(CI, ArgNos: 1);
399
400	// We don't do anything if length is not constant.
401	ConstantInt *LengthArg = dyn_cast<ConstantInt>(Val: Size);
402	if (LengthArg) {
403	Len = LengthArg->getZExtValue();
404	// strncat(x, c, 0) -> x
405	if (!Len)
406	return Dst;
407	} else {
408	return nullptr;
409	}
410
411	// See if we can get the length of the input string.
412	uint64_t SrcLen = GetStringLength(V: Src);
413	if (SrcLen) {
414	annotateDereferenceableBytes(CI, ArgNos: 1, DereferenceableBytes: SrcLen);
415	--SrcLen; // Unbias length.
416	} else {
417	return nullptr;
418	}
419
420	// strncat(x, "", c) -> x
421	if (SrcLen == 0)
422	return Dst;
423
424	// We don't optimize this case.
425	if (Len < SrcLen)
426	return nullptr;
427
428	// strncat(x, s, c) -> strcat(x, s)
429	// s is constant so the strcat can be optimized further.
430	return copyFlags(Old: *CI, New: emitStrLenMemCpy(Src, Dst, Len: SrcLen, B));
431	}
432
433	// Helper to transform memchr(S, C, N) == S to N && *S == C and, when
434	// NBytes is null, strchr(S, C) to *S == C. A precondition of the function
435	// is that either S is dereferenceable or the value of N is nonzero.
436	static Value* memChrToCharCompare(CallInst *CI, Value *NBytes,
437	IRBuilderBase &B, const DataLayout &DL)
438	{
439	Value *Src = CI->getArgOperand(i: 0);
440	Value *CharVal = CI->getArgOperand(i: 1);
441
442	// Fold memchr(A, C, N) == A to N && *A == C.
443	Type *CharTy = B.getInt8Ty();
444	Value *Char0 = B.CreateLoad(Ty: CharTy, Ptr: Src);
445	CharVal = B.CreateTrunc(V: CharVal, DestTy: CharTy);
446	Value *Cmp = B.CreateICmpEQ(LHS: Char0, RHS: CharVal, Name: "char0cmp");
447
448	if (NBytes) {
449	Value *Zero = ConstantInt::get(Ty: NBytes->getType(), V: 0);
450	Value *And = B.CreateICmpNE(LHS: NBytes, RHS: Zero);
451	Cmp = B.CreateLogicalAnd(Cond1: And, Cond2: Cmp);
452	}
453
454	Value *NullPtr = Constant::getNullValue(Ty: CI->getType());
455	return B.CreateSelect(C: Cmp, True: Src, False: NullPtr);
456	}
457
458	Value *LibCallSimplifier::optimizeStrChr(CallInst *CI, IRBuilderBase &B) {
459	Value *SrcStr = CI->getArgOperand(i: 0);
460	Value *CharVal = CI->getArgOperand(i: 1);
461	annotateNonNullNoUndefBasedOnAccess(CI, ArgNos: 0);
462
463	if (isOnlyUsedInEqualityComparison(V: CI, With: SrcStr))
464	return memChrToCharCompare(CI, NBytes: nullptr, B, DL);
465
466	// If the second operand is non-constant, see if we can compute the length
467	// of the input string and turn this into memchr.
468	ConstantInt *CharC = dyn_cast<ConstantInt>(Val: CharVal);
469	if (!CharC) {
470	uint64_t Len = GetStringLength(V: SrcStr);
471	if (Len)
472	annotateDereferenceableBytes(CI, ArgNos: 0, DereferenceableBytes: Len);
473	else
474	return nullptr;
475
476	Function *Callee = CI->getCalledFunction();
477	FunctionType *FT = Callee->getFunctionType();
478	unsigned IntBits = TLI->getIntSize();
479	if (!FT->getParamType(i: 1)->isIntegerTy(Bitwidth: IntBits)) // memchr needs 'int'.
480	return nullptr;
481
482	unsigned SizeTBits = TLI->getSizeTSize(M: *CI->getModule());
483	Type *SizeTTy = IntegerType::get(C&: CI->getContext(), NumBits: SizeTBits);
484	return copyFlags(Old: *CI,
485	New: emitMemChr(Ptr: SrcStr, Val: CharVal, // include nul.
486	Len: ConstantInt::get(Ty: SizeTTy, V: Len), B,
487	DL, TLI));
488	}
489
490	if (CharC->isZero()) {
491	Value *NullPtr = Constant::getNullValue(Ty: CI->getType());
492	if (isOnlyUsedInEqualityComparison(V: CI, With: NullPtr))
493	// Pre-empt the transformation to strlen below and fold
494	// strchr(A, '\0') == null to false.
495	return B.CreateIntToPtr(V: B.getTrue(), DestTy: CI->getType());
496	}
497
498	// Otherwise, the character is a constant, see if the first argument is
499	// a string literal. If so, we can constant fold.
500	StringRef Str;
501	if (!getConstantStringInfo(V: SrcStr, Str)) {
502	if (CharC->isZero()) // strchr(p, 0) -> p + strlen(p)
503	if (Value *StrLen = emitStrLen(Ptr: SrcStr, B, DL, TLI))
504	return B.CreateInBoundsGEP(Ty: B.getInt8Ty(), Ptr: SrcStr, IdxList: StrLen, Name: "strchr");
505	return nullptr;
506	}
507
508	// Compute the offset, make sure to handle the case when we're searching for
509	// zero (a weird way to spell strlen).
510	size_t I = (0xFF & CharC->getSExtValue()) == 0
511	? Str.size()
512	: Str.find(C: CharC->getSExtValue());
513	if (I == StringRef::npos) // Didn't find the char. strchr returns null.
514	return Constant::getNullValue(Ty: CI->getType());
515
516	// strchr(s+n,c) -> gep(s+n+i,c)
517	return B.CreateInBoundsGEP(Ty: B.getInt8Ty(), Ptr: SrcStr, IdxList: B.getInt64(C: I), Name: "strchr");
518	}
519
520	Value *LibCallSimplifier::optimizeStrRChr(CallInst *CI, IRBuilderBase &B) {
521	Value *SrcStr = CI->getArgOperand(i: 0);
522	Value *CharVal = CI->getArgOperand(i: 1);
523	ConstantInt *CharC = dyn_cast<ConstantInt>(Val: CharVal);
524	annotateNonNullNoUndefBasedOnAccess(CI, ArgNos: 0);
525
526	StringRef Str;
527	if (!getConstantStringInfo(V: SrcStr, Str)) {
528	// strrchr(s, 0) -> strchr(s, 0)
529	if (CharC && CharC->isZero())
530	return copyFlags(Old: *CI, New: emitStrChr(Ptr: SrcStr, C: '\0', B, TLI));
531	return nullptr;
532	}
533
534	unsigned SizeTBits = TLI->getSizeTSize(M: *CI->getModule());
535	Type *SizeTTy = IntegerType::get(C&: CI->getContext(), NumBits: SizeTBits);
536
537	// Try to expand strrchr to the memrchr nonstandard extension if it's
538	// available, or simply fail otherwise.
539	uint64_t NBytes = Str.size() + 1; // Include the terminating nul.
540	Value *Size = ConstantInt::get(Ty: SizeTTy, V: NBytes);
541	return copyFlags(Old: *CI, New: emitMemRChr(Ptr: SrcStr, Val: CharVal, Len: Size, B, DL, TLI));
542	}
543
544	Value *LibCallSimplifier::optimizeStrCmp(CallInst *CI, IRBuilderBase &B) {
545	Value *Str1P = CI->getArgOperand(i: 0), *Str2P = CI->getArgOperand(i: 1);
546	if (Str1P == Str2P) // strcmp(x,x) -> 0
547	return ConstantInt::get(Ty: CI->getType(), V: 0);
548
549	StringRef Str1, Str2;
550	bool HasStr1 = getConstantStringInfo(V: Str1P, Str&: Str1);
551	bool HasStr2 = getConstantStringInfo(V: Str2P, Str&: Str2);
552
553	// strcmp(x, y) -> cnst (if both x and y are constant strings)
554	if (HasStr1 && HasStr2)
555	return ConstantInt::get(Ty: CI->getType(),
556	V: std::clamp(val: Str1.compare(RHS: Str2), lo: -1, hi: 1));
557
558	if (HasStr1 && Str1.empty()) // strcmp("", x) -> -*x
559	return B.CreateNeg(V: B.CreateZExt(
560	V: B.CreateLoad(Ty: B.getInt8Ty(), Ptr: Str2P, Name: "strcmpload"), DestTy: CI->getType()));
561
562	if (HasStr2 && Str2.empty()) // strcmp(x,"") -> *x
563	return B.CreateZExt(V: B.CreateLoad(Ty: B.getInt8Ty(), Ptr: Str1P, Name: "strcmpload"),
564	DestTy: CI->getType());
565
566	// strcmp(P, "x") -> memcmp(P, "x", 2)
567	uint64_t Len1 = GetStringLength(V: Str1P);
568	if (Len1)
569	annotateDereferenceableBytes(CI, ArgNos: 0, DereferenceableBytes: Len1);
570	uint64_t Len2 = GetStringLength(V: Str2P);
571	if (Len2)
572	annotateDereferenceableBytes(CI, ArgNos: 1, DereferenceableBytes: Len2);
573
574	if (Len1 && Len2) {
575	return copyFlags(
576	Old: *CI, New: emitMemCmp(Ptr1: Str1P, Ptr2: Str2P,
577	Len: ConstantInt::get(Ty: DL.getIntPtrType(C&: CI->getContext()),
578	V: std::min(a: Len1, b: Len2)),
579	B, DL, TLI));
580	}
581
582	// strcmp to memcmp
583	if (!HasStr1 && HasStr2) {
584	if (canTransformToMemCmp(CI, Str: Str1P, Len: Len2, DL))
585	return copyFlags(
586	Old: *CI,
587	New: emitMemCmp(Ptr1: Str1P, Ptr2: Str2P,
588	Len: ConstantInt::get(Ty: DL.getIntPtrType(C&: CI->getContext()), V: Len2),
589	B, DL, TLI));
590	} else if (HasStr1 && !HasStr2) {
591	if (canTransformToMemCmp(CI, Str: Str2P, Len: Len1, DL))
592	return copyFlags(
593	Old: *CI,
594	New: emitMemCmp(Ptr1: Str1P, Ptr2: Str2P,
595	Len: ConstantInt::get(Ty: DL.getIntPtrType(C&: CI->getContext()), V: Len1),
596	B, DL, TLI));
597	}
598
599	annotateNonNullNoUndefBasedOnAccess(CI, ArgNos: {0, 1});
600	return nullptr;
601	}
602
603	// Optimize a memcmp or, when StrNCmp is true, strncmp call CI with constant
604	// arrays LHS and RHS and nonconstant Size.
605	static Value *optimizeMemCmpVarSize(CallInst *CI, Value *LHS, Value *RHS,
606	Value *Size, bool StrNCmp,
607	IRBuilderBase &B, const DataLayout &DL);
608
609	Value *LibCallSimplifier::optimizeStrNCmp(CallInst *CI, IRBuilderBase &B) {
610	Value *Str1P = CI->getArgOperand(i: 0);
611	Value *Str2P = CI->getArgOperand(i: 1);
612	Value *Size = CI->getArgOperand(i: 2);
613	if (Str1P == Str2P) // strncmp(x,x,n) -> 0
614	return ConstantInt::get(Ty: CI->getType(), V: 0);
615
616	if (isKnownNonZero(V: Size, Q: DL))
617	annotateNonNullNoUndefBasedOnAccess(CI, ArgNos: {0, 1});
618	// Get the length argument if it is constant.
619	uint64_t Length;
620	if (ConstantInt *LengthArg = dyn_cast<ConstantInt>(Val: Size))
621	Length = LengthArg->getZExtValue();
622	else
623	return optimizeMemCmpVarSize(CI, LHS: Str1P, RHS: Str2P, Size, StrNCmp: true, B, DL);
624
625	if (Length == 0) // strncmp(x,y,0) -> 0
626	return ConstantInt::get(Ty: CI->getType(), V: 0);
627
628	if (Length == 1) // strncmp(x,y,1) -> memcmp(x,y,1)
629	return copyFlags(Old: *CI, New: emitMemCmp(Ptr1: Str1P, Ptr2: Str2P, Len: Size, B, DL, TLI));
630
631	StringRef Str1, Str2;
632	bool HasStr1 = getConstantStringInfo(V: Str1P, Str&: Str1);
633	bool HasStr2 = getConstantStringInfo(V: Str2P, Str&: Str2);
634
635	// strncmp(x, y) -> cnst (if both x and y are constant strings)
636	if (HasStr1 && HasStr2) {
637	// Avoid truncating the 64-bit Length to 32 bits in ILP32.
638	StringRef SubStr1 = substr(Str: Str1, Len: Length);
639	StringRef SubStr2 = substr(Str: Str2, Len: Length);
640	return ConstantInt::get(Ty: CI->getType(),
641	V: std::clamp(val: SubStr1.compare(RHS: SubStr2), lo: -1, hi: 1));
642	}
643
644	if (HasStr1 && Str1.empty()) // strncmp("", x, n) -> -*x
645	return B.CreateNeg(V: B.CreateZExt(
646	V: B.CreateLoad(Ty: B.getInt8Ty(), Ptr: Str2P, Name: "strcmpload"), DestTy: CI->getType()));
647
648	if (HasStr2 && Str2.empty()) // strncmp(x, "", n) -> *x
649	return B.CreateZExt(V: B.CreateLoad(Ty: B.getInt8Ty(), Ptr: Str1P, Name: "strcmpload"),
650	DestTy: CI->getType());
651
652	uint64_t Len1 = GetStringLength(V: Str1P);
653	if (Len1)
654	annotateDereferenceableBytes(CI, ArgNos: 0, DereferenceableBytes: Len1);
655	uint64_t Len2 = GetStringLength(V: Str2P);
656	if (Len2)
657	annotateDereferenceableBytes(CI, ArgNos: 1, DereferenceableBytes: Len2);
658
659	// strncmp to memcmp
660	if (!HasStr1 && HasStr2) {
661	Len2 = std::min(a: Len2, b: Length);
662	if (canTransformToMemCmp(CI, Str: Str1P, Len: Len2, DL))
663	return copyFlags(
664	Old: *CI,
665	New: emitMemCmp(Ptr1: Str1P, Ptr2: Str2P,
666	Len: ConstantInt::get(Ty: DL.getIntPtrType(C&: CI->getContext()), V: Len2),
667	B, DL, TLI));
668	} else if (HasStr1 && !HasStr2) {
669	Len1 = std::min(a: Len1, b: Length);
670	if (canTransformToMemCmp(CI, Str: Str2P, Len: Len1, DL))
671	return copyFlags(
672	Old: *CI,
673	New: emitMemCmp(Ptr1: Str1P, Ptr2: Str2P,
674	Len: ConstantInt::get(Ty: DL.getIntPtrType(C&: CI->getContext()), V: Len1),
675	B, DL, TLI));
676	}
677
678	return nullptr;
679	}
680
681	Value *LibCallSimplifier::optimizeStrNDup(CallInst *CI, IRBuilderBase &B) {
682	Value *Src = CI->getArgOperand(i: 0);
683	ConstantInt *Size = dyn_cast<ConstantInt>(Val: CI->getArgOperand(i: 1));
684	uint64_t SrcLen = GetStringLength(V: Src);
685	if (SrcLen && Size) {
686	annotateDereferenceableBytes(CI, ArgNos: 0, DereferenceableBytes: SrcLen);
687	if (SrcLen <= Size->getZExtValue() + 1)
688	return copyFlags(Old: *CI, New: emitStrDup(Ptr: Src, B, TLI));
689	}
690
691	return nullptr;
692	}
693
694	Value *LibCallSimplifier::optimizeStrCpy(CallInst *CI, IRBuilderBase &B) {
695	Value *Dst = CI->getArgOperand(i: 0), *Src = CI->getArgOperand(i: 1);
696	if (Dst == Src) // strcpy(x,x) -> x
697	return Src;
698
699	annotateNonNullNoUndefBasedOnAccess(CI, ArgNos: {0, 1});
700	// See if we can get the length of the input string.
701	uint64_t Len = GetStringLength(V: Src);
702	if (Len)
703	annotateDereferenceableBytes(CI, ArgNos: 1, DereferenceableBytes: Len);
704	else
705	return nullptr;
706
707	// We have enough information to now generate the memcpy call to do the
708	// copy for us. Make a memcpy to copy the nul byte with align = 1.
709	CallInst *NewCI =
710	B.CreateMemCpy(Dst, DstAlign: Align(1), Src, SrcAlign: Align(1),
711	Size: ConstantInt::get(Ty: DL.getIntPtrType(C&: CI->getContext()), V: Len));
712	mergeAttributesAndFlags(NewCI, Old: *CI);
713	return Dst;
714	}
715
716	Value *LibCallSimplifier::optimizeStpCpy(CallInst *CI, IRBuilderBase &B) {
717	Function *Callee = CI->getCalledFunction();
718	Value *Dst = CI->getArgOperand(i: 0), *Src = CI->getArgOperand(i: 1);
719
720	// stpcpy(d,s) -> strcpy(d,s) if the result is not used.
721	if (CI->use_empty())
722	return copyFlags(Old: *CI, New: emitStrCpy(Dst, Src, B, TLI));
723
724	if (Dst == Src) { // stpcpy(x,x) -> x+strlen(x)
725	Value *StrLen = emitStrLen(Ptr: Src, B, DL, TLI);
726	return StrLen ? B.CreateInBoundsGEP(Ty: B.getInt8Ty(), Ptr: Dst, IdxList: StrLen) : nullptr;
727	}
728
729	// See if we can get the length of the input string.
730	uint64_t Len = GetStringLength(V: Src);
731	if (Len)
732	annotateDereferenceableBytes(CI, ArgNos: 1, DereferenceableBytes: Len);
733	else
734	return nullptr;
735
736	Type *PT = Callee->getFunctionType()->getParamType(i: 0);
737	Value *LenV = ConstantInt::get(Ty: DL.getIntPtrType(PT), V: Len);
738	Value *DstEnd = B.CreateInBoundsGEP(
739	Ty: B.getInt8Ty(), Ptr: Dst, IdxList: ConstantInt::get(Ty: DL.getIntPtrType(PT), V: Len - 1));
740
741	// We have enough information to now generate the memcpy call to do the
742	// copy for us. Make a memcpy to copy the nul byte with align = 1.
743	CallInst *NewCI = B.CreateMemCpy(Dst, DstAlign: Align(1), Src, SrcAlign: Align(1), Size: LenV);
744	mergeAttributesAndFlags(NewCI, Old: *CI);
745	return DstEnd;
746	}
747
748	// Optimize a call to size_t strlcpy(char*, const char*, size_t).
749
750	Value *LibCallSimplifier::optimizeStrLCpy(CallInst *CI, IRBuilderBase &B) {
751	Value *Size = CI->getArgOperand(i: 2);
752	if (isKnownNonZero(V: Size, Q: DL))
753	// Like snprintf, the function stores into the destination only when
754	// the size argument is nonzero.
755	annotateNonNullNoUndefBasedOnAccess(CI, ArgNos: 0);
756	// The function reads the source argument regardless of Size (it returns
757	// its length).
758	annotateNonNullNoUndefBasedOnAccess(CI, ArgNos: 1);
759
760	uint64_t NBytes;
761	if (ConstantInt *SizeC = dyn_cast<ConstantInt>(Val: Size))
762	NBytes = SizeC->getZExtValue();
763	else
764	return nullptr;
765
766	Value *Dst = CI->getArgOperand(i: 0);
767	Value *Src = CI->getArgOperand(i: 1);
768	if (NBytes <= 1) {
769	if (NBytes == 1)
770	// For a call to strlcpy(D, S, 1) first store a nul in *D.
771	B.CreateStore(Val: B.getInt8(C: 0), Ptr: Dst);
772
773	// Transform strlcpy(D, S, 0) to a call to strlen(S).
774	return copyFlags(Old: *CI, New: emitStrLen(Ptr: Src, B, DL, TLI));
775	}
776
777	// Try to determine the length of the source, substituting its size
778	// when it's not nul-terminated (as it's required to be) to avoid
779	// reading past its end.
780	StringRef Str;
781	if (!getConstantStringInfo(V: Src, Str, /*TrimAtNul=*/false))
782	return nullptr;
783
784	uint64_t SrcLen = Str.find(C: '\0');
785	// Set if the terminating nul should be copied by the call to memcpy
786	// below.
787	bool NulTerm = SrcLen < NBytes;
788
789	if (NulTerm)
790	// Overwrite NBytes with the number of bytes to copy, including
791	// the terminating nul.
792	NBytes = SrcLen + 1;
793	else {
794	// Set the length of the source for the function to return to its
795	// size, and cap NBytes at the same.
796	SrcLen = std::min(a: SrcLen, b: uint64_t(Str.size()));
797	NBytes = std::min(a: NBytes - 1, b: SrcLen);
798	}
799
800	if (SrcLen == 0) {
801	// Transform strlcpy(D, "", N) to (*D = '\0, 0).
802	B.CreateStore(Val: B.getInt8(C: 0), Ptr: Dst);
803	return ConstantInt::get(Ty: CI->getType(), V: 0);
804	}
805
806	Function *Callee = CI->getCalledFunction();
807	Type *PT = Callee->getFunctionType()->getParamType(i: 0);
808	// Transform strlcpy(D, S, N) to memcpy(D, S, N') where N' is the lower
809	// bound on strlen(S) + 1 and N, optionally followed by a nul store to
810	// D[N' - 1] if necessary.
811	CallInst *NewCI = B.CreateMemCpy(Dst, DstAlign: Align(1), Src, SrcAlign: Align(1),
812	Size: ConstantInt::get(Ty: DL.getIntPtrType(PT), V: NBytes));
813	mergeAttributesAndFlags(NewCI, Old: *CI);
814
815	if (!NulTerm) {
816	Value *EndOff = ConstantInt::get(Ty: CI->getType(), V: NBytes);
817	Value *EndPtr = B.CreateInBoundsGEP(Ty: B.getInt8Ty(), Ptr: Dst, IdxList: EndOff);
818	B.CreateStore(Val: B.getInt8(C: 0), Ptr: EndPtr);
819	}
820
821	// Like snprintf, strlcpy returns the number of nonzero bytes that would
822	// have been copied if the bound had been sufficiently big (which in this
823	// case is strlen(Src)).
824	return ConstantInt::get(Ty: CI->getType(), V: SrcLen);
825	}
826
827	// Optimize a call CI to either stpncpy when RetEnd is true, or to strncpy
828	// otherwise.
829	Value *LibCallSimplifier::optimizeStringNCpy(CallInst *CI, bool RetEnd,
830	IRBuilderBase &B) {
831	Function *Callee = CI->getCalledFunction();
832	Value *Dst = CI->getArgOperand(i: 0);
833	Value *Src = CI->getArgOperand(i: 1);
834	Value *Size = CI->getArgOperand(i: 2);
835
836	if (isKnownNonZero(V: Size, Q: DL)) {
837	// Both st{p,r}ncpy(D, S, N) access the source and destination arrays
838	// only when N is nonzero.
839	annotateNonNullNoUndefBasedOnAccess(CI, ArgNos: 0);
840	annotateNonNullNoUndefBasedOnAccess(CI, ArgNos: 1);
841	}
842
843	// If the "bound" argument is known set N to it. Otherwise set it to
844	// UINT64_MAX and handle it later.
845	uint64_t N = UINT64_MAX;
846	if (ConstantInt *SizeC = dyn_cast<ConstantInt>(Val: Size))
847	N = SizeC->getZExtValue();
848
849	if (N == 0)
850	// Fold st{p,r}ncpy(D, S, 0) to D.
851	return Dst;
852
853	if (N == 1) {
854	Type *CharTy = B.getInt8Ty();
855	Value *CharVal = B.CreateLoad(Ty: CharTy, Ptr: Src, Name: "stxncpy.char0");
856	B.CreateStore(Val: CharVal, Ptr: Dst);
857	if (!RetEnd)
858	// Transform strncpy(D, S, 1) to return (*D = *S), D.
859	return Dst;
860
861	// Transform stpncpy(D, S, 1) to return (*D = *S) ? D + 1 : D.
862	Value *ZeroChar = ConstantInt::get(Ty: CharTy, V: 0);
863	Value *Cmp = B.CreateICmpEQ(LHS: CharVal, RHS: ZeroChar, Name: "stpncpy.char0cmp");
864
865	Value *Off1 = B.getInt32(C: 1);
866	Value *EndPtr = B.CreateInBoundsGEP(Ty: CharTy, Ptr: Dst, IdxList: Off1, Name: "stpncpy.end");
867	return B.CreateSelect(C: Cmp, True: Dst, False: EndPtr, Name: "stpncpy.sel");
868	}
869
870	// If the length of the input string is known set SrcLen to it.
871	uint64_t SrcLen = GetStringLength(V: Src);
872	if (SrcLen)
873	annotateDereferenceableBytes(CI, ArgNos: 1, DereferenceableBytes: SrcLen);
874	else
875	return nullptr;
876
877	--SrcLen; // Unbias length.
878
879	if (SrcLen == 0) {
880	// Transform st{p,r}ncpy(D, "", N) to memset(D, '\0', N) for any N.
881	Align MemSetAlign =
882	CI->getAttributes().getParamAttrs(ArgNo: 0).getAlignment().valueOrOne();
883	CallInst *NewCI = B.CreateMemSet(Ptr: Dst, Val: B.getInt8(C: '\0'), Size, Align: MemSetAlign);
884	AttrBuilder ArgAttrs(CI->getContext(), CI->getAttributes().getParamAttrs(ArgNo: 0));
885	NewCI->setAttributes(NewCI->getAttributes().addParamAttributes(
886	C&: CI->getContext(), ArgNo: 0, B: ArgAttrs));
887	copyFlags(Old: *CI, New: NewCI);
888	return Dst;
889	}
890
891	if (N > SrcLen + 1) {
892	if (N > 128)
893	// Bail if N is large or unknown.
894	return nullptr;
895
896	// st{p,r}ncpy(D, "a", N) -> memcpy(D, "a\0\0\0", N) for N <= 128.
897	StringRef Str;
898	if (!getConstantStringInfo(V: Src, Str))
899	return nullptr;
900	std::string SrcStr = Str.str();
901	// Create a bigger, nul-padded array with the same length, SrcLen,
902	// as the original string.
903	SrcStr.resize(n: N, c: '\0');
904	Src = B.CreateGlobalString(Str: SrcStr, Name: "str");
905	}
906
907	Type *PT = Callee->getFunctionType()->getParamType(i: 0);
908	// st{p,r}ncpy(D, S, N) -> memcpy(align 1 D, align 1 S, N) when both
909	// S and N are constant.
910	CallInst *NewCI = B.CreateMemCpy(Dst, DstAlign: Align(1), Src, SrcAlign: Align(1),
911	Size: ConstantInt::get(Ty: DL.getIntPtrType(PT), V: N));
912	mergeAttributesAndFlags(NewCI, Old: *CI);
913	if (!RetEnd)
914	return Dst;
915
916	// stpncpy(D, S, N) returns the address of the first null in D if it writes
917	// one, otherwise D + N.
918	Value *Off = B.getInt64(C: std::min(a: SrcLen, b: N));
919	return B.CreateInBoundsGEP(Ty: B.getInt8Ty(), Ptr: Dst, IdxList: Off, Name: "endptr");
920	}
921
922	Value *LibCallSimplifier::optimizeStringLength(CallInst *CI, IRBuilderBase &B,
923	unsigned CharSize,
924	Value *Bound) {
925	Value *Src = CI->getArgOperand(i: 0);
926	Type *CharTy = B.getIntNTy(N: CharSize);
927
928	if (isOnlyUsedInZeroEqualityComparison(CxtI: CI) &&
929	(!Bound || isKnownNonZero(V: Bound, Q: DL))) {
930	// Fold strlen:
931	// strlen(x) != 0 --> *x != 0
932	// strlen(x) == 0 --> *x == 0
933	// and likewise strnlen with constant N > 0:
934	// strnlen(x, N) != 0 --> *x != 0
935	// strnlen(x, N) == 0 --> *x == 0
936	return B.CreateZExt(V: B.CreateLoad(Ty: CharTy, Ptr: Src, Name: "char0"),
937	DestTy: CI->getType());
938	}
939
940	if (Bound) {
941	if (ConstantInt *BoundCst = dyn_cast<ConstantInt>(Val: Bound)) {
942	if (BoundCst->isZero())
943	// Fold strnlen(s, 0) -> 0 for any s, constant or otherwise.
944	return ConstantInt::get(Ty: CI->getType(), V: 0);
945
946	if (BoundCst->isOne()) {
947	// Fold strnlen(s, 1) -> *s ? 1 : 0 for any s.
948	Value *CharVal = B.CreateLoad(Ty: CharTy, Ptr: Src, Name: "strnlen.char0");
949	Value *ZeroChar = ConstantInt::get(Ty: CharTy, V: 0);
950	Value *Cmp = B.CreateICmpNE(LHS: CharVal, RHS: ZeroChar, Name: "strnlen.char0cmp");
951	return B.CreateZExt(V: Cmp, DestTy: CI->getType());
952	}
953	}
954	}
955
956	if (uint64_t Len = GetStringLength(V: Src, CharSize)) {
957	Value *LenC = ConstantInt::get(Ty: CI->getType(), V: Len - 1);
958	// Fold strlen("xyz") -> 3 and strnlen("xyz", 2) -> 2
959	// and strnlen("xyz", Bound) -> min(3, Bound) for nonconstant Bound.
960	if (Bound)
961	return B.CreateBinaryIntrinsic(Intrinsic::ID: umin, LHS: LenC, RHS: Bound);
962	return LenC;
963	}
964
965	if (Bound)
966	// Punt for strnlen for now.
967	return nullptr;
968
969	// If s is a constant pointer pointing to a string literal, we can fold
970	// strlen(s + x) to strlen(s) - x, when x is known to be in the range
971	// [0, strlen(s)] or the string has a single null terminator '\0' at the end.
972	// We only try to simplify strlen when the pointer s points to an array
973	// of CharSize elements. Otherwise, we would need to scale the offset x before
974	// doing the subtraction. This will make the optimization more complex, and
975	// it's not very useful because calling strlen for a pointer of other types is
976	// very uncommon.
977	if (GEPOperator *GEP = dyn_cast<GEPOperator>(Val: Src)) {
978	// TODO: Handle subobjects.
979	if (!isGEPBasedOnPointerToString(GEP, CharSize))
980	return nullptr;
981
982	ConstantDataArraySlice Slice;
983	if (getConstantDataArrayInfo(V: GEP->getOperand(i_nocapture: 0), Slice, ElementSize: CharSize)) {
984	uint64_t NullTermIdx;
985	if (Slice.Array == nullptr) {
986	NullTermIdx = 0;
987	} else {
988	NullTermIdx = ~((uint64_t)0);
989	for (uint64_t I = 0, E = Slice.Length; I < E; ++I) {
990	if (Slice.Array->getElementAsInteger(i: I + Slice.Offset) == 0) {
991	NullTermIdx = I;
992	break;
993	}
994	}
995	// If the string does not have '\0', leave it to strlen to compute
996	// its length.
997	if (NullTermIdx == ~((uint64_t)0))
998	return nullptr;
999	}
1000
1001	Value *Offset = GEP->getOperand(i_nocapture: 2);
1002	KnownBits Known = computeKnownBits(V: Offset, DL, Depth: 0, AC: nullptr, CxtI: CI, DT: nullptr);
1003	uint64_t ArrSize =
1004	cast<ArrayType>(Val: GEP->getSourceElementType())->getNumElements();
1005
1006	// If Offset is not provably in the range [0, NullTermIdx], we can still
1007	// optimize if we can prove that the program has undefined behavior when
1008	// Offset is outside that range. That is the case when GEP->getOperand(0)
1009	// is a pointer to an object whose memory extent is NullTermIdx+1.
1010	if ((Known.isNonNegative() && Known.getMaxValue().ule(RHS: NullTermIdx)) ||
1011	(isa<GlobalVariable>(Val: GEP->getOperand(i_nocapture: 0)) &&
1012	NullTermIdx == ArrSize - 1)) {
1013	Offset = B.CreateSExtOrTrunc(V: Offset, DestTy: CI->getType());
1014	return B.CreateSub(LHS: ConstantInt::get(Ty: CI->getType(), V: NullTermIdx),
1015	RHS: Offset);
1016	}
1017	}
1018	}
1019
1020	// strlen(x?"foo":"bars") --> x ? 3 : 4
1021	if (SelectInst *SI = dyn_cast<SelectInst>(Val: Src)) {
1022	uint64_t LenTrue = GetStringLength(V: SI->getTrueValue(), CharSize);
1023	uint64_t LenFalse = GetStringLength(V: SI->getFalseValue(), CharSize);
1024	if (LenTrue && LenFalse) {
1025	ORE.emit(RemarkBuilder: [&]() {
1026	return OptimizationRemark("instcombine", "simplify-libcalls", CI)
1027	<< "folded strlen(select) to select of constants";
1028	});
1029	return B.CreateSelect(C: SI->getCondition(),
1030	True: ConstantInt::get(Ty: CI->getType(), V: LenTrue - 1),
1031	False: ConstantInt::get(Ty: CI->getType(), V: LenFalse - 1));
1032	}
1033	}
1034
1035	return nullptr;
1036	}
1037
1038	Value *LibCallSimplifier::optimizeStrLen(CallInst *CI, IRBuilderBase &B) {
1039	if (Value *V = optimizeStringLength(CI, B, CharSize: 8))
1040	return V;
1041	annotateNonNullNoUndefBasedOnAccess(CI, ArgNos: 0);
1042	return nullptr;
1043	}
1044
1045	Value *LibCallSimplifier::optimizeStrNLen(CallInst *CI, IRBuilderBase &B) {
1046	Value *Bound = CI->getArgOperand(i: 1);
1047	if (Value *V = optimizeStringLength(CI, B, CharSize: 8, Bound))
1048	return V;
1049
1050	if (isKnownNonZero(V: Bound, Q: DL))
1051	annotateNonNullNoUndefBasedOnAccess(CI, ArgNos: 0);
1052	return nullptr;
1053	}
1054
1055	Value *LibCallSimplifier::optimizeWcslen(CallInst *CI, IRBuilderBase &B) {
1056	Module &M = *CI->getModule();
1057	unsigned WCharSize = TLI->getWCharSize(M) * 8;
1058	// We cannot perform this optimization without wchar_size metadata.
1059	if (WCharSize == 0)
1060	return nullptr;
1061
1062	return optimizeStringLength(CI, B, CharSize: WCharSize);
1063	}
1064
1065	Value *LibCallSimplifier::optimizeStrPBrk(CallInst *CI, IRBuilderBase &B) {
1066	StringRef S1, S2;
1067	bool HasS1 = getConstantStringInfo(V: CI->getArgOperand(i: 0), Str&: S1);
1068	bool HasS2 = getConstantStringInfo(V: CI->getArgOperand(i: 1), Str&: S2);
1069
1070	// strpbrk(s, "") -> nullptr
1071	// strpbrk("", s) -> nullptr
1072	if ((HasS1 && S1.empty()) || (HasS2 && S2.empty()))
1073	return Constant::getNullValue(Ty: CI->getType());
1074
1075	// Constant folding.
1076	if (HasS1 && HasS2) {
1077	size_t I = S1.find_first_of(Chars: S2);
1078	if (I == StringRef::npos) // No match.
1079	return Constant::getNullValue(Ty: CI->getType());
1080
1081	return B.CreateInBoundsGEP(Ty: B.getInt8Ty(), Ptr: CI->getArgOperand(i: 0),
1082	IdxList: B.getInt64(C: I), Name: "strpbrk");
1083	}
1084
1085	// strpbrk(s, "a") -> strchr(s, 'a')
1086	if (HasS2 && S2.size() == 1)
1087	return copyFlags(Old: *CI, New: emitStrChr(Ptr: CI->getArgOperand(i: 0), C: S2[0], B, TLI));
1088
1089	return nullptr;
1090	}
1091
1092	Value *LibCallSimplifier::optimizeStrTo(CallInst *CI, IRBuilderBase &B) {
1093	Value *EndPtr = CI->getArgOperand(i: 1);
1094	if (isa<ConstantPointerNull>(Val: EndPtr)) {
1095	// With a null EndPtr, this function won't capture the main argument.
1096	// It would be readonly too, except that it still may write to errno.
1097	CI->addParamAttr(0, Attribute::NoCapture);
1098	}
1099
1100	return nullptr;
1101	}
1102
1103	Value *LibCallSimplifier::optimizeStrSpn(CallInst *CI, IRBuilderBase &B) {
1104	StringRef S1, S2;
1105	bool HasS1 = getConstantStringInfo(V: CI->getArgOperand(i: 0), Str&: S1);
1106	bool HasS2 = getConstantStringInfo(V: CI->getArgOperand(i: 1), Str&: S2);
1107
1108	// strspn(s, "") -> 0
1109	// strspn("", s) -> 0
1110	if ((HasS1 && S1.empty()) || (HasS2 && S2.empty()))
1111	return Constant::getNullValue(Ty: CI->getType());
1112
1113	// Constant folding.
1114	if (HasS1 && HasS2) {
1115	size_t Pos = S1.find_first_not_of(Chars: S2);
1116	if (Pos == StringRef::npos)
1117	Pos = S1.size();
1118	return ConstantInt::get(Ty: CI->getType(), V: Pos);
1119	}
1120
1121	return nullptr;
1122	}
1123
1124	Value *LibCallSimplifier::optimizeStrCSpn(CallInst *CI, IRBuilderBase &B) {
1125	StringRef S1, S2;
1126	bool HasS1 = getConstantStringInfo(V: CI->getArgOperand(i: 0), Str&: S1);
1127	bool HasS2 = getConstantStringInfo(V: CI->getArgOperand(i: 1), Str&: S2);
1128
1129	// strcspn("", s) -> 0
1130	if (HasS1 && S1.empty())
1131	return Constant::getNullValue(Ty: CI->getType());
1132
1133	// Constant folding.
1134	if (HasS1 && HasS2) {
1135	size_t Pos = S1.find_first_of(Chars: S2);
1136	if (Pos == StringRef::npos)
1137	Pos = S1.size();
1138	return ConstantInt::get(Ty: CI->getType(), V: Pos);
1139	}
1140
1141	// strcspn(s, "") -> strlen(s)
1142	if (HasS2 && S2.empty())
1143	return copyFlags(Old: *CI, New: emitStrLen(Ptr: CI->getArgOperand(i: 0), B, DL, TLI));
1144
1145	return nullptr;
1146	}
1147
1148	Value *LibCallSimplifier::optimizeStrStr(CallInst *CI, IRBuilderBase &B) {
1149	// fold strstr(x, x) -> x.
1150	if (CI->getArgOperand(i: 0) == CI->getArgOperand(i: 1))
1151	return CI->getArgOperand(i: 0);
1152
1153	// fold strstr(a, b) == a -> strncmp(a, b, strlen(b)) == 0
1154	if (isOnlyUsedInEqualityComparison(V: CI, With: CI->getArgOperand(i: 0))) {
1155	Value *StrLen = emitStrLen(Ptr: CI->getArgOperand(i: 1), B, DL, TLI);
1156	if (!StrLen)
1157	return nullptr;
1158	Value *StrNCmp = emitStrNCmp(Ptr1: CI->getArgOperand(i: 0), Ptr2: CI->getArgOperand(i: 1),
1159	Len: StrLen, B, DL, TLI);
1160	if (!StrNCmp)
1161	return nullptr;
1162	for (User *U : llvm::make_early_inc_range(Range: CI->users())) {
1163	ICmpInst *Old = cast<ICmpInst>(Val: U);
1164	Value *Cmp =
1165	B.CreateICmp(P: Old->getPredicate(), LHS: StrNCmp,
1166	RHS: ConstantInt::getNullValue(Ty: StrNCmp->getType()), Name: "cmp");
1167	replaceAllUsesWith(I: Old, With: Cmp);
1168	}
1169	return CI;
1170	}
1171
1172	// See if either input string is a constant string.
1173	StringRef SearchStr, ToFindStr;
1174	bool HasStr1 = getConstantStringInfo(V: CI->getArgOperand(i: 0), Str&: SearchStr);
1175	bool HasStr2 = getConstantStringInfo(V: CI->getArgOperand(i: 1), Str&: ToFindStr);
1176
1177	// fold strstr(x, "") -> x.
1178	if (HasStr2 && ToFindStr.empty())
1179	return CI->getArgOperand(i: 0);
1180
1181	// If both strings are known, constant fold it.
1182	if (HasStr1 && HasStr2) {
1183	size_t Offset = SearchStr.find(Str: ToFindStr);
1184
1185	if (Offset == StringRef::npos) // strstr("foo", "bar") -> null
1186	return Constant::getNullValue(Ty: CI->getType());
1187
1188	// strstr("abcd", "bc") -> gep((char*)"abcd", 1)
1189	return B.CreateConstInBoundsGEP1_64(Ty: B.getInt8Ty(), Ptr: CI->getArgOperand(i: 0),
1190	Idx0: Offset, Name: "strstr");
1191	}
1192
1193	// fold strstr(x, "y") -> strchr(x, 'y').
1194	if (HasStr2 && ToFindStr.size() == 1) {
1195	return emitStrChr(Ptr: CI->getArgOperand(i: 0), C: ToFindStr[0], B, TLI);
1196	}
1197
1198	annotateNonNullNoUndefBasedOnAccess(CI, ArgNos: {0, 1});
1199	return nullptr;
1200	}
1201
1202	Value *LibCallSimplifier::optimizeMemRChr(CallInst *CI, IRBuilderBase &B) {
1203	Value *SrcStr = CI->getArgOperand(i: 0);
1204	Value *Size = CI->getArgOperand(i: 2);
1205	annotateNonNullAndDereferenceable(CI, ArgNos: 0, Size, DL);
1206	Value *CharVal = CI->getArgOperand(i: 1);
1207	ConstantInt *LenC = dyn_cast<ConstantInt>(Val: Size);
1208	Value *NullPtr = Constant::getNullValue(Ty: CI->getType());
1209
1210	if (LenC) {
1211	if (LenC->isZero())
1212	// Fold memrchr(x, y, 0) --> null.
1213	return NullPtr;
1214
1215	if (LenC->isOne()) {
1216	// Fold memrchr(x, y, 1) --> *x == y ? x : null for any x and y,
1217	// constant or otherwise.
1218	Value *Val = B.CreateLoad(Ty: B.getInt8Ty(), Ptr: SrcStr, Name: "memrchr.char0");
1219	// Slice off the character's high end bits.
1220	CharVal = B.CreateTrunc(V: CharVal, DestTy: B.getInt8Ty());
1221	Value *Cmp = B.CreateICmpEQ(LHS: Val, RHS: CharVal, Name: "memrchr.char0cmp");
1222	return B.CreateSelect(C: Cmp, True: SrcStr, False: NullPtr, Name: "memrchr.sel");
1223	}
1224	}
1225
1226	StringRef Str;
1227	if (!getConstantStringInfo(V: SrcStr, Str, /*TrimAtNul=*/false))
1228	return nullptr;
1229
1230	if (Str.size() == 0)
1231	// If the array is empty fold memrchr(A, C, N) to null for any value
1232	// of C and N on the basis that the only valid value of N is zero
1233	// (otherwise the call is undefined).
1234	return NullPtr;
1235
1236	uint64_t EndOff = UINT64_MAX;
1237	if (LenC) {
1238	EndOff = LenC->getZExtValue();
1239	if (Str.size() < EndOff)
1240	// Punt out-of-bounds accesses to sanitizers and/or libc.
1241	return nullptr;
1242	}
1243
1244	if (ConstantInt *CharC = dyn_cast<ConstantInt>(Val: CharVal)) {
1245	// Fold memrchr(S, C, N) for a constant C.
1246	size_t Pos = Str.rfind(C: CharC->getZExtValue(), From: EndOff);
1247	if (Pos == StringRef::npos)
1248	// When the character is not in the source array fold the result
1249	// to null regardless of Size.
1250	return NullPtr;
1251
1252	if (LenC)
1253	// Fold memrchr(s, c, N) --> s + Pos for constant N > Pos.
1254	return B.CreateInBoundsGEP(Ty: B.getInt8Ty(), Ptr: SrcStr, IdxList: B.getInt64(C: Pos));
1255
1256	if (Str.find(C: Str[Pos]) == Pos) {
1257	// When there is just a single occurrence of C in S, i.e., the one
1258	// in Str[Pos], fold
1259	// memrchr(s, c, N) --> N <= Pos ? null : s + Pos
1260	// for nonconstant N.
1261	Value *Cmp = B.CreateICmpULE(LHS: Size, RHS: ConstantInt::get(Ty: Size->getType(), V: Pos),
1262	Name: "memrchr.cmp");
1263	Value *SrcPlus = B.CreateInBoundsGEP(Ty: B.getInt8Ty(), Ptr: SrcStr,
1264	IdxList: B.getInt64(C: Pos), Name: "memrchr.ptr_plus");
1265	return B.CreateSelect(C: Cmp, True: NullPtr, False: SrcPlus, Name: "memrchr.sel");
1266	}
1267	}
1268
1269	// Truncate the string to search at most EndOff characters.
1270	Str = Str.substr(Start: 0, N: EndOff);
1271	if (Str.find_first_not_of(C: Str[0]) != StringRef::npos)
1272	return nullptr;
1273
1274	// If the source array consists of all equal characters, then for any
1275	// C and N (whether in bounds or not), fold memrchr(S, C, N) to
1276	// N != 0 && *S == C ? S + N - 1 : null
1277	Type *SizeTy = Size->getType();
1278	Type *Int8Ty = B.getInt8Ty();
1279	Value *NNeZ = B.CreateICmpNE(LHS: Size, RHS: ConstantInt::get(Ty: SizeTy, V: 0));
1280	// Slice off the sought character's high end bits.
1281	CharVal = B.CreateTrunc(V: CharVal, DestTy: Int8Ty);
1282	Value *CEqS0 = B.CreateICmpEQ(LHS: ConstantInt::get(Ty: Int8Ty, V: Str[0]), RHS: CharVal);
1283	Value *And = B.CreateLogicalAnd(Cond1: NNeZ, Cond2: CEqS0);
1284	Value *SizeM1 = B.CreateSub(LHS: Size, RHS: ConstantInt::get(Ty: SizeTy, V: 1));
1285	Value *SrcPlus =
1286	B.CreateInBoundsGEP(Ty: Int8Ty, Ptr: SrcStr, IdxList: SizeM1, Name: "memrchr.ptr_plus");
1287	return B.CreateSelect(C: And, True: SrcPlus, False: NullPtr, Name: "memrchr.sel");
1288	}
1289
1290	Value *LibCallSimplifier::optimizeMemChr(CallInst *CI, IRBuilderBase &B) {
1291	Value *SrcStr = CI->getArgOperand(i: 0);
1292	Value *Size = CI->getArgOperand(i: 2);
1293
1294	if (isKnownNonZero(V: Size, Q: DL)) {
1295	annotateNonNullNoUndefBasedOnAccess(CI, ArgNos: 0);
1296	if (isOnlyUsedInEqualityComparison(V: CI, With: SrcStr))
1297	return memChrToCharCompare(CI, NBytes: Size, B, DL);
1298	}
1299
1300	Value *CharVal = CI->getArgOperand(i: 1);
1301	ConstantInt *CharC = dyn_cast<ConstantInt>(Val: CharVal);
1302	ConstantInt *LenC = dyn_cast<ConstantInt>(Val: Size);
1303	Value *NullPtr = Constant::getNullValue(Ty: CI->getType());
1304
1305	// memchr(x, y, 0) -> null
1306	if (LenC) {
1307	if (LenC->isZero())
1308	return NullPtr;
1309
1310	if (LenC->isOne()) {
1311	// Fold memchr(x, y, 1) --> *x == y ? x : null for any x and y,
1312	// constant or otherwise.
1313	Value *Val = B.CreateLoad(Ty: B.getInt8Ty(), Ptr: SrcStr, Name: "memchr.char0");
1314	// Slice off the character's high end bits.
1315	CharVal = B.CreateTrunc(V: CharVal, DestTy: B.getInt8Ty());
1316	Value *Cmp = B.CreateICmpEQ(LHS: Val, RHS: CharVal, Name: "memchr.char0cmp");
1317	return B.CreateSelect(C: Cmp, True: SrcStr, False: NullPtr, Name: "memchr.sel");
1318	}
1319	}
1320
1321	StringRef Str;
1322	if (!getConstantStringInfo(V: SrcStr, Str, /*TrimAtNul=*/false))
1323	return nullptr;
1324
1325	if (CharC) {
1326	size_t Pos = Str.find(C: CharC->getZExtValue());
1327	if (Pos == StringRef::npos)
1328	// When the character is not in the source array fold the result
1329	// to null regardless of Size.
1330	return NullPtr;
1331
1332	// Fold memchr(s, c, n) -> n <= Pos ? null : s + Pos
1333	// When the constant Size is less than or equal to the character
1334	// position also fold the result to null.
1335	Value *Cmp = B.CreateICmpULE(LHS: Size, RHS: ConstantInt::get(Ty: Size->getType(), V: Pos),
1336	Name: "memchr.cmp");
1337	Value *SrcPlus = B.CreateInBoundsGEP(Ty: B.getInt8Ty(), Ptr: SrcStr, IdxList: B.getInt64(C: Pos),
1338	Name: "memchr.ptr");
1339	return B.CreateSelect(C: Cmp, True: NullPtr, False: SrcPlus);
1340	}
1341
1342	if (Str.size() == 0)
1343	// If the array is empty fold memchr(A, C, N) to null for any value
1344	// of C and N on the basis that the only valid value of N is zero
1345	// (otherwise the call is undefined).
1346	return NullPtr;
1347
1348	if (LenC)
1349	Str = substr(Str, Len: LenC->getZExtValue());
1350
1351	size_t Pos = Str.find_first_not_of(C: Str[0]);
1352	if (Pos == StringRef::npos
1353	|| Str.find_first_not_of(C: Str[Pos], From: Pos) == StringRef::npos) {
1354	// If the source array consists of at most two consecutive sequences
1355	// of the same characters, then for any C and N (whether in bounds or
1356	// not), fold memchr(S, C, N) to
1357	// N != 0 && *S == C ? S : null
1358	// or for the two sequences to:
1359	// N != 0 && *S == C ? S : (N > Pos && S[Pos] == C ? S + Pos : null)
1360	// ^Sel2 ^Sel1 are denoted above.
1361	// The latter makes it also possible to fold strchr() calls with strings
1362	// of the same characters.
1363	Type *SizeTy = Size->getType();
1364	Type *Int8Ty = B.getInt8Ty();
1365
1366	// Slice off the sought character's high end bits.
1367	CharVal = B.CreateTrunc(V: CharVal, DestTy: Int8Ty);
1368
1369	Value *Sel1 = NullPtr;
1370	if (Pos != StringRef::npos) {
1371	// Handle two consecutive sequences of the same characters.
1372	Value *PosVal = ConstantInt::get(Ty: SizeTy, V: Pos);
1373	Value *StrPos = ConstantInt::get(Ty: Int8Ty, V: Str[Pos]);
1374	Value *CEqSPos = B.CreateICmpEQ(LHS: CharVal, RHS: StrPos);
1375	Value *NGtPos = B.CreateICmp(P: ICmpInst::ICMP_UGT, LHS: Size, RHS: PosVal);
1376	Value *And = B.CreateAnd(LHS: CEqSPos, RHS: NGtPos);
1377	Value *SrcPlus = B.CreateInBoundsGEP(Ty: B.getInt8Ty(), Ptr: SrcStr, IdxList: PosVal);
1378	Sel1 = B.CreateSelect(C: And, True: SrcPlus, False: NullPtr, Name: "memchr.sel1");
1379	}
1380
1381	Value *Str0 = ConstantInt::get(Ty: Int8Ty, V: Str[0]);
1382	Value *CEqS0 = B.CreateICmpEQ(LHS: Str0, RHS: CharVal);
1383	Value *NNeZ = B.CreateICmpNE(LHS: Size, RHS: ConstantInt::get(Ty: SizeTy, V: 0));
1384	Value *And = B.CreateAnd(LHS: NNeZ, RHS: CEqS0);
1385	return B.CreateSelect(C: And, True: SrcStr, False: Sel1, Name: "memchr.sel2");
1386	}
1387
1388	if (!LenC) {
1389	if (isOnlyUsedInEqualityComparison(V: CI, With: SrcStr))
1390	// S is dereferenceable so it's safe to load from it and fold
1391	// memchr(S, C, N) == S to N && *S == C for any C and N.
1392	// TODO: This is safe even for nonconstant S.
1393	return memChrToCharCompare(CI, NBytes: Size, B, DL);
1394
1395	// From now on we need a constant length and constant array.
1396	return nullptr;
1397	}
1398
1399	bool OptForSize = CI->getFunction()->hasOptSize() ||
1400	llvm::shouldOptimizeForSize(BB: CI->getParent(), PSI, BFI,
1401	QueryType: PGSOQueryType::IRPass);
1402
1403	// If the char is variable but the input str and length are not we can turn
1404	// this memchr call into a simple bit field test. Of course this only works
1405	// when the return value is only checked against null.
1406	//
1407	// It would be really nice to reuse switch lowering here but we can't change
1408	// the CFG at this point.
1409	//
1410	// memchr("\r\n", C, 2) != nullptr -> (1 << C & ((1 << '\r') | (1 << '\n')))
1411	// != 0
1412	// after bounds check.
1413	if (OptForSize || Str.empty() || !isOnlyUsedInZeroEqualityComparison(CxtI: CI))
1414	return nullptr;
1415
1416	unsigned char Max =
1417	*std::max_element(first: reinterpret_cast<const unsigned char *>(Str.begin()),
1418	last: reinterpret_cast<const unsigned char *>(Str.end()));
1419
1420	// Make sure the bit field we're about to create fits in a register on the
1421	// target.
1422	// FIXME: On a 64 bit architecture this prevents us from using the
1423	// interesting range of alpha ascii chars. We could do better by emitting
1424	// two bitfields or shifting the range by 64 if no lower chars are used.
1425	if (!DL.fitsInLegalInteger(Width: Max + 1)) {
1426	// Build chain of ORs
1427	// Transform:
1428	// memchr("abcd", C, 4) != nullptr
1429	// to:
1430	// (C == 'a' || C == 'b' || C == 'c' || C == 'd') != 0
1431	std::string SortedStr = Str.str();
1432	llvm::sort(C&: SortedStr);
1433	// Compute the number of of non-contiguous ranges.
1434	unsigned NonContRanges = 1;
1435	for (size_t i = 1; i < SortedStr.size(); ++i) {
1436	if (SortedStr[i] > SortedStr[i - 1] + 1) {
1437	NonContRanges++;
1438	}
1439	}
1440
1441	// Restrict this optimization to profitable cases with one or two range
1442	// checks.
1443	if (NonContRanges > 2)
1444	return nullptr;
1445
1446	SmallVector<Value *> CharCompares;
1447	for (unsigned char C : SortedStr)
1448	CharCompares.push_back(
1449	Elt: B.CreateICmpEQ(LHS: CharVal, RHS: ConstantInt::get(Ty: CharVal->getType(), V: C)));
1450
1451	return B.CreateIntToPtr(V: B.CreateOr(Ops: CharCompares), DestTy: CI->getType());
1452	}
1453
1454	// For the bit field use a power-of-2 type with at least 8 bits to avoid
1455	// creating unnecessary illegal types.
1456	unsigned char Width = NextPowerOf2(A: std::max(a: (unsigned char)7, b: Max));
1457
1458	// Now build the bit field.
1459	APInt Bitfield(Width, 0);
1460	for (char C : Str)
1461	Bitfield.setBit((unsigned char)C);
1462	Value *BitfieldC = B.getInt(AI: Bitfield);
1463
1464	// Adjust width of "C" to the bitfield width, then mask off the high bits.
1465	Value *C = B.CreateZExtOrTrunc(V: CharVal, DestTy: BitfieldC->getType());
1466	C = B.CreateAnd(LHS: C, RHS: B.getIntN(N: Width, C: 0xFF));
1467
1468	// First check that the bit field access is within bounds.
1469	Value *Bounds = B.CreateICmp(P: ICmpInst::ICMP_ULT, LHS: C, RHS: B.getIntN(N: Width, C: Width),
1470	Name: "memchr.bounds");
1471
1472	// Create code that checks if the given bit is set in the field.
1473	Value *Shl = B.CreateShl(LHS: B.getIntN(N: Width, C: 1ULL), RHS: C);
1474	Value *Bits = B.CreateIsNotNull(Arg: B.CreateAnd(LHS: Shl, RHS: BitfieldC), Name: "memchr.bits");
1475
1476	// Finally merge both checks and cast to pointer type. The inttoptr
1477	// implicitly zexts the i1 to intptr type.
1478	return B.CreateIntToPtr(V: B.CreateLogicalAnd(Cond1: Bounds, Cond2: Bits, Name: "memchr"),
1479	DestTy: CI->getType());
1480	}
1481
1482	// Optimize a memcmp or, when StrNCmp is true, strncmp call CI with constant
1483	// arrays LHS and RHS and nonconstant Size.
1484	static Value *optimizeMemCmpVarSize(CallInst *CI, Value *LHS, Value *RHS,
1485	Value *Size, bool StrNCmp,
1486	IRBuilderBase &B, const DataLayout &DL) {
1487	if (LHS == RHS) // memcmp(s,s,x) -> 0
1488	return Constant::getNullValue(Ty: CI->getType());
1489
1490	StringRef LStr, RStr;
1491	if (!getConstantStringInfo(V: LHS, Str&: LStr, /*TrimAtNul=*/false) ||
1492	!getConstantStringInfo(V: RHS, Str&: RStr, /*TrimAtNul=*/false))
1493	return nullptr;
1494
1495	// If the contents of both constant arrays are known, fold a call to
1496	// memcmp(A, B, N) to
1497	// N <= Pos ? 0 : (A < B ? -1 : B < A ? +1 : 0)
1498	// where Pos is the first mismatch between A and B, determined below.
1499
1500	uint64_t Pos = 0;
1501	Value *Zero = ConstantInt::get(Ty: CI->getType(), V: 0);
1502	for (uint64_t MinSize = std::min(a: LStr.size(), b: RStr.size()); ; ++Pos) {
1503	if (Pos == MinSize ||
1504	(StrNCmp && (LStr[Pos] == '\0' && RStr[Pos] == '\0'))) {
1505	// One array is a leading part of the other of equal or greater
1506	// size, or for strncmp, the arrays are equal strings.
1507	// Fold the result to zero. Size is assumed to be in bounds, since
1508	// otherwise the call would be undefined.
1509	return Zero;
1510	}
1511
1512	if (LStr[Pos] != RStr[Pos])
1513	break;
1514	}
1515
1516	// Normalize the result.
1517	typedef unsigned char UChar;
1518	int IRes = UChar(LStr[Pos]) < UChar(RStr[Pos]) ? -1 : 1;
1519	Value *MaxSize = ConstantInt::get(Ty: Size->getType(), V: Pos);
1520	Value *Cmp = B.CreateICmp(P: ICmpInst::ICMP_ULE, LHS: Size, RHS: MaxSize);
1521	Value *Res = ConstantInt::get(Ty: CI->getType(), V: IRes);
1522	return B.CreateSelect(C: Cmp, True: Zero, False: Res);
1523	}
1524
1525	// Optimize a memcmp call CI with constant size Len.
1526	static Value *optimizeMemCmpConstantSize(CallInst *CI, Value *LHS, Value *RHS,
1527	uint64_t Len, IRBuilderBase &B,
1528	const DataLayout &DL) {
1529	if (Len == 0) // memcmp(s1,s2,0) -> 0
1530	return Constant::getNullValue(Ty: CI->getType());
1531
1532	// memcmp(S1,S2,1) -> *(unsigned char*)LHS - *(unsigned char*)RHS
1533	if (Len == 1) {
1534	Value *LHSV = B.CreateZExt(V: B.CreateLoad(Ty: B.getInt8Ty(), Ptr: LHS, Name: "lhsc"),
1535	DestTy: CI->getType(), Name: "lhsv");
1536	Value *RHSV = B.CreateZExt(V: B.CreateLoad(Ty: B.getInt8Ty(), Ptr: RHS, Name: "rhsc"),
1537	DestTy: CI->getType(), Name: "rhsv");
1538	return B.CreateSub(LHS: LHSV, RHS: RHSV, Name: "chardiff");
1539	}
1540
1541	// memcmp(S1,S2,N/8)==0 -> (*(intN_t*)S1 != *(intN_t*)S2)==0
1542	// TODO: The case where both inputs are constants does not need to be limited
1543	// to legal integers or equality comparison. See block below this.
1544	if (DL.isLegalInteger(Width: Len * 8) && isOnlyUsedInZeroEqualityComparison(CxtI: CI)) {
1545	IntegerType *IntType = IntegerType::get(C&: CI->getContext(), NumBits: Len * 8);
1546	Align PrefAlignment = DL.getPrefTypeAlign(Ty: IntType);
1547
1548	// First, see if we can fold either argument to a constant.
1549	Value *LHSV = nullptr;
1550	if (auto *LHSC = dyn_cast<Constant>(Val: LHS))
1551	LHSV = ConstantFoldLoadFromConstPtr(C: LHSC, Ty: IntType, DL);
1552
1553	Value *RHSV = nullptr;
1554	if (auto *RHSC = dyn_cast<Constant>(Val: RHS))
1555	RHSV = ConstantFoldLoadFromConstPtr(C: RHSC, Ty: IntType, DL);
1556
1557	// Don't generate unaligned loads. If either source is constant data,
1558	// alignment doesn't matter for that source because there is no load.
1559	if ((LHSV || getKnownAlignment(V: LHS, DL, CxtI: CI) >= PrefAlignment) &&
1560	(RHSV || getKnownAlignment(V: RHS, DL, CxtI: CI) >= PrefAlignment)) {
1561	if (!LHSV)
1562	LHSV = B.CreateLoad(Ty: IntType, Ptr: LHS, Name: "lhsv");
1563	if (!RHSV)
1564	RHSV = B.CreateLoad(Ty: IntType, Ptr: RHS, Name: "rhsv");
1565	return B.CreateZExt(V: B.CreateICmpNE(LHS: LHSV, RHS: RHSV), DestTy: CI->getType(), Name: "memcmp");
1566	}
1567	}
1568
1569	return nullptr;
1570	}
1571
1572	// Most simplifications for memcmp also apply to bcmp.
1573	Value *LibCallSimplifier::optimizeMemCmpBCmpCommon(CallInst *CI,
1574	IRBuilderBase &B) {
1575	Value *LHS = CI->getArgOperand(i: 0), *RHS = CI->getArgOperand(i: 1);
1576	Value *Size = CI->getArgOperand(i: 2);
1577
1578	annotateNonNullAndDereferenceable(CI, ArgNos: {0, 1}, Size, DL);
1579
1580	if (Value *Res = optimizeMemCmpVarSize(CI, LHS, RHS, Size, StrNCmp: false, B, DL))
1581	return Res;
1582
1583	// Handle constant Size.
1584	ConstantInt *LenC = dyn_cast<ConstantInt>(Val: Size);
1585	if (!LenC)
1586	return nullptr;
1587
1588	return optimizeMemCmpConstantSize(CI, LHS, RHS, Len: LenC->getZExtValue(), B, DL);
1589	}
1590
1591	Value *LibCallSimplifier::optimizeMemCmp(CallInst *CI, IRBuilderBase &B) {
1592	Module *M = CI->getModule();
1593	if (Value *V = optimizeMemCmpBCmpCommon(CI, B))
1594	return V;
1595
1596	// memcmp(x, y, Len) == 0 -> bcmp(x, y, Len) == 0
1597	// bcmp can be more efficient than memcmp because it only has to know that
1598	// there is a difference, not how different one is to the other.
1599	if (isLibFuncEmittable(M, TLI, TheLibFunc: LibFunc_bcmp) &&
1600	isOnlyUsedInZeroEqualityComparison(CxtI: CI)) {
1601	Value *LHS = CI->getArgOperand(i: 0);
1602	Value *RHS = CI->getArgOperand(i: 1);
1603	Value *Size = CI->getArgOperand(i: 2);
1604	return copyFlags(Old: *CI, New: emitBCmp(Ptr1: LHS, Ptr2: RHS, Len: Size, B, DL, TLI));
1605	}
1606
1607	return nullptr;
1608	}
1609
1610	Value *LibCallSimplifier::optimizeBCmp(CallInst *CI, IRBuilderBase &B) {
1611	return optimizeMemCmpBCmpCommon(CI, B);
1612	}
1613
1614	Value *LibCallSimplifier::optimizeMemCpy(CallInst *CI, IRBuilderBase &B) {
1615	Value *Size = CI->getArgOperand(i: 2);
1616	annotateNonNullAndDereferenceable(CI, ArgNos: {0, 1}, Size, DL);
1617	if (isa<IntrinsicInst>(Val: CI))
1618	return nullptr;
1619
1620	// memcpy(x, y, n) -> llvm.memcpy(align 1 x, align 1 y, n)
1621	CallInst *NewCI = B.CreateMemCpy(Dst: CI->getArgOperand(i: 0), DstAlign: Align(1),
1622	Src: CI->getArgOperand(i: 1), SrcAlign: Align(1), Size);
1623	mergeAttributesAndFlags(NewCI, Old: *CI);
1624	return CI->getArgOperand(i: 0);
1625	}
1626
1627	Value *LibCallSimplifier::optimizeMemCCpy(CallInst *CI, IRBuilderBase &B) {
1628	Value *Dst = CI->getArgOperand(i: 0);
1629	Value *Src = CI->getArgOperand(i: 1);
1630	ConstantInt *StopChar = dyn_cast<ConstantInt>(Val: CI->getArgOperand(i: 2));
1631	ConstantInt *N = dyn_cast<ConstantInt>(Val: CI->getArgOperand(i: 3));
1632	StringRef SrcStr;
1633	if (CI->use_empty() && Dst == Src)
1634	return Dst;
1635	// memccpy(d, s, c, 0) -> nullptr
1636	if (N) {
1637	if (N->isNullValue())
1638	return Constant::getNullValue(Ty: CI->getType());
1639	if (!getConstantStringInfo(V: Src, Str&: SrcStr, /*TrimAtNul=*/false) ||
1640	// TODO: Handle zeroinitializer.
1641	!StopChar)
1642	return nullptr;
1643	} else {
1644	return nullptr;
1645	}
1646
1647	// Wrap arg 'c' of type int to char
1648	size_t Pos = SrcStr.find(C: StopChar->getSExtValue() & 0xFF);
1649	if (Pos == StringRef::npos) {
1650	if (N->getZExtValue() <= SrcStr.size()) {
1651	copyFlags(Old: *CI, New: B.CreateMemCpy(Dst, DstAlign: Align(1), Src, SrcAlign: Align(1),
1652	Size: CI->getArgOperand(i: 3)));
1653	return Constant::getNullValue(Ty: CI->getType());
1654	}
1655	return nullptr;
1656	}
1657
1658	Value *NewN =
1659	ConstantInt::get(Ty: N->getType(), V: std::min(a: uint64_t(Pos + 1), b: N->getZExtValue()));
1660	// memccpy -> llvm.memcpy
1661	copyFlags(Old: *CI, New: B.CreateMemCpy(Dst, DstAlign: Align(1), Src, SrcAlign: Align(1), Size: NewN));
1662	return Pos + 1 <= N->getZExtValue()
1663	? B.CreateInBoundsGEP(Ty: B.getInt8Ty(), Ptr: Dst, IdxList: NewN)
1664	: Constant::getNullValue(Ty: CI->getType());
1665	}
1666
1667	Value *LibCallSimplifier::optimizeMemPCpy(CallInst *CI, IRBuilderBase &B) {
1668	Value *Dst = CI->getArgOperand(i: 0);
1669	Value *N = CI->getArgOperand(i: 2);
1670	// mempcpy(x, y, n) -> llvm.memcpy(align 1 x, align 1 y, n), x + n
1671	CallInst *NewCI =
1672	B.CreateMemCpy(Dst, DstAlign: Align(1), Src: CI->getArgOperand(i: 1), SrcAlign: Align(1), Size: N);
1673	// Propagate attributes, but memcpy has no return value, so make sure that
1674	// any return attributes are compliant.
1675	// TODO: Attach return value attributes to the 1st operand to preserve them?
1676	mergeAttributesAndFlags(NewCI, Old: *CI);
1677	return B.CreateInBoundsGEP(Ty: B.getInt8Ty(), Ptr: Dst, IdxList: N);
1678	}
1679
1680	Value *LibCallSimplifier::optimizeMemMove(CallInst *CI, IRBuilderBase &B) {
1681	Value *Size = CI->getArgOperand(i: 2);
1682	annotateNonNullAndDereferenceable(CI, ArgNos: {0, 1}, Size, DL);
1683	if (isa<IntrinsicInst>(Val: CI))
1684	return nullptr;
1685
1686	// memmove(x, y, n) -> llvm.memmove(align 1 x, align 1 y, n)
1687	CallInst *NewCI = B.CreateMemMove(Dst: CI->getArgOperand(i: 0), DstAlign: Align(1),
1688	Src: CI->getArgOperand(i: 1), SrcAlign: Align(1), Size);
1689	mergeAttributesAndFlags(NewCI, Old: *CI);
1690	return CI->getArgOperand(i: 0);
1691	}
1692
1693	Value *LibCallSimplifier::optimizeMemSet(CallInst *CI, IRBuilderBase &B) {
1694	Value *Size = CI->getArgOperand(i: 2);
1695	annotateNonNullAndDereferenceable(CI, ArgNos: 0, Size, DL);
1696	if (isa<IntrinsicInst>(Val: CI))
1697	return nullptr;
1698
1699	// memset(p, v, n) -> llvm.memset(align 1 p, v, n)
1700	Value *Val = B.CreateIntCast(V: CI->getArgOperand(i: 1), DestTy: B.getInt8Ty(), isSigned: false);
1701	CallInst *NewCI = B.CreateMemSet(Ptr: CI->getArgOperand(i: 0), Val, Size, Align: Align(1));
1702	mergeAttributesAndFlags(NewCI, Old: *CI);
1703	return CI->getArgOperand(i: 0);
1704	}
1705
1706	Value *LibCallSimplifier::optimizeRealloc(CallInst *CI, IRBuilderBase &B) {
1707	if (isa<ConstantPointerNull>(Val: CI->getArgOperand(i: 0)))
1708	return copyFlags(Old: *CI, New: emitMalloc(Num: CI->getArgOperand(i: 1), B, DL, TLI));
1709
1710	return nullptr;
1711	}
1712
1713	// When enabled, replace operator new() calls marked with a hot or cold memprof
1714	// attribute with an operator new() call that takes a __hot_cold_t parameter.
1715	// Currently this is supported by the open source version of tcmalloc, see:
1716	// https://github.com/google/tcmalloc/blob/master/tcmalloc/new_extension.h
1717	Value *LibCallSimplifier::optimizeNew(CallInst *CI, IRBuilderBase &B,
1718	LibFunc &Func) {
1719	if (!OptimizeHotColdNew)
1720	return nullptr;
1721
1722	uint8_t HotCold;
1723	if (CI->getAttributes().getFnAttr(Kind: "memprof").getValueAsString() == "cold")
1724	HotCold = ColdNewHintValue;
1725	else if (CI->getAttributes().getFnAttr(Kind: "memprof").getValueAsString() == "hot")
1726	HotCold = HotNewHintValue;
1727	else
1728	return nullptr;
1729
1730	switch (Func) {
1731	case LibFunc_Znwm:
1732	return emitHotColdNew(Num: CI->getArgOperand(i: 0), B, TLI,
1733	NewFunc: LibFunc_Znwm12__hot_cold_t, HotCold);
1734	case LibFunc_Znam:
1735	return emitHotColdNew(Num: CI->getArgOperand(i: 0), B, TLI,
1736	NewFunc: LibFunc_Znam12__hot_cold_t, HotCold);
1737	case LibFunc_ZnwmRKSt9nothrow_t:
1738	return emitHotColdNewNoThrow(Num: CI->getArgOperand(i: 0), NoThrow: CI->getArgOperand(i: 1), B,
1739	TLI, NewFunc: LibFunc_ZnwmRKSt9nothrow_t12__hot_cold_t,
1740	HotCold);
1741	case LibFunc_ZnamRKSt9nothrow_t:
1742	return emitHotColdNewNoThrow(Num: CI->getArgOperand(i: 0), NoThrow: CI->getArgOperand(i: 1), B,
1743	TLI, NewFunc: LibFunc_ZnamRKSt9nothrow_t12__hot_cold_t,
1744	HotCold);
1745	case LibFunc_ZnwmSt11align_val_t:
1746	return emitHotColdNewAligned(Num: CI->getArgOperand(i: 0), Align: CI->getArgOperand(i: 1), B,
1747	TLI, NewFunc: LibFunc_ZnwmSt11align_val_t12__hot_cold_t,
1748	HotCold);
1749	case LibFunc_ZnamSt11align_val_t:
1750	return emitHotColdNewAligned(Num: CI->getArgOperand(i: 0), Align: CI->getArgOperand(i: 1), B,
1751	TLI, NewFunc: LibFunc_ZnamSt11align_val_t12__hot_cold_t,
1752	HotCold);
1753	case LibFunc_ZnwmSt11align_val_tRKSt9nothrow_t:
1754	return emitHotColdNewAlignedNoThrow(
1755	Num: CI->getArgOperand(i: 0), Align: CI->getArgOperand(i: 1), NoThrow: CI->getArgOperand(i: 2), B,
1756	TLI, NewFunc: LibFunc_ZnwmSt11align_val_tRKSt9nothrow_t12__hot_cold_t, HotCold);
1757	case LibFunc_ZnamSt11align_val_tRKSt9nothrow_t:
1758	return emitHotColdNewAlignedNoThrow(
1759	Num: CI->getArgOperand(i: 0), Align: CI->getArgOperand(i: 1), NoThrow: CI->getArgOperand(i: 2), B,
1760	TLI, NewFunc: LibFunc_ZnamSt11align_val_tRKSt9nothrow_t12__hot_cold_t, HotCold);
1761	default:
1762	return nullptr;
1763	}
1764	}
1765
1766	//===----------------------------------------------------------------------===//
1767	// Math Library Optimizations
1768	//===----------------------------------------------------------------------===//
1769
1770	// Replace a libcall \p CI with a call to intrinsic \p IID
1771	static Value *replaceUnaryCall(CallInst *CI, IRBuilderBase &B,
1772	Intrinsic::ID IID) {
1773	// Propagate fast-math flags from the existing call to the new call.
1774	IRBuilderBase::FastMathFlagGuard Guard(B);
1775	B.setFastMathFlags(CI->getFastMathFlags());
1776
1777	Module *M = CI->getModule();
1778	Value *V = CI->getArgOperand(i: 0);
1779	Function *F = Intrinsic::getDeclaration(M, id: IID, Tys: CI->getType());
1780	CallInst *NewCall = B.CreateCall(Callee: F, Args: V);
1781	NewCall->takeName(V: CI);
1782	return copyFlags(Old: *CI, New: NewCall);
1783	}
1784
1785	/// Return a variant of Val with float type.
1786	/// Currently this works in two cases: If Val is an FPExtension of a float
1787	/// value to something bigger, simply return the operand.
1788	/// If Val is a ConstantFP but can be converted to a float ConstantFP without
1789	/// loss of precision do so.
1790	static Value *valueHasFloatPrecision(Value *Val) {
1791	if (FPExtInst *Cast = dyn_cast<FPExtInst>(Val)) {
1792	Value *Op = Cast->getOperand(i_nocapture: 0);
1793	if (Op->getType()->isFloatTy())
1794	return Op;
1795	}
1796	if (ConstantFP *Const = dyn_cast<ConstantFP>(Val)) {
1797	APFloat F = Const->getValueAPF();
1798	bool losesInfo;
1799	(void)F.convert(ToSemantics: APFloat::IEEEsingle(), RM: APFloat::rmNearestTiesToEven,
1800	losesInfo: &losesInfo);
1801	if (!losesInfo)
1802	return ConstantFP::get(Context&: Const->getContext(), V: F);
1803	}
1804	return nullptr;
1805	}
1806
1807	/// Shrink double -> float functions.
1808	static Value *optimizeDoubleFP(CallInst *CI, IRBuilderBase &B,
1809	bool isBinary, const TargetLibraryInfo *TLI,
1810	bool isPrecise = false) {
1811	Function *CalleeFn = CI->getCalledFunction();
1812	if (!CI->getType()->isDoubleTy() || !CalleeFn)
1813	return nullptr;
1814
1815	// If not all the uses of the function are converted to float, then bail out.
1816	// This matters if the precision of the result is more important than the
1817	// precision of the arguments.
1818	if (isPrecise)
1819	for (User *U : CI->users()) {
1820	FPTruncInst *Cast = dyn_cast<FPTruncInst>(Val: U);
1821	if (!Cast || !Cast->getType()->isFloatTy())
1822	return nullptr;
1823	}
1824
1825	// If this is something like 'g((double) float)', convert to 'gf(float)'.
1826	Value *V[2];
1827	V[0] = valueHasFloatPrecision(Val: CI->getArgOperand(i: 0));
1828	V[1] = isBinary ? valueHasFloatPrecision(Val: CI->getArgOperand(i: 1)) : nullptr;
1829	if (!V[0] || (isBinary && !V[1]))
1830	return nullptr;
1831
1832	// If call isn't an intrinsic, check that it isn't within a function with the
1833	// same name as the float version of this call, otherwise the result is an
1834	// infinite loop. For example, from MinGW-w64:
1835	//
1836	// float expf(float val) { return (float) exp((double) val); }
1837	StringRef CalleeName = CalleeFn->getName();
1838	bool IsIntrinsic = CalleeFn->isIntrinsic();
1839	if (!IsIntrinsic) {
1840	StringRef CallerName = CI->getFunction()->getName();
1841	if (!CallerName.empty() && CallerName.back() == 'f' &&
1842	CallerName.size() == (CalleeName.size() + 1) &&
1843	CallerName.starts_with(Prefix: CalleeName))
1844	return nullptr;
1845	}
1846
1847	// Propagate the math semantics from the current function to the new function.
1848	IRBuilderBase::FastMathFlagGuard Guard(B);
1849	B.setFastMathFlags(CI->getFastMathFlags());
1850
1851	// g((double) float) -> (double) gf(float)
1852	Value *R;
1853	if (IsIntrinsic) {
1854	Module *M = CI->getModule();
1855	Intrinsic::ID IID = CalleeFn->getIntrinsicID();
1856	Function *Fn = Intrinsic::getDeclaration(M, id: IID, Tys: B.getFloatTy());
1857	R = isBinary ? B.CreateCall(Callee: Fn, Args: V) : B.CreateCall(Callee: Fn, Args: V[0]);
1858	} else {
1859	AttributeList CalleeAttrs = CalleeFn->getAttributes();
1860	R = isBinary ? emitBinaryFloatFnCall(Op1: V[0], Op2: V[1], TLI, Name: CalleeName, B,
1861	Attrs: CalleeAttrs)
1862	: emitUnaryFloatFnCall(Op: V[0], TLI, Name: CalleeName, B, Attrs: CalleeAttrs);
1863	}
1864	return B.CreateFPExt(V: R, DestTy: B.getDoubleTy());
1865	}
1866
1867	/// Shrink double -> float for unary functions.
1868	static Value *optimizeUnaryDoubleFP(CallInst *CI, IRBuilderBase &B,
1869	const TargetLibraryInfo *TLI,
1870	bool isPrecise = false) {
1871	return optimizeDoubleFP(CI, B, isBinary: false, TLI, isPrecise);
1872	}
1873
1874	/// Shrink double -> float for binary functions.
1875	static Value *optimizeBinaryDoubleFP(CallInst *CI, IRBuilderBase &B,
1876	const TargetLibraryInfo *TLI,
1877	bool isPrecise = false) {
1878	return optimizeDoubleFP(CI, B, isBinary: true, TLI, isPrecise);
1879	}
1880
1881	// cabs(z) -> sqrt((creal(z)*creal(z)) + (cimag(z)*cimag(z)))
1882	Value *LibCallSimplifier::optimizeCAbs(CallInst *CI, IRBuilderBase &B) {
1883	if (!CI->isFast())
1884	return nullptr;
1885
1886	// Propagate fast-math flags from the existing call to new instructions.
1887	IRBuilderBase::FastMathFlagGuard Guard(B);
1888	B.setFastMathFlags(CI->getFastMathFlags());
1889
1890	Value *Real, *Imag;
1891	if (CI->arg_size() == 1) {
1892	Value *Op = CI->getArgOperand(i: 0);
1893	assert(Op->getType()->isArrayTy() && "Unexpected signature for cabs!");
1894	Real = B.CreateExtractValue(Agg: Op, Idxs: 0, Name: "real");
1895	Imag = B.CreateExtractValue(Agg: Op, Idxs: 1, Name: "imag");
1896	} else {
1897	assert(CI->arg_size() == 2 && "Unexpected signature for cabs!");
1898	Real = CI->getArgOperand(i: 0);
1899	Imag = CI->getArgOperand(i: 1);
1900	}
1901
1902	Value *RealReal = B.CreateFMul(L: Real, R: Real);
1903	Value *ImagImag = B.CreateFMul(L: Imag, R: Imag);
1904
1905	Function *FSqrt = Intrinsic::getDeclaration(M: CI->getModule(), Intrinsic::id: sqrt,
1906	Tys: CI->getType());
1907	return copyFlags(
1908	Old: *CI, New: B.CreateCall(Callee: FSqrt, Args: B.CreateFAdd(L: RealReal, R: ImagImag), Name: "cabs"));
1909	}
1910
1911	// Return a properly extended integer (DstWidth bits wide) if the operation is
1912	// an itofp.
1913	static Value *getIntToFPVal(Value *I2F, IRBuilderBase &B, unsigned DstWidth) {
1914	if (isa<SIToFPInst>(Val: I2F) || isa<UIToFPInst>(Val: I2F)) {
1915	Value *Op = cast<Instruction>(Val: I2F)->getOperand(i: 0);
1916	// Make sure that the exponent fits inside an "int" of size DstWidth,
1917	// thus avoiding any range issues that FP has not.
1918	unsigned BitWidth = Op->getType()->getPrimitiveSizeInBits();
1919	if (BitWidth < DstWidth ||
1920	(BitWidth == DstWidth && isa<SIToFPInst>(Val: I2F)))
1921	return isa<SIToFPInst>(Val: I2F) ? B.CreateSExt(V: Op, DestTy: B.getIntNTy(N: DstWidth))
1922	: B.CreateZExt(V: Op, DestTy: B.getIntNTy(N: DstWidth));
1923	}
1924
1925	return nullptr;
1926	}
1927
1928	/// Use exp{,2}(x * y) for pow(exp{,2}(x), y);
1929	/// ldexp(1.0, x) for pow(2.0, itofp(x)); exp2(n * x) for pow(2.0 ** n, x);
1930	/// exp10(x) for pow(10.0, x); exp2(log2(n) * x) for pow(n, x).
1931	Value *LibCallSimplifier::replacePowWithExp(CallInst *Pow, IRBuilderBase &B) {
1932	Module *M = Pow->getModule();
1933	Value *Base = Pow->getArgOperand(i: 0), *Expo = Pow->getArgOperand(i: 1);
1934	Module *Mod = Pow->getModule();
1935	Type *Ty = Pow->getType();
1936	bool Ignored;
1937
1938	// Evaluate special cases related to a nested function as the base.
1939
1940	// pow(exp(x), y) -> exp(x * y)
1941	// pow(exp2(x), y) -> exp2(x * y)
1942	// If exp{,2}() is used only once, it is better to fold two transcendental
1943	// math functions into one. If used again, exp{,2}() would still have to be
1944	// called with the original argument, then keep both original transcendental
1945	// functions. However, this transformation is only safe with fully relaxed
1946	// math semantics, since, besides rounding differences, it changes overflow
1947	// and underflow behavior quite dramatically. For example:
1948	// pow(exp(1000), 0.001) = pow(inf, 0.001) = inf
1949	// Whereas:
1950	// exp(1000 * 0.001) = exp(1)
1951	// TODO: Loosen the requirement for fully relaxed math semantics.
1952	// TODO: Handle exp10() when more targets have it available.
1953	CallInst *BaseFn = dyn_cast<CallInst>(Val: Base);
1954	if (BaseFn && BaseFn->hasOneUse() && BaseFn->isFast() && Pow->isFast()) {
1955	LibFunc LibFn;
1956
1957	Function *CalleeFn = BaseFn->getCalledFunction();
1958	if (CalleeFn && TLI->getLibFunc(funcName: CalleeFn->getName(), F&: LibFn) &&
1959	isLibFuncEmittable(M, TLI, TheLibFunc: LibFn)) {
1960	StringRef ExpName;
1961	Intrinsic::ID ID;
1962	Value *ExpFn;
1963	LibFunc LibFnFloat, LibFnDouble, LibFnLongDouble;
1964
1965	switch (LibFn) {
1966	default:
1967	return nullptr;
1968	case LibFunc_expf:
1969	case LibFunc_exp:
1970	case LibFunc_expl:
1971	ExpName = TLI->getName(F: LibFunc_exp);
1972	ID = Intrinsic::exp;
1973	LibFnFloat = LibFunc_expf;
1974	LibFnDouble = LibFunc_exp;
1975	LibFnLongDouble = LibFunc_expl;
1976	break;
1977	case LibFunc_exp2f:
1978	case LibFunc_exp2:
1979	case LibFunc_exp2l:
1980	ExpName = TLI->getName(F: LibFunc_exp2);
1981	ID = Intrinsic::exp2;
1982	LibFnFloat = LibFunc_exp2f;
1983	LibFnDouble = LibFunc_exp2;
1984	LibFnLongDouble = LibFunc_exp2l;
1985	break;
1986	}
1987
1988	// Create new exp{,2}() with the product as its argument.
1989	Value *FMul = B.CreateFMul(L: BaseFn->getArgOperand(i: 0), R: Expo, Name: "mul");
1990	ExpFn = BaseFn->doesNotAccessMemory()
1991	? B.CreateCall(Callee: Intrinsic::getDeclaration(M: Mod, id: ID, Tys: Ty),
1992	Args: FMul, Name: ExpName)
1993	: emitUnaryFloatFnCall(Op: FMul, TLI, DoubleFn: LibFnDouble, FloatFn: LibFnFloat,
1994	LongDoubleFn: LibFnLongDouble, B,
1995	Attrs: BaseFn->getAttributes());
1996
1997	// Since the new exp{,2}() is different from the original one, dead code
1998	// elimination cannot be trusted to remove it, since it may have side
1999	// effects (e.g., errno). When the only consumer for the original
2000	// exp{,2}() is pow(), then it has to be explicitly erased.
2001	substituteInParent(I: BaseFn, With: ExpFn);
2002	return ExpFn;
2003	}
2004	}
2005
2006	// Evaluate special cases related to a constant base.
2007
2008	const APFloat *BaseF;
2009	if (!match(V: Pow->getArgOperand(i: 0), P: m_APFloat(Res&: BaseF)))
2010	return nullptr;
2011
2012	AttributeList NoAttrs; // Attributes are only meaningful on the original call
2013
2014	// pow(2.0, itofp(x)) -> ldexp(1.0, x)
2015	// TODO: This does not work for vectors because there is no ldexp intrinsic.
2016	if (!Ty->isVectorTy() && match(V: Base, P: m_SpecificFP(V: 2.0)) &&
2017	(isa<SIToFPInst>(Val: Expo) || isa<UIToFPInst>(Val: Expo)) &&
2018	hasFloatFn(M, TLI, Ty, DoubleFn: LibFunc_ldexp, FloatFn: LibFunc_ldexpf, LongDoubleFn: LibFunc_ldexpl)) {
2019	if (Value *ExpoI = getIntToFPVal(I2F: Expo, B, DstWidth: TLI->getIntSize()))
2020	return copyFlags(Old: *Pow,
2021	New: emitBinaryFloatFnCall(Op1: ConstantFP::get(Ty, V: 1.0), Op2: ExpoI,
2022	TLI, DoubleFn: LibFunc_ldexp, FloatFn: LibFunc_ldexpf,
2023	LongDoubleFn: LibFunc_ldexpl, B, Attrs: NoAttrs));
2024	}
2025
2026	// pow(2.0 ** n, x) -> exp2(n * x)
2027	if (hasFloatFn(M, TLI, Ty, DoubleFn: LibFunc_exp2, FloatFn: LibFunc_exp2f, LongDoubleFn: LibFunc_exp2l)) {
2028	APFloat BaseR = APFloat(1.0);
2029	BaseR.convert(ToSemantics: BaseF->getSemantics(), RM: APFloat::rmTowardZero, losesInfo: &Ignored);
2030	BaseR = BaseR / *BaseF;
2031	bool IsInteger = BaseF->isInteger(), IsReciprocal = BaseR.isInteger();
2032	const APFloat *NF = IsReciprocal ? &BaseR : BaseF;
2033	APSInt NI(64, false);
2034	if ((IsInteger || IsReciprocal) &&
2035	NF->convertToInteger(Result&: NI, RM: APFloat::rmTowardZero, IsExact: &Ignored) ==
2036	APFloat::opOK &&
2037	NI > 1 && NI.isPowerOf2()) {
2038	double N = NI.logBase2() * (IsReciprocal ? -1.0 : 1.0);
2039	Value *FMul = B.CreateFMul(L: Expo, R: ConstantFP::get(Ty, V: N), Name: "mul");
2040	if (Pow->doesNotAccessMemory())
2041	return copyFlags(*Pow, B.CreateCall(Intrinsic::getDeclaration(
2042	M: Mod, Intrinsic::id: exp2, Tys: Ty),
2043	FMul, "exp2"));
2044	else
2045	return copyFlags(Old: *Pow, New: emitUnaryFloatFnCall(Op: FMul, TLI, DoubleFn: LibFunc_exp2,
2046	FloatFn: LibFunc_exp2f,
2047	LongDoubleFn: LibFunc_exp2l, B, Attrs: NoAttrs));
2048	}
2049	}
2050
2051	// pow(10.0, x) -> exp10(x)
2052	// TODO: There is no exp10() intrinsic yet, but some day there shall be one.
2053	if (match(V: Base, P: m_SpecificFP(V: 10.0)) &&
2054	hasFloatFn(M, TLI, Ty, DoubleFn: LibFunc_exp10, FloatFn: LibFunc_exp10f, LongDoubleFn: LibFunc_exp10l))
2055	return copyFlags(Old: *Pow, New: emitUnaryFloatFnCall(Op: Expo, TLI, DoubleFn: LibFunc_exp10,
2056	FloatFn: LibFunc_exp10f, LongDoubleFn: LibFunc_exp10l,
2057	B, Attrs: NoAttrs));
2058
2059	// pow(x, y) -> exp2(log2(x) * y)
2060	if (Pow->hasApproxFunc() && Pow->hasNoNaNs() && BaseF->isFiniteNonZero() &&
2061	!BaseF->isNegative()) {
2062	// pow(1, inf) is defined to be 1 but exp2(log2(1) * inf) evaluates to NaN.
2063	// Luckily optimizePow has already handled the x == 1 case.
2064	assert(!match(Base, m_FPOne()) &&
2065	"pow(1.0, y) should have been simplified earlier!");
2066
2067	Value *Log = nullptr;
2068	if (Ty->isFloatTy())
2069	Log = ConstantFP::get(Ty, V: std::log2(x: BaseF->convertToFloat()));
2070	else if (Ty->isDoubleTy())
2071	Log = ConstantFP::get(Ty, V: std::log2(x: BaseF->convertToDouble()));
2072
2073	if (Log) {
2074	Value *FMul = B.CreateFMul(L: Log, R: Expo, Name: "mul");
2075	if (Pow->doesNotAccessMemory())
2076	return copyFlags(*Pow, B.CreateCall(Intrinsic::getDeclaration(
2077	M: Mod, Intrinsic::id: exp2, Tys: Ty),
2078	FMul, "exp2"));
2079	else if (hasFloatFn(M, TLI, Ty, DoubleFn: LibFunc_exp2, FloatFn: LibFunc_exp2f,
2080	LongDoubleFn: LibFunc_exp2l))
2081	return copyFlags(Old: *Pow, New: emitUnaryFloatFnCall(Op: FMul, TLI, DoubleFn: LibFunc_exp2,
2082	FloatFn: LibFunc_exp2f,
2083	LongDoubleFn: LibFunc_exp2l, B, Attrs: NoAttrs));
2084	}
2085	}
2086
2087	return nullptr;
2088	}
2089
2090	static Value *getSqrtCall(Value *V, AttributeList Attrs, bool NoErrno,
2091	Module *M, IRBuilderBase &B,
2092	const TargetLibraryInfo *TLI) {
2093	// If errno is never set, then use the intrinsic for sqrt().
2094	if (NoErrno) {
2095	Function *SqrtFn =
2096	Intrinsic::getDeclaration(M, Intrinsic::id: sqrt, Tys: V->getType());
2097	return B.CreateCall(Callee: SqrtFn, Args: V, Name: "sqrt");
2098	}
2099
2100	// Otherwise, use the libcall for sqrt().
2101	if (hasFloatFn(M, TLI, Ty: V->getType(), DoubleFn: LibFunc_sqrt, FloatFn: LibFunc_sqrtf,
2102	LongDoubleFn: LibFunc_sqrtl))
2103	// TODO: We also should check that the target can in fact lower the sqrt()
2104	// libcall. We currently have no way to ask this question, so we ask if
2105	// the target has a sqrt() libcall, which is not exactly the same.
2106	return emitUnaryFloatFnCall(Op: V, TLI, DoubleFn: LibFunc_sqrt, FloatFn: LibFunc_sqrtf,
2107	LongDoubleFn: LibFunc_sqrtl, B, Attrs);
2108
2109	return nullptr;
2110	}
2111
2112	/// Use square root in place of pow(x, +/-0.5).
2113	Value *LibCallSimplifier::replacePowWithSqrt(CallInst *Pow, IRBuilderBase &B) {
2114	Value *Sqrt, *Base = Pow->getArgOperand(i: 0), *Expo = Pow->getArgOperand(i: 1);
2115	Module *Mod = Pow->getModule();
2116	Type *Ty = Pow->getType();
2117
2118	const APFloat *ExpoF;
2119	if (!match(V: Expo, P: m_APFloat(Res&: ExpoF)) ||
2120	(!ExpoF->isExactlyValue(V: 0.5) && !ExpoF->isExactlyValue(V: -0.5)))
2121	return nullptr;
2122
2123	// Converting pow(X, -0.5) to 1/sqrt(X) may introduce an extra rounding step,
2124	// so that requires fast-math-flags (afn or reassoc).
2125	if (ExpoF->isNegative() && (!Pow->hasApproxFunc() && !Pow->hasAllowReassoc()))
2126	return nullptr;
2127
2128	// If we have a pow() library call (accesses memory) and we can't guarantee
2129	// that the base is not an infinity, give up:
2130	// pow(-Inf, 0.5) is optionally required to have a result of +Inf (not setting
2131	// errno), but sqrt(-Inf) is required by various standards to set errno.
2132	if (!Pow->doesNotAccessMemory() && !Pow->hasNoInfs() &&
2133	!isKnownNeverInfinity(V: Base, Depth: 0,
2134	SQ: SimplifyQuery(DL, TLI, /*DT=*/nullptr, AC, Pow)))
2135	return nullptr;
2136
2137	Sqrt = getSqrtCall(V: Base, Attrs: AttributeList(), NoErrno: Pow->doesNotAccessMemory(), M: Mod, B,
2138	TLI);
2139	if (!Sqrt)
2140	return nullptr;
2141
2142	// Handle signed zero base by expanding to fabs(sqrt(x)).
2143	if (!Pow->hasNoSignedZeros()) {
2144	Function *FAbsFn = Intrinsic::getDeclaration(M: Mod, Intrinsic::id: fabs, Tys: Ty);
2145	Sqrt = B.CreateCall(Callee: FAbsFn, Args: Sqrt, Name: "abs");
2146	}
2147
2148	Sqrt = copyFlags(Old: *Pow, New: Sqrt);
2149
2150	// Handle non finite base by expanding to
2151	// (x == -infinity ? +infinity : sqrt(x)).
2152	if (!Pow->hasNoInfs()) {
2153	Value *PosInf = ConstantFP::getInfinity(Ty),
2154	*NegInf = ConstantFP::getInfinity(Ty, Negative: true);
2155	Value *FCmp = B.CreateFCmpOEQ(LHS: Base, RHS: NegInf, Name: "isinf");
2156	Sqrt = B.CreateSelect(C: FCmp, True: PosInf, False: Sqrt);
2157	}
2158
2159	// If the exponent is negative, then get the reciprocal.
2160	if (ExpoF->isNegative())
2161	Sqrt = B.CreateFDiv(L: ConstantFP::get(Ty, V: 1.0), R: Sqrt, Name: "reciprocal");
2162
2163	return Sqrt;
2164	}
2165
2166	static Value *createPowWithIntegerExponent(Value *Base, Value *Expo, Module *M,
2167	IRBuilderBase &B) {
2168	Value *Args[] = {Base, Expo};
2169	Type *Types[] = {Base->getType(), Expo->getType()};
2170	Function *F = Intrinsic::getDeclaration(M, Intrinsic::id: powi, Tys: Types);
2171	return B.CreateCall(Callee: F, Args);
2172	}
2173
2174	Value *LibCallSimplifier::optimizePow(CallInst *Pow, IRBuilderBase &B) {
2175	Value *Base = Pow->getArgOperand(i: 0);
2176	Value *Expo = Pow->getArgOperand(i: 1);
2177	Function *Callee = Pow->getCalledFunction();
2178	StringRef Name = Callee->getName();
2179	Type *Ty = Pow->getType();
2180	Module *M = Pow->getModule();
2181	bool AllowApprox = Pow->hasApproxFunc();
2182	bool Ignored;
2183
2184	// Propagate the math semantics from the call to any created instructions.
2185	IRBuilderBase::FastMathFlagGuard Guard(B);
2186	B.setFastMathFlags(Pow->getFastMathFlags());
2187	// Evaluate special cases related to the base.
2188
2189	// pow(1.0, x) -> 1.0
2190	if (match(V: Base, P: m_FPOne()))
2191	return Base;
2192
2193	if (Value *Exp = replacePowWithExp(Pow, B))
2194	return Exp;
2195
2196	// Evaluate special cases related to the exponent.
2197
2198	// pow(x, -1.0) -> 1.0 / x
2199	if (match(V: Expo, P: m_SpecificFP(V: -1.0)))
2200	return B.CreateFDiv(L: ConstantFP::get(Ty, V: 1.0), R: Base, Name: "reciprocal");
2201
2202	// pow(x, +/-0.0) -> 1.0
2203	if (match(V: Expo, P: m_AnyZeroFP()))
2204	return ConstantFP::get(Ty, V: 1.0);
2205
2206	// pow(x, 1.0) -> x
2207	if (match(V: Expo, P: m_FPOne()))
2208	return Base;
2209
2210	// pow(x, 2.0) -> x * x
2211	if (match(V: Expo, P: m_SpecificFP(V: 2.0)))
2212	return B.CreateFMul(L: Base, R: Base, Name: "square");
2213
2214	if (Value *Sqrt = replacePowWithSqrt(Pow, B))
2215	return Sqrt;
2216
2217	// If we can approximate pow:
2218	// pow(x, n) -> powi(x, n) * sqrt(x) if n has exactly a 0.5 fraction
2219	// pow(x, n) -> powi(x, n) if n is a constant signed integer value
2220	const APFloat *ExpoF;
2221	if (AllowApprox && match(V: Expo, P: m_APFloat(Res&: ExpoF)) &&
2222	!ExpoF->isExactlyValue(V: 0.5) && !ExpoF->isExactlyValue(V: -0.5)) {
2223	APFloat ExpoA(abs(X: *ExpoF));
2224	APFloat ExpoI(*ExpoF);
2225	Value *Sqrt = nullptr;
2226	if (!ExpoA.isInteger()) {
2227	APFloat Expo2 = ExpoA;
2228	// To check if ExpoA is an integer + 0.5, we add it to itself. If there
2229	// is no floating point exception and the result is an integer, then
2230	// ExpoA == integer + 0.5
2231	if (Expo2.add(RHS: ExpoA, RM: APFloat::rmNearestTiesToEven) != APFloat::opOK)
2232	return nullptr;
2233
2234	if (!Expo2.isInteger())
2235	return nullptr;
2236
2237	if (ExpoI.roundToIntegral(RM: APFloat::rmTowardNegative) !=
2238	APFloat::opInexact)
2239	return nullptr;
2240	if (!ExpoI.isInteger())
2241	return nullptr;
2242	ExpoF = &ExpoI;
2243
2244	Sqrt = getSqrtCall(V: Base, Attrs: AttributeList(), NoErrno: Pow->doesNotAccessMemory(), M,
2245	B, TLI);
2246	if (!Sqrt)
2247	return nullptr;
2248	}
2249
2250	// 0.5 fraction is now optionally handled.
2251	// Do pow -> powi for remaining integer exponent
2252	APSInt IntExpo(TLI->getIntSize(), /*isUnsigned=*/false);
2253	if (ExpoF->isInteger() &&
2254	ExpoF->convertToInteger(Result&: IntExpo, RM: APFloat::rmTowardZero, IsExact: &Ignored) ==
2255	APFloat::opOK) {
2256	Value *PowI = copyFlags(
2257	Old: *Pow,
2258	New: createPowWithIntegerExponent(
2259	Base, Expo: ConstantInt::get(Ty: B.getIntNTy(N: TLI->getIntSize()), V: IntExpo),
2260	M, B));
2261
2262	if (PowI && Sqrt)
2263	return B.CreateFMul(L: PowI, R: Sqrt);
2264
2265	return PowI;
2266	}
2267	}
2268
2269	// powf(x, itofp(y)) -> powi(x, y)
2270	if (AllowApprox && (isa<SIToFPInst>(Val: Expo) || isa<UIToFPInst>(Val: Expo))) {
2271	if (Value *ExpoI = getIntToFPVal(I2F: Expo, B, DstWidth: TLI->getIntSize()))
2272	return copyFlags(Old: *Pow, New: createPowWithIntegerExponent(Base, Expo: ExpoI, M, B));
2273	}
2274
2275	// Shrink pow() to powf() if the arguments are single precision,
2276	// unless the result is expected to be double precision.
2277	if (UnsafeFPShrink && Name == TLI->getName(F: LibFunc_pow) &&
2278	hasFloatVersion(M, FuncName: Name)) {
2279	if (Value *Shrunk = optimizeBinaryDoubleFP(CI: Pow, B, TLI, isPrecise: true))
2280	return Shrunk;
2281	}
2282
2283	return nullptr;
2284	}
2285
2286	Value *LibCallSimplifier::optimizeExp2(CallInst *CI, IRBuilderBase &B) {
2287	Module *M = CI->getModule();
2288	Function *Callee = CI->getCalledFunction();
2289	StringRef Name = Callee->getName();
2290	Value *Ret = nullptr;
2291	if (UnsafeFPShrink && Name == TLI->getName(F: LibFunc_exp2) &&
2292	hasFloatVersion(M, FuncName: Name))
2293	Ret = optimizeUnaryDoubleFP(CI, B, TLI, isPrecise: true);
2294
2295	// Bail out for vectors because the code below only expects scalars.
2296	// TODO: This could be allowed if we had a ldexp intrinsic (D14327).
2297	Type *Ty = CI->getType();
2298	if (Ty->isVectorTy())
2299	return Ret;
2300
2301	// exp2(sitofp(x)) -> ldexp(1.0, sext(x)) if sizeof(x) <= IntSize
2302	// exp2(uitofp(x)) -> ldexp(1.0, zext(x)) if sizeof(x) < IntSize
2303	Value *Op = CI->getArgOperand(i: 0);
2304	if ((isa<SIToFPInst>(Val: Op) || isa<UIToFPInst>(Val: Op)) &&
2305	hasFloatFn(M, TLI, Ty, DoubleFn: LibFunc_ldexp, FloatFn: LibFunc_ldexpf, LongDoubleFn: LibFunc_ldexpl)) {
2306	if (Value *Exp = getIntToFPVal(I2F: Op, B, DstWidth: TLI->getIntSize())) {
2307	IRBuilderBase::FastMathFlagGuard Guard(B);
2308	B.setFastMathFlags(CI->getFastMathFlags());
2309	return copyFlags(
2310	Old: *CI, New: emitBinaryFloatFnCall(Op1: ConstantFP::get(Ty, V: 1.0), Op2: Exp, TLI,
2311	DoubleFn: LibFunc_ldexp, FloatFn: LibFunc_ldexpf,
2312	LongDoubleFn: LibFunc_ldexpl, B, Attrs: AttributeList()));
2313	}
2314	}
2315
2316	return Ret;
2317	}
2318
2319	Value *LibCallSimplifier::optimizeFMinFMax(CallInst *CI, IRBuilderBase &B) {
2320	Module *M = CI->getModule();
2321
2322	// If we can shrink the call to a float function rather than a double
2323	// function, do that first.
2324	Function *Callee = CI->getCalledFunction();
2325	StringRef Name = Callee->getName();
2326	if ((Name == "fmin" || Name == "fmax") && hasFloatVersion(M, FuncName: Name))
2327	if (Value *Ret = optimizeBinaryDoubleFP(CI, B, TLI))
2328	return Ret;
2329
2330	// The LLVM intrinsics minnum/maxnum correspond to fmin/fmax. Canonicalize to
2331	// the intrinsics for improved optimization (for example, vectorization).
2332	// No-signed-zeros is implied by the definitions of fmax/fmin themselves.
2333	// From the C standard draft WG14/N1256:
2334	// "Ideally, fmax would be sensitive to the sign of zero, for example
2335	// fmax(-0.0, +0.0) would return +0; however, implementation in software
2336	// might be impractical."
2337	IRBuilderBase::FastMathFlagGuard Guard(B);
2338	FastMathFlags FMF = CI->getFastMathFlags();
2339	FMF.setNoSignedZeros();
2340	B.setFastMathFlags(FMF);
2341
2342	Intrinsic::ID IID = Callee->getName().starts_with(Prefix: "fmin") ? Intrinsic::minnum
2343	: Intrinsic::maxnum;
2344	Function *F = Intrinsic::getDeclaration(M: CI->getModule(), id: IID, Tys: CI->getType());
2345	return copyFlags(
2346	Old: *CI, New: B.CreateCall(Callee: F, Args: {CI->getArgOperand(i: 0), CI->getArgOperand(i: 1)}));
2347	}
2348
2349	Value *LibCallSimplifier::optimizeLog(CallInst *Log, IRBuilderBase &B) {
2350	Function *LogFn = Log->getCalledFunction();
2351	StringRef LogNm = LogFn->getName();
2352	Intrinsic::ID LogID = LogFn->getIntrinsicID();
2353	Module *Mod = Log->getModule();
2354	Type *Ty = Log->getType();
2355	Value *Ret = nullptr;
2356
2357	if (UnsafeFPShrink && hasFloatVersion(M: Mod, FuncName: LogNm))
2358	Ret = optimizeUnaryDoubleFP(CI: Log, B, TLI, isPrecise: true);
2359
2360	// The earlier call must also be 'fast' in order to do these transforms.
2361	CallInst *Arg = dyn_cast<CallInst>(Val: Log->getArgOperand(i: 0));
2362	if (!Log->isFast() || !Arg || !Arg->isFast() || !Arg->hasOneUse())
2363	return Ret;
2364
2365	LibFunc LogLb, ExpLb, Exp2Lb, Exp10Lb, PowLb;
2366
2367	// This is only applicable to log(), log2(), log10().
2368	if (TLI->getLibFunc(funcName: LogNm, F&: LogLb))
2369	switch (LogLb) {
2370	case LibFunc_logf:
2371	LogID = Intrinsic::log;
2372	ExpLb = LibFunc_expf;
2373	Exp2Lb = LibFunc_exp2f;
2374	Exp10Lb = LibFunc_exp10f;
2375	PowLb = LibFunc_powf;
2376	break;
2377	case LibFunc_log:
2378	LogID = Intrinsic::log;
2379	ExpLb = LibFunc_exp;
2380	Exp2Lb = LibFunc_exp2;
2381	Exp10Lb = LibFunc_exp10;
2382	PowLb = LibFunc_pow;
2383	break;
2384	case LibFunc_logl:
2385	LogID = Intrinsic::log;
2386	ExpLb = LibFunc_expl;
2387	Exp2Lb = LibFunc_exp2l;
2388	Exp10Lb = LibFunc_exp10l;
2389	PowLb = LibFunc_powl;
2390	break;
2391	case LibFunc_log2f:
2392	LogID = Intrinsic::log2;
2393	ExpLb = LibFunc_expf;
2394	Exp2Lb = LibFunc_exp2f;
2395	Exp10Lb = LibFunc_exp10f;
2396	PowLb = LibFunc_powf;
2397	break;
2398	case LibFunc_log2:
2399	LogID = Intrinsic::log2;
2400	ExpLb = LibFunc_exp;
2401	Exp2Lb = LibFunc_exp2;
2402	Exp10Lb = LibFunc_exp10;
2403	PowLb = LibFunc_pow;
2404	break;
2405	case LibFunc_log2l:
2406	LogID = Intrinsic::log2;
2407	ExpLb = LibFunc_expl;
2408	Exp2Lb = LibFunc_exp2l;
2409	Exp10Lb = LibFunc_exp10l;
2410	PowLb = LibFunc_powl;
2411	break;
2412	case LibFunc_log10f:
2413	LogID = Intrinsic::log10;
2414	ExpLb = LibFunc_expf;
2415	Exp2Lb = LibFunc_exp2f;
2416	Exp10Lb = LibFunc_exp10f;
2417	PowLb = LibFunc_powf;
2418	break;
2419	case LibFunc_log10:
2420	LogID = Intrinsic::log10;
2421	ExpLb = LibFunc_exp;
2422	Exp2Lb = LibFunc_exp2;
2423	Exp10Lb = LibFunc_exp10;
2424	PowLb = LibFunc_pow;
2425	break;
2426	case LibFunc_log10l:
2427	LogID = Intrinsic::log10;
2428	ExpLb = LibFunc_expl;
2429	Exp2Lb = LibFunc_exp2l;
2430	Exp10Lb = LibFunc_exp10l;
2431	PowLb = LibFunc_powl;
2432	break;
2433	default:
2434	return Ret;
2435	}
2436	else if (LogID == Intrinsic::log || LogID == Intrinsic::log2 ||
2437	LogID == Intrinsic::log10) {
2438	if (Ty->getScalarType()->isFloatTy()) {
2439	ExpLb = LibFunc_expf;
2440	Exp2Lb = LibFunc_exp2f;
2441	Exp10Lb = LibFunc_exp10f;
2442	PowLb = LibFunc_powf;
2443	} else if (Ty->getScalarType()->isDoubleTy()) {
2444	ExpLb = LibFunc_exp;
2445	Exp2Lb = LibFunc_exp2;
2446	Exp10Lb = LibFunc_exp10;
2447	PowLb = LibFunc_pow;
2448	} else
2449	return Ret;
2450	} else
2451	return Ret;
2452
2453	IRBuilderBase::FastMathFlagGuard Guard(B);
2454	B.setFastMathFlags(FastMathFlags::getFast());
2455
2456	Intrinsic::ID ArgID = Arg->getIntrinsicID();
2457	LibFunc ArgLb = NotLibFunc;
2458	TLI->getLibFunc(CB: *Arg, F&: ArgLb);
2459
2460	// log(pow(x,y)) -> y*log(x)
2461	AttributeList NoAttrs;
2462	if (ArgLb == PowLb || ArgID == Intrinsic::pow || ArgID == Intrinsic::powi) {
2463	Value *LogX =
2464	Log->doesNotAccessMemory()
2465	? B.CreateCall(Callee: Intrinsic::getDeclaration(M: Mod, id: LogID, Tys: Ty),
2466	Args: Arg->getOperand(i_nocapture: 0), Name: "log")
2467	: emitUnaryFloatFnCall(Op: Arg->getOperand(i_nocapture: 0), TLI, Name: LogNm, B, Attrs: NoAttrs);
2468	Value *Y = Arg->getArgOperand(i: 1);
2469	// Cast exponent to FP if integer.
2470	if (ArgID == Intrinsic::powi)
2471	Y = B.CreateSIToFP(V: Y, DestTy: Ty, Name: "cast");
2472	Value *MulY = B.CreateFMul(L: Y, R: LogX, Name: "mul");
2473	// Since pow() may have side effects, e.g. errno,
2474	// dead code elimination may not be trusted to remove it.
2475	substituteInParent(I: Arg, With: MulY);
2476	return MulY;
2477	}
2478
2479	// log(exp{,2,10}(y)) -> y*log({e,2,10})
2480	// TODO: There is no exp10() intrinsic yet.
2481	if (ArgLb == ExpLb || ArgLb == Exp2Lb || ArgLb == Exp10Lb ||
2482	ArgID == Intrinsic::exp || ArgID == Intrinsic::exp2) {
2483	Constant *Eul;
2484	if (ArgLb == ExpLb || ArgID == Intrinsic::exp)
2485	// FIXME: Add more precise value of e for long double.
2486	Eul = ConstantFP::get(Ty: Log->getType(), V: numbers::e);
2487	else if (ArgLb == Exp2Lb || ArgID == Intrinsic::exp2)
2488	Eul = ConstantFP::get(Ty: Log->getType(), V: 2.0);
2489	else
2490	Eul = ConstantFP::get(Ty: Log->getType(), V: 10.0);
2491	Value *LogE = Log->doesNotAccessMemory()
2492	? B.CreateCall(Callee: Intrinsic::getDeclaration(M: Mod, id: LogID, Tys: Ty),
2493	Args: Eul, Name: "log")
2494	: emitUnaryFloatFnCall(Op: Eul, TLI, Name: LogNm, B, Attrs: NoAttrs);
2495	Value *MulY = B.CreateFMul(L: Arg->getArgOperand(i: 0), R: LogE, Name: "mul");
2496	// Since exp() may have side effects, e.g. errno,
2497	// dead code elimination may not be trusted to remove it.
2498	substituteInParent(I: Arg, With: MulY);
2499	return MulY;
2500	}
2501
2502	return Ret;
2503	}
2504
2505	// sqrt(exp(X)) -> exp(X * 0.5)
2506	Value *LibCallSimplifier::mergeSqrtToExp(CallInst *CI, IRBuilderBase &B) {
2507	if (!CI->hasAllowReassoc())
2508	return nullptr;
2509
2510	Function *SqrtFn = CI->getCalledFunction();
2511	CallInst *Arg = dyn_cast<CallInst>(Val: CI->getArgOperand(i: 0));
2512	if (!Arg || !Arg->hasAllowReassoc() || !Arg->hasOneUse())
2513	return nullptr;
2514	Intrinsic::ID ArgID = Arg->getIntrinsicID();
2515	LibFunc ArgLb = NotLibFunc;
2516	TLI->getLibFunc(CB: *Arg, F&: ArgLb);
2517
2518	LibFunc SqrtLb, ExpLb, Exp2Lb, Exp10Lb;
2519
2520	if (TLI->getLibFunc(funcName: SqrtFn->getName(), F&: SqrtLb))
2521	switch (SqrtLb) {
2522	case LibFunc_sqrtf:
2523	ExpLb = LibFunc_expf;
2524	Exp2Lb = LibFunc_exp2f;
2525	Exp10Lb = LibFunc_exp10f;
2526	break;
2527	case LibFunc_sqrt:
2528	ExpLb = LibFunc_exp;
2529	Exp2Lb = LibFunc_exp2;
2530	Exp10Lb = LibFunc_exp10;
2531	break;
2532	case LibFunc_sqrtl:
2533	ExpLb = LibFunc_expl;
2534	Exp2Lb = LibFunc_exp2l;
2535	Exp10Lb = LibFunc_exp10l;
2536	break;
2537	default:
2538	return nullptr;
2539	}
2540	else if (SqrtFn->getIntrinsicID() == Intrinsic::sqrt) {
2541	if (CI->getType()->getScalarType()->isFloatTy()) {
2542	ExpLb = LibFunc_expf;
2543	Exp2Lb = LibFunc_exp2f;
2544	Exp10Lb = LibFunc_exp10f;
2545	} else if (CI->getType()->getScalarType()->isDoubleTy()) {
2546	ExpLb = LibFunc_exp;
2547	Exp2Lb = LibFunc_exp2;
2548	Exp10Lb = LibFunc_exp10;
2549	} else
2550	return nullptr;
2551	} else
2552	return nullptr;
2553
2554	if (ArgLb != ExpLb && ArgLb != Exp2Lb && ArgLb != Exp10Lb &&
2555	ArgID != Intrinsic::exp && ArgID != Intrinsic::exp2)
2556	return nullptr;
2557
2558	IRBuilderBase::InsertPointGuard Guard(B);
2559	B.SetInsertPoint(Arg);
2560	auto *ExpOperand = Arg->getOperand(i_nocapture: 0);
2561	auto *FMul =
2562	B.CreateFMulFMF(L: ExpOperand, R: ConstantFP::get(Ty: ExpOperand->getType(), V: 0.5),
2563	FMFSource: CI, Name: "merged.sqrt");
2564
2565	Arg->setOperand(i_nocapture: 0, Val_nocapture: FMul);
2566	return Arg;
2567	}
2568
2569	Value *LibCallSimplifier::optimizeSqrt(CallInst *CI, IRBuilderBase &B) {
2570	Module *M = CI->getModule();
2571	Function *Callee = CI->getCalledFunction();
2572	Value *Ret = nullptr;
2573	// TODO: Once we have a way (other than checking for the existince of the
2574	// libcall) to tell whether our target can lower @llvm.sqrt, relax the
2575	// condition below.
2576	if (isLibFuncEmittable(M, TLI, TheLibFunc: LibFunc_sqrtf) &&
2577	(Callee->getName() == "sqrt" ||
2578	Callee->getIntrinsicID() == Intrinsic::sqrt))
2579	Ret = optimizeUnaryDoubleFP(CI, B, TLI, isPrecise: true);
2580
2581	if (Value *Opt = mergeSqrtToExp(CI, B))
2582	return Opt;
2583
2584	if (!CI->isFast())
2585	return Ret;
2586
2587	Instruction *I = dyn_cast<Instruction>(Val: CI->getArgOperand(i: 0));
2588	if (!I || I->getOpcode() != Instruction::FMul || !I->isFast())
2589	return Ret;
2590
2591	// We're looking for a repeated factor in a multiplication tree,
2592	// so we can do this fold: sqrt(x * x) -> fabs(x);
2593	// or this fold: sqrt((x * x) * y) -> fabs(x) * sqrt(y).
2594	Value *Op0 = I->getOperand(i: 0);
2595	Value *Op1 = I->getOperand(i: 1);
2596	Value *RepeatOp = nullptr;
2597	Value *OtherOp = nullptr;
2598	if (Op0 == Op1) {
2599	// Simple match: the operands of the multiply are identical.
2600	RepeatOp = Op0;
2601	} else {
2602	// Look for a more complicated pattern: one of the operands is itself
2603	// a multiply, so search for a common factor in that multiply.
2604	// Note: We don't bother looking any deeper than this first level or for
2605	// variations of this pattern because instcombine's visitFMUL and/or the
2606	// reassociation pass should give us this form.
2607	Value *OtherMul0, *OtherMul1;
2608	if (match(V: Op0, P: m_FMul(L: m_Value(V&: OtherMul0), R: m_Value(V&: OtherMul1)))) {
2609	// Pattern: sqrt((x * y) * z)
2610	if (OtherMul0 == OtherMul1 && cast<Instruction>(Val: Op0)->isFast()) {
2611	// Matched: sqrt((x * x) * z)
2612	RepeatOp = OtherMul0;
2613	OtherOp = Op1;
2614	}
2615	}
2616	}
2617	if (!RepeatOp)
2618	return Ret;
2619
2620	// Fast math flags for any created instructions should match the sqrt
2621	// and multiply.
2622	IRBuilderBase::FastMathFlagGuard Guard(B);
2623	B.setFastMathFlags(I->getFastMathFlags());
2624
2625	// If we found a repeated factor, hoist it out of the square root and
2626	// replace it with the fabs of that factor.
2627	Type *ArgType = I->getType();
2628	Function *Fabs = Intrinsic::getDeclaration(M, Intrinsic::id: fabs, Tys: ArgType);
2629	Value *FabsCall = B.CreateCall(Callee: Fabs, Args: RepeatOp, Name: "fabs");
2630	if (OtherOp) {
2631	// If we found a non-repeated factor, we still need to get its square
2632	// root. We then multiply that by the value that was simplified out
2633	// of the square root calculation.
2634	Function *Sqrt = Intrinsic::getDeclaration(M, Intrinsic::id: sqrt, Tys: ArgType);
2635	Value *SqrtCall = B.CreateCall(Callee: Sqrt, Args: OtherOp, Name: "sqrt");
2636	return copyFlags(Old: *CI, New: B.CreateFMul(L: FabsCall, R: SqrtCall));
2637	}
2638	return copyFlags(Old: *CI, New: FabsCall);
2639	}
2640
2641	Value *LibCallSimplifier::optimizeTrigInversionPairs(CallInst *CI,
2642	IRBuilderBase &B) {
2643	Module *M = CI->getModule();
2644	Function *Callee = CI->getCalledFunction();
2645	Value *Ret = nullptr;
2646	StringRef Name = Callee->getName();
2647	if (UnsafeFPShrink &&
2648	(Name == "tan" || Name == "atanh" || Name == "sinh" || Name == "cosh" ||
2649	Name == "asinh") &&
2650	hasFloatVersion(M, FuncName: Name))
2651	Ret = optimizeUnaryDoubleFP(CI, B, TLI, isPrecise: true);
2652
2653	Value *Op1 = CI->getArgOperand(i: 0);
2654	auto *OpC = dyn_cast<CallInst>(Val: Op1);
2655	if (!OpC)
2656	return Ret;
2657
2658	// Both calls must be 'fast' in order to remove them.
2659	if (!CI->isFast() || !OpC->isFast())
2660	return Ret;
2661
2662	// tan(atan(x)) -> x
2663	// atanh(tanh(x)) -> x
2664	// sinh(asinh(x)) -> x
2665	// asinh(sinh(x)) -> x
2666	// cosh(acosh(x)) -> x
2667	LibFunc Func;
2668	Function *F = OpC->getCalledFunction();
2669	if (F && TLI->getLibFunc(funcName: F->getName(), F&: Func) &&
2670	isLibFuncEmittable(M, TLI, TheLibFunc: Func)) {
2671	LibFunc inverseFunc = llvm::StringSwitch<LibFunc>(Callee->getName())
2672	.Case(S: "tan", Value: LibFunc_atan)
2673	.Case(S: "atanh", Value: LibFunc_tanh)
2674	.Case(S: "sinh", Value: LibFunc_asinh)
2675	.Case(S: "cosh", Value: LibFunc_acosh)
2676	.Case(S: "tanf", Value: LibFunc_atanf)
2677	.Case(S: "atanhf", Value: LibFunc_tanhf)
2678	.Case(S: "sinhf", Value: LibFunc_asinhf)
2679	.Case(S: "coshf", Value: LibFunc_acoshf)
2680	.Case(S: "tanl", Value: LibFunc_atanl)
2681	.Case(S: "atanhl", Value: LibFunc_tanhl)
2682	.Case(S: "sinhl", Value: LibFunc_asinhl)
2683	.Case(S: "coshl", Value: LibFunc_acoshl)
2684	.Case(S: "asinh", Value: LibFunc_sinh)
2685	.Case(S: "asinhf", Value: LibFunc_sinhf)
2686	.Case(S: "asinhl", Value: LibFunc_sinhl)
2687	.Default(Value: NumLibFuncs); // Used as error value
2688	if (Func == inverseFunc)
2689	Ret = OpC->getArgOperand(i: 0);
2690	}
2691	return Ret;
2692	}
2693
2694	static bool isTrigLibCall(CallInst *CI) {
2695	// We can only hope to do anything useful if we can ignore things like errno
2696	// and floating-point exceptions.
2697	// We already checked the prototype.
2698	return CI->doesNotThrow() && CI->doesNotAccessMemory();
2699	}
2700
2701	static bool insertSinCosCall(IRBuilderBase &B, Function *OrigCallee, Value *Arg,
2702	bool UseFloat, Value *&Sin, Value *&Cos,
2703	Value *&SinCos, const TargetLibraryInfo *TLI) {
2704	Module *M = OrigCallee->getParent();
2705	Type *ArgTy = Arg->getType();
2706	Type *ResTy;
2707	StringRef Name;
2708
2709	Triple T(OrigCallee->getParent()->getTargetTriple());
2710	if (UseFloat) {
2711	Name = "__sincospif_stret";
2712
2713	assert(T.getArch() != Triple::x86 && "x86 messy and unsupported for now");
2714	// x86_64 can't use {float, float} since that would be returned in both
2715	// xmm0 and xmm1, which isn't what a real struct would do.
2716	ResTy = T.getArch() == Triple::x86_64
2717	? static_cast<Type *>(FixedVectorType::get(ElementType: ArgTy, NumElts: 2))
2718	: static_cast<Type *>(StructType::get(elt1: ArgTy, elts: ArgTy));
2719	} else {
2720	Name = "__sincospi_stret";
2721	ResTy = StructType::get(elt1: ArgTy, elts: ArgTy);
2722	}
2723
2724	if (!isLibFuncEmittable(M, TLI, Name))
2725	return false;
2726	LibFunc TheLibFunc;
2727	TLI->getLibFunc(funcName: Name, F&: TheLibFunc);
2728	FunctionCallee Callee = getOrInsertLibFunc(
2729	M, TLI: *TLI, TheLibFunc, AttributeList: OrigCallee->getAttributes(), RetTy: ResTy, Args: ArgTy);
2730
2731	if (Instruction *ArgInst = dyn_cast<Instruction>(Val: Arg)) {
2732	// If the argument is an instruction, it must dominate all uses so put our
2733	// sincos call there.
2734	B.SetInsertPoint(TheBB: ArgInst->getParent(), IP: ++ArgInst->getIterator());
2735	} else {
2736	// Otherwise (e.g. for a constant) the beginning of the function is as
2737	// good a place as any.
2738	BasicBlock &EntryBB = B.GetInsertBlock()->getParent()->getEntryBlock();
2739	B.SetInsertPoint(TheBB: &EntryBB, IP: EntryBB.begin());
2740	}
2741
2742	SinCos = B.CreateCall(Callee, Args: Arg, Name: "sincospi");
2743
2744	if (SinCos->getType()->isStructTy()) {
2745	Sin = B.CreateExtractValue(Agg: SinCos, Idxs: 0, Name: "sinpi");
2746	Cos = B.CreateExtractValue(Agg: SinCos, Idxs: 1, Name: "cospi");
2747	} else {
2748	Sin = B.CreateExtractElement(Vec: SinCos, Idx: ConstantInt::get(Ty: B.getInt32Ty(), V: 0),
2749	Name: "sinpi");
2750	Cos = B.CreateExtractElement(Vec: SinCos, Idx: ConstantInt::get(Ty: B.getInt32Ty(), V: 1),
2751	Name: "cospi");
2752	}
2753
2754	return true;
2755	}
2756
2757	static Value *optimizeSymmetricCall(CallInst *CI, bool IsEven,
2758	IRBuilderBase &B) {
2759	Value *X;
2760	Value *Src = CI->getArgOperand(i: 0);
2761
2762	if (match(V: Src, P: m_OneUse(SubPattern: m_FNeg(X: m_Value(V&: X))))) {
2763	IRBuilderBase::FastMathFlagGuard Guard(B);
2764	B.setFastMathFlags(CI->getFastMathFlags());
2765
2766	auto *CallInst = copyFlags(Old: *CI, New: B.CreateCall(Callee: CI->getCalledFunction(), Args: {X}));
2767	if (IsEven) {
2768	// Even function: f(-x) = f(x)
2769	return CallInst;
2770	}
2771	// Odd function: f(-x) = -f(x)
2772	return B.CreateFNeg(V: CallInst);
2773	}
2774
2775	// Even function: f(abs(x)) = f(x), f(copysign(x, y)) = f(x)
2776	if (IsEven && (match(V: Src, P: m_FAbs(Op0: m_Value(V&: X))) ||
2777	match(V: Src, P: m_CopySign(Op0: m_Value(V&: X), Op1: m_Value())))) {
2778	IRBuilderBase::FastMathFlagGuard Guard(B);
2779	B.setFastMathFlags(CI->getFastMathFlags());
2780
2781	auto *CallInst = copyFlags(Old: *CI, New: B.CreateCall(Callee: CI->getCalledFunction(), Args: {X}));
2782	return CallInst;
2783	}
2784
2785	return nullptr;
2786	}
2787
2788	Value *LibCallSimplifier::optimizeSymmetric(CallInst *CI, LibFunc Func,
2789	IRBuilderBase &B) {
2790	switch (Func) {
2791	case LibFunc_cos:
2792	case LibFunc_cosf:
2793	case LibFunc_cosl:
2794	return optimizeSymmetricCall(CI, /*IsEven*/ true, B);
2795
2796	case LibFunc_sin:
2797	case LibFunc_sinf:
2798	case LibFunc_sinl:
2799
2800	case LibFunc_tan:
2801	case LibFunc_tanf:
2802	case LibFunc_tanl:
2803
2804	case LibFunc_erf:
2805	case LibFunc_erff:
2806	case LibFunc_erfl:
2807	return optimizeSymmetricCall(CI, /*IsEven*/ false, B);
2808
2809	default:
2810	return nullptr;
2811	}
2812	}
2813
2814	Value *LibCallSimplifier::optimizeSinCosPi(CallInst *CI, bool IsSin, IRBuilderBase &B) {
2815	// Make sure the prototype is as expected, otherwise the rest of the
2816	// function is probably invalid and likely to abort.
2817	if (!isTrigLibCall(CI))
2818	return nullptr;
2819
2820	Value *Arg = CI->getArgOperand(i: 0);
2821	SmallVector<CallInst *, 1> SinCalls;
2822	SmallVector<CallInst *, 1> CosCalls;
2823	SmallVector<CallInst *, 1> SinCosCalls;
2824
2825	bool IsFloat = Arg->getType()->isFloatTy();
2826
2827	// Look for all compatible sinpi, cospi and sincospi calls with the same
2828	// argument. If there are enough (in some sense) we can make the
2829	// substitution.
2830	Function *F = CI->getFunction();
2831	for (User *U : Arg->users())
2832	classifyArgUse(Val: U, F, IsFloat, SinCalls, CosCalls, SinCosCalls);
2833
2834	// It's only worthwhile if both sinpi and cospi are actually used.
2835	if (SinCalls.empty() || CosCalls.empty())
2836	return nullptr;
2837
2838	Value *Sin, *Cos, *SinCos;
2839	if (!insertSinCosCall(B, OrigCallee: CI->getCalledFunction(), Arg, UseFloat: IsFloat, Sin, Cos,
2840	SinCos, TLI))
2841	return nullptr;
2842
2843	auto replaceTrigInsts = [this](SmallVectorImpl<CallInst *> &Calls,
2844	Value *Res) {
2845	for (CallInst *C : Calls)
2846	replaceAllUsesWith(I: C, With: Res);
2847	};
2848
2849	replaceTrigInsts(SinCalls, Sin);
2850	replaceTrigInsts(CosCalls, Cos);
2851	replaceTrigInsts(SinCosCalls, SinCos);
2852
2853	return IsSin ? Sin : Cos;
2854	}
2855
2856	void LibCallSimplifier::classifyArgUse(
2857	Value *Val, Function *F, bool IsFloat,
2858	SmallVectorImpl<CallInst *> &SinCalls,
2859	SmallVectorImpl<CallInst *> &CosCalls,
2860	SmallVectorImpl<CallInst *> &SinCosCalls) {
2861	auto *CI = dyn_cast<CallInst>(Val);
2862	if (!CI || CI->use_empty())
2863	return;
2864
2865	// Don't consider calls in other functions.
2866	if (CI->getFunction() != F)
2867	return;
2868
2869	Module *M = CI->getModule();
2870	Function *Callee = CI->getCalledFunction();
2871	LibFunc Func;
2872	if (!Callee || !TLI->getLibFunc(FDecl: *Callee, F&: Func) ||
2873	!isLibFuncEmittable(M, TLI, TheLibFunc: Func) ||
2874	!isTrigLibCall(CI))
2875	return;
2876
2877	if (IsFloat) {
2878	if (Func == LibFunc_sinpif)
2879	SinCalls.push_back(Elt: CI);
2880	else if (Func == LibFunc_cospif)
2881	CosCalls.push_back(Elt: CI);
2882	else if (Func == LibFunc_sincospif_stret)
2883	SinCosCalls.push_back(Elt: CI);
2884	} else {
2885	if (Func == LibFunc_sinpi)
2886	SinCalls.push_back(Elt: CI);
2887	else if (Func == LibFunc_cospi)
2888	CosCalls.push_back(Elt: CI);
2889	else if (Func == LibFunc_sincospi_stret)
2890	SinCosCalls.push_back(Elt: CI);
2891	}
2892	}
2893
2894	//===----------------------------------------------------------------------===//
2895	// Integer Library Call Optimizations
2896	//===----------------------------------------------------------------------===//
2897
2898	Value *LibCallSimplifier::optimizeFFS(CallInst *CI, IRBuilderBase &B) {
2899	// All variants of ffs return int which need not be 32 bits wide.
2900	// ffs{,l,ll}(x) -> x != 0 ? (int)llvm.cttz(x)+1 : 0
2901	Type *RetType = CI->getType();
2902	Value *Op = CI->getArgOperand(i: 0);
2903	Type *ArgType = Op->getType();
2904	Function *F = Intrinsic::getDeclaration(M: CI->getCalledFunction()->getParent(),
2905	Intrinsic::id: cttz, Tys: ArgType);
2906	Value *V = B.CreateCall(Callee: F, Args: {Op, B.getTrue()}, Name: "cttz");
2907	V = B.CreateAdd(LHS: V, RHS: ConstantInt::get(Ty: V->getType(), V: 1));
2908	V = B.CreateIntCast(V, DestTy: RetType, isSigned: false);
2909
2910	Value *Cond = B.CreateICmpNE(LHS: Op, RHS: Constant::getNullValue(Ty: ArgType));
2911	return B.CreateSelect(C: Cond, True: V, False: ConstantInt::get(Ty: RetType, V: 0));
2912	}
2913
2914	Value *LibCallSimplifier::optimizeFls(CallInst *CI, IRBuilderBase &B) {
2915	// All variants of fls return int which need not be 32 bits wide.
2916	// fls{,l,ll}(x) -> (int)(sizeInBits(x) - llvm.ctlz(x, false))
2917	Value *Op = CI->getArgOperand(i: 0);
2918	Type *ArgType = Op->getType();
2919	Function *F = Intrinsic::getDeclaration(M: CI->getCalledFunction()->getParent(),
2920	Intrinsic::id: ctlz, Tys: ArgType);
2921	Value *V = B.CreateCall(Callee: F, Args: {Op, B.getFalse()}, Name: "ctlz");
2922	V = B.CreateSub(LHS: ConstantInt::get(Ty: V->getType(), V: ArgType->getIntegerBitWidth()),
2923	RHS: V);
2924	return B.CreateIntCast(V, DestTy: CI->getType(), isSigned: false);
2925	}
2926
2927	Value *LibCallSimplifier::optimizeAbs(CallInst *CI, IRBuilderBase &B) {
2928	// abs(x) -> x <s 0 ? -x : x
2929	// The negation has 'nsw' because abs of INT_MIN is undefined.
2930	Value *X = CI->getArgOperand(i: 0);
2931	Value *IsNeg = B.CreateIsNeg(Arg: X);
2932	Value *NegX = B.CreateNSWNeg(V: X, Name: "neg");
2933	return B.CreateSelect(C: IsNeg, True: NegX, False: X);
2934	}
2935
2936	Value *LibCallSimplifier::optimizeIsDigit(CallInst *CI, IRBuilderBase &B) {
2937	// isdigit(c) -> (c-'0') <u 10
2938	Value *Op = CI->getArgOperand(i: 0);
2939	Type *ArgType = Op->getType();
2940	Op = B.CreateSub(LHS: Op, RHS: ConstantInt::get(Ty: ArgType, V: '0'), Name: "isdigittmp");
2941	Op = B.CreateICmpULT(LHS: Op, RHS: ConstantInt::get(Ty: ArgType, V: 10), Name: "isdigit");
2942	return B.CreateZExt(V: Op, DestTy: CI->getType());
2943	}
2944
2945	Value *LibCallSimplifier::optimizeIsAscii(CallInst *CI, IRBuilderBase &B) {
2946	// isascii(c) -> c <u 128
2947	Value *Op = CI->getArgOperand(i: 0);
2948	Type *ArgType = Op->getType();
2949	Op = B.CreateICmpULT(LHS: Op, RHS: ConstantInt::get(Ty: ArgType, V: 128), Name: "isascii");
2950	return B.CreateZExt(V: Op, DestTy: CI->getType());
2951	}
2952
2953	Value *LibCallSimplifier::optimizeToAscii(CallInst *CI, IRBuilderBase &B) {
2954	// toascii(c) -> c & 0x7f
2955	return B.CreateAnd(LHS: CI->getArgOperand(i: 0),
2956	RHS: ConstantInt::get(Ty: CI->getType(), V: 0x7F));
2957	}
2958
2959	// Fold calls to atoi, atol, and atoll.
2960	Value *LibCallSimplifier::optimizeAtoi(CallInst *CI, IRBuilderBase &B) {
2961	CI->addParamAttr(0, Attribute::NoCapture);
2962
2963	StringRef Str;
2964	if (!getConstantStringInfo(V: CI->getArgOperand(i: 0), Str))
2965	return nullptr;
2966
2967	return convertStrToInt(CI, Str, EndPtr: nullptr, Base: 10, /*AsSigned=*/true, B);
2968	}
2969
2970	// Fold calls to strtol, strtoll, strtoul, and strtoull.
2971	Value *LibCallSimplifier::optimizeStrToInt(CallInst *CI, IRBuilderBase &B,
2972	bool AsSigned) {
2973	Value *EndPtr = CI->getArgOperand(i: 1);
2974	if (isa<ConstantPointerNull>(Val: EndPtr)) {
2975	// With a null EndPtr, this function won't capture the main argument.
2976	// It would be readonly too, except that it still may write to errno.
2977	CI->addParamAttr(0, Attribute::NoCapture);
2978	EndPtr = nullptr;
2979	} else if (!isKnownNonZero(V: EndPtr, Q: DL))
2980	return nullptr;
2981
2982	StringRef Str;
2983	if (!getConstantStringInfo(V: CI->getArgOperand(i: 0), Str))
2984	return nullptr;
2985
2986	if (ConstantInt *CInt = dyn_cast<ConstantInt>(Val: CI->getArgOperand(i: 2))) {
2987	return convertStrToInt(CI, Str, EndPtr, Base: CInt->getSExtValue(), AsSigned, B);
2988	}
2989
2990	return nullptr;
2991	}
2992
2993	//===----------------------------------------------------------------------===//
2994	// Formatting and IO Library Call Optimizations
2995	//===----------------------------------------------------------------------===//
2996
2997	static bool isReportingError(Function *Callee, CallInst *CI, int StreamArg);
2998
2999	Value *LibCallSimplifier::optimizeErrorReporting(CallInst *CI, IRBuilderBase &B,
3000	int StreamArg) {
3001	Function *Callee = CI->getCalledFunction();
3002	// Error reporting calls should be cold, mark them as such.
3003	// This applies even to non-builtin calls: it is only a hint and applies to
3004	// functions that the frontend might not understand as builtins.
3005
3006	// This heuristic was suggested in:
3007	// Improving Static Branch Prediction in a Compiler
3008	// Brian L. Deitrich, Ben-Chung Cheng, Wen-mei W. Hwu
3009	// Proceedings of PACT'98, Oct. 1998, IEEE
3010	if (!CI->hasFnAttr(Attribute::Cold) &&
3011	isReportingError(Callee, CI, StreamArg)) {
3012	CI->addFnAttr(Attribute::Cold);
3013	}
3014
3015	return nullptr;
3016	}
3017
3018	static bool isReportingError(Function *Callee, CallInst *CI, int StreamArg) {
3019	if (!Callee || !Callee->isDeclaration())
3020	return false;
3021
3022	if (StreamArg < 0)
3023	return true;
3024
3025	// These functions might be considered cold, but only if their stream
3026	// argument is stderr.
3027
3028	if (StreamArg >= (int)CI->arg_size())
3029	return false;
3030	LoadInst *LI = dyn_cast<LoadInst>(Val: CI->getArgOperand(i: StreamArg));
3031	if (!LI)
3032	return false;
3033	GlobalVariable *GV = dyn_cast<GlobalVariable>(Val: LI->getPointerOperand());
3034	if (!GV || !GV->isDeclaration())
3035	return false;
3036	return GV->getName() == "stderr";
3037	}
3038
3039	Value *LibCallSimplifier::optimizePrintFString(CallInst *CI, IRBuilderBase &B) {
3040	// Check for a fixed format string.
3041	StringRef FormatStr;
3042	if (!getConstantStringInfo(V: CI->getArgOperand(i: 0), Str&: FormatStr))
3043	return nullptr;
3044
3045	// Empty format string -> noop.
3046	if (FormatStr.empty()) // Tolerate printf's declared void.
3047	return CI->use_empty() ? (Value *)CI : ConstantInt::get(Ty: CI->getType(), V: 0);
3048
3049	// Do not do any of the following transformations if the printf return value
3050	// is used, in general the printf return value is not compatible with either
3051	// putchar() or puts().
3052	if (!CI->use_empty())
3053	return nullptr;
3054
3055	Type *IntTy = CI->getType();
3056	// printf("x") -> putchar('x'), even for "%" and "%%".
3057	if (FormatStr.size() == 1 || FormatStr == "%%") {
3058	// Convert the character to unsigned char before passing it to putchar
3059	// to avoid host-specific sign extension in the IR. Putchar converts
3060	// it to unsigned char regardless.
3061	Value *IntChar = ConstantInt::get(Ty: IntTy, V: (unsigned char)FormatStr[0]);
3062	return copyFlags(Old: *CI, New: emitPutChar(Char: IntChar, B, TLI));
3063	}
3064
3065	// Try to remove call or emit putchar/puts.
3066	if (FormatStr == "%s" && CI->arg_size() > 1) {
3067	StringRef OperandStr;
3068	if (!getConstantStringInfo(V: CI->getOperand(i_nocapture: 1), Str&: OperandStr))
3069	return nullptr;
3070	// printf("%s", "") --> NOP
3071	if (OperandStr.empty())
3072	return (Value *)CI;
3073	// printf("%s", "a") --> putchar('a')
3074	if (OperandStr.size() == 1) {
3075	// Convert the character to unsigned char before passing it to putchar
3076	// to avoid host-specific sign extension in the IR. Putchar converts
3077	// it to unsigned char regardless.
3078	Value *IntChar = ConstantInt::get(Ty: IntTy, V: (unsigned char)OperandStr[0]);
3079	return copyFlags(Old: *CI, New: emitPutChar(Char: IntChar, B, TLI));
3080	}
3081	// printf("%s", str"\n") --> puts(str)
3082	if (OperandStr.back() == '\n') {
3083	OperandStr = OperandStr.drop_back();
3084	Value *GV = B.CreateGlobalString(Str: OperandStr, Name: "str");
3085	return copyFlags(Old: *CI, New: emitPutS(Str: GV, B, TLI));
3086	}
3087	return nullptr;
3088	}
3089
3090	// printf("foo\n") --> puts("foo")
3091	if (FormatStr.back() == '\n' &&
3092	!FormatStr.contains(C: '%')) { // No format characters.
3093	// Create a string literal with no \n on it. We expect the constant merge
3094	// pass to be run after this pass, to merge duplicate strings.
3095	FormatStr = FormatStr.drop_back();
3096	Value *GV = B.CreateGlobalString(Str: FormatStr, Name: "str");
3097	return copyFlags(Old: *CI, New: emitPutS(Str: GV, B, TLI));
3098	}
3099
3100	// Optimize specific format strings.
3101	// printf("%c", chr) --> putchar(chr)
3102	if (FormatStr == "%c" && CI->arg_size() > 1 &&
3103	CI->getArgOperand(i: 1)->getType()->isIntegerTy()) {
3104	// Convert the argument to the type expected by putchar, i.e., int, which
3105	// need not be 32 bits wide but which is the same as printf's return type.
3106	Value *IntChar = B.CreateIntCast(V: CI->getArgOperand(i: 1), DestTy: IntTy, isSigned: false);
3107	return copyFlags(Old: *CI, New: emitPutChar(Char: IntChar, B, TLI));
3108	}
3109
3110	// printf("%s\n", str) --> puts(str)
3111	if (FormatStr == "%s\n" && CI->arg_size() > 1 &&
3112	CI->getArgOperand(i: 1)->getType()->isPointerTy())
3113	return copyFlags(Old: *CI, New: emitPutS(Str: CI->getArgOperand(i: 1), B, TLI));
3114	return nullptr;
3115	}
3116
3117	Value *LibCallSimplifier::optimizePrintF(CallInst *CI, IRBuilderBase &B) {
3118
3119	Module *M = CI->getModule();
3120	Function *Callee = CI->getCalledFunction();
3121	FunctionType *FT = Callee->getFunctionType();
3122	if (Value *V = optimizePrintFString(CI, B)) {
3123	return V;
3124	}
3125
3126	annotateNonNullNoUndefBasedOnAccess(CI, ArgNos: 0);
3127
3128	// printf(format, ...) -> iprintf(format, ...) if no floating point
3129	// arguments.
3130	if (isLibFuncEmittable(M, TLI, TheLibFunc: LibFunc_iprintf) &&
3131	!callHasFloatingPointArgument(CI)) {
3132	FunctionCallee IPrintFFn = getOrInsertLibFunc(M, TLI: *TLI, TheLibFunc: LibFunc_iprintf, T: FT,
3133	AttributeList: Callee->getAttributes());
3134	CallInst *New = cast<CallInst>(Val: CI->clone());
3135	New->setCalledFunction(IPrintFFn);
3136	B.Insert(I: New);
3137	return New;
3138	}
3139
3140	// printf(format, ...) -> __small_printf(format, ...) if no 128-bit floating point
3141	// arguments.
3142	if (isLibFuncEmittable(M, TLI, TheLibFunc: LibFunc_small_printf) &&
3143	!callHasFP128Argument(CI)) {
3144	auto SmallPrintFFn = getOrInsertLibFunc(M, TLI: *TLI, TheLibFunc: LibFunc_small_printf, T: FT,
3145	AttributeList: Callee->getAttributes());
3146	CallInst *New = cast<CallInst>(Val: CI->clone());
3147	New->setCalledFunction(SmallPrintFFn);
3148	B.Insert(I: New);
3149	return New;
3150	}
3151
3152	return nullptr;
3153	}
3154
3155	Value *LibCallSimplifier::optimizeSPrintFString(CallInst *CI,
3156	IRBuilderBase &B) {
3157	// Check for a fixed format string.
3158	StringRef FormatStr;
3159	if (!getConstantStringInfo(V: CI->getArgOperand(i: 1), Str&: FormatStr))
3160	return nullptr;
3161
3162	// If we just have a format string (nothing else crazy) transform it.
3163	Value *Dest = CI->getArgOperand(i: 0);
3164	if (CI->arg_size() == 2) {
3165	// Make sure there's no % in the constant array. We could try to handle
3166	// %% -> % in the future if we cared.
3167	if (FormatStr.contains(C: '%'))
3168	return nullptr; // we found a format specifier, bail out.
3169
3170	// sprintf(str, fmt) -> llvm.memcpy(align 1 str, align 1 fmt, strlen(fmt)+1)
3171	B.CreateMemCpy(
3172	Dst: Dest, DstAlign: Align(1), Src: CI->getArgOperand(i: 1), SrcAlign: Align(1),
3173	Size: ConstantInt::get(Ty: DL.getIntPtrType(C&: CI->getContext()),
3174	V: FormatStr.size() + 1)); // Copy the null byte.
3175	return ConstantInt::get(Ty: CI->getType(), V: FormatStr.size());
3176	}
3177
3178	// The remaining optimizations require the format string to be "%s" or "%c"
3179	// and have an extra operand.
3180	if (FormatStr.size() != 2 || FormatStr[0] != '%' || CI->arg_size() < 3)
3181	return nullptr;
3182
3183	// Decode the second character of the format string.
3184	if (FormatStr[1] == 'c') {
3185	// sprintf(dst, "%c", chr) --> *(i8*)dst = chr; *((i8*)dst+1) = 0
3186	if (!CI->getArgOperand(i: 2)->getType()->isIntegerTy())
3187	return nullptr;
3188	Value *V = B.CreateTrunc(V: CI->getArgOperand(i: 2), DestTy: B.getInt8Ty(), Name: "char");
3189	Value *Ptr = Dest;
3190	B.CreateStore(Val: V, Ptr);
3191	Ptr = B.CreateInBoundsGEP(Ty: B.getInt8Ty(), Ptr, IdxList: B.getInt32(C: 1), Name: "nul");
3192	B.CreateStore(Val: B.getInt8(C: 0), Ptr);
3193
3194	return ConstantInt::get(Ty: CI->getType(), V: 1);
3195	}
3196
3197	if (FormatStr[1] == 's') {
3198	// sprintf(dest, "%s", str) -> llvm.memcpy(align 1 dest, align 1 str,
3199	// strlen(str)+1)
3200	if (!CI->getArgOperand(i: 2)->getType()->isPointerTy())
3201	return nullptr;
3202
3203	if (CI->use_empty())
3204	// sprintf(dest, "%s", str) -> strcpy(dest, str)
3205	return copyFlags(Old: *CI, New: emitStrCpy(Dst: Dest, Src: CI->getArgOperand(i: 2), B, TLI));
3206
3207	uint64_t SrcLen = GetStringLength(V: CI->getArgOperand(i: 2));
3208	if (SrcLen) {
3209	B.CreateMemCpy(
3210	Dst: Dest, DstAlign: Align(1), Src: CI->getArgOperand(i: 2), SrcAlign: Align(1),
3211	Size: ConstantInt::get(Ty: DL.getIntPtrType(C&: CI->getContext()), V: SrcLen));
3212	// Returns total number of characters written without null-character.
3213	return ConstantInt::get(Ty: CI->getType(), V: SrcLen - 1);
3214	} else if (Value *V = emitStpCpy(Dst: Dest, Src: CI->getArgOperand(i: 2), B, TLI)) {
3215	// sprintf(dest, "%s", str) -> stpcpy(dest, str) - dest
3216	Value *PtrDiff = B.CreatePtrDiff(ElemTy: B.getInt8Ty(), LHS: V, RHS: Dest);
3217	return B.CreateIntCast(V: PtrDiff, DestTy: CI->getType(), isSigned: false);
3218	}
3219
3220	bool OptForSize = CI->getFunction()->hasOptSize() ||
3221	llvm::shouldOptimizeForSize(BB: CI->getParent(), PSI, BFI,
3222	QueryType: PGSOQueryType::IRPass);
3223	if (OptForSize)
3224	return nullptr;
3225
3226	Value *Len = emitStrLen(Ptr: CI->getArgOperand(i: 2), B, DL, TLI);
3227	if (!Len)
3228	return nullptr;
3229	Value *IncLen =
3230	B.CreateAdd(LHS: Len, RHS: ConstantInt::get(Ty: Len->getType(), V: 1), Name: "leninc");
3231	B.CreateMemCpy(Dst: Dest, DstAlign: Align(1), Src: CI->getArgOperand(i: 2), SrcAlign: Align(1), Size: IncLen);
3232
3233	// The sprintf result is the unincremented number of bytes in the string.
3234	return B.CreateIntCast(V: Len, DestTy: CI->getType(), isSigned: false);
3235	}
3236	return nullptr;
3237	}
3238
3239	Value *LibCallSimplifier::optimizeSPrintF(CallInst *CI, IRBuilderBase &B) {
3240	Module *M = CI->getModule();
3241	Function *Callee = CI->getCalledFunction();
3242	FunctionType *FT = Callee->getFunctionType();
3243	if (Value *V = optimizeSPrintFString(CI, B)) {
3244	return V;
3245	}
3246
3247	annotateNonNullNoUndefBasedOnAccess(CI, ArgNos: {0, 1});
3248
3249	// sprintf(str, format, ...) -> siprintf(str, format, ...) if no floating
3250	// point arguments.
3251	if (isLibFuncEmittable(M, TLI, TheLibFunc: LibFunc_siprintf) &&
3252	!callHasFloatingPointArgument(CI)) {
3253	FunctionCallee SIPrintFFn = getOrInsertLibFunc(M, TLI: *TLI, TheLibFunc: LibFunc_siprintf,
3254	T: FT, AttributeList: Callee->getAttributes());
3255	CallInst *New = cast<CallInst>(Val: CI->clone());
3256	New->setCalledFunction(SIPrintFFn);
3257	B.Insert(I: New);
3258	return New;
3259	}
3260
3261	// sprintf(str, format, ...) -> __small_sprintf(str, format, ...) if no 128-bit
3262	// floating point arguments.
3263	if (isLibFuncEmittable(M, TLI, TheLibFunc: LibFunc_small_sprintf) &&
3264	!callHasFP128Argument(CI)) {
3265	auto SmallSPrintFFn = getOrInsertLibFunc(M, TLI: *TLI, TheLibFunc: LibFunc_small_sprintf, T: FT,
3266	AttributeList: Callee->getAttributes());
3267	CallInst *New = cast<CallInst>(Val: CI->clone());
3268	New->setCalledFunction(SmallSPrintFFn);
3269	B.Insert(I: New);
3270	return New;
3271	}
3272
3273	return nullptr;
3274	}
3275
3276	// Transform an snprintf call CI with the bound N to format the string Str
3277	// either to a call to memcpy, or to single character a store, or to nothing,
3278	// and fold the result to a constant. A nonnull StrArg refers to the string
3279	// argument being formatted. Otherwise the call is one with N < 2 and
3280	// the "%c" directive to format a single character.
3281	Value *LibCallSimplifier::emitSnPrintfMemCpy(CallInst *CI, Value *StrArg,
3282	StringRef Str, uint64_t N,
3283	IRBuilderBase &B) {
3284	assert(StrArg || (N < 2 && Str.size() == 1));
3285
3286	unsigned IntBits = TLI->getIntSize();
3287	uint64_t IntMax = maxIntN(N: IntBits);
3288	if (Str.size() > IntMax)
3289	// Bail if the string is longer than INT_MAX. POSIX requires
3290	// implementations to set errno to EOVERFLOW in this case, in
3291	// addition to when N is larger than that (checked by the caller).
3292	return nullptr;
3293
3294	Value *StrLen = ConstantInt::get(Ty: CI->getType(), V: Str.size());
3295	if (N == 0)
3296	return StrLen;
3297
3298	// Set to the number of bytes to copy fron StrArg which is also
3299	// the offset of the terinating nul.
3300	uint64_t NCopy;
3301	if (N > Str.size())
3302	// Copy the full string, including the terminating nul (which must
3303	// be present regardless of the bound).
3304	NCopy = Str.size() + 1;
3305	else
3306	NCopy = N - 1;
3307
3308	Value *DstArg = CI->getArgOperand(i: 0);
3309	if (NCopy && StrArg)
3310	// Transform the call to lvm.memcpy(dst, fmt, N).
3311	copyFlags(
3312	Old: *CI,
3313	New: B.CreateMemCpy(
3314	Dst: DstArg, DstAlign: Align(1), Src: StrArg, SrcAlign: Align(1),
3315	Size: ConstantInt::get(Ty: DL.getIntPtrType(C&: CI->getContext()), V: NCopy)));
3316
3317	if (N > Str.size())
3318	// Return early when the whole format string, including the final nul,
3319	// has been copied.
3320	return StrLen;
3321
3322	// Otherwise, when truncating the string append a terminating nul.
3323	Type *Int8Ty = B.getInt8Ty();
3324	Value *NulOff = B.getIntN(N: IntBits, C: NCopy);
3325	Value *DstEnd = B.CreateInBoundsGEP(Ty: Int8Ty, Ptr: DstArg, IdxList: NulOff, Name: "endptr");
3326	B.CreateStore(Val: ConstantInt::get(Ty: Int8Ty, V: 0), Ptr: DstEnd);
3327	return StrLen;
3328	}
3329
3330	Value *LibCallSimplifier::optimizeSnPrintFString(CallInst *CI,
3331	IRBuilderBase &B) {
3332	// Check for size
3333	ConstantInt *Size = dyn_cast<ConstantInt>(Val: CI->getArgOperand(i: 1));
3334	if (!Size)
3335	return nullptr;
3336
3337	uint64_t N = Size->getZExtValue();
3338	uint64_t IntMax = maxIntN(N: TLI->getIntSize());
3339	if (N > IntMax)
3340	// Bail if the bound exceeds INT_MAX. POSIX requires implementations
3341	// to set errno to EOVERFLOW in this case.
3342	return nullptr;
3343
3344	Value *DstArg = CI->getArgOperand(i: 0);
3345	Value *FmtArg = CI->getArgOperand(i: 2);
3346
3347	// Check for a fixed format string.
3348	StringRef FormatStr;
3349	if (!getConstantStringInfo(V: FmtArg, Str&: FormatStr))
3350	return nullptr;
3351
3352	// If we just have a format string (nothing else crazy) transform it.
3353	if (CI->arg_size() == 3) {
3354	if (FormatStr.contains(C: '%'))
3355	// Bail if the format string contains a directive and there are
3356	// no arguments. We could handle "%%" in the future.
3357	return nullptr;
3358
3359	return emitSnPrintfMemCpy(CI, StrArg: FmtArg, Str: FormatStr, N, B);
3360	}
3361
3362	// The remaining optimizations require the format string to be "%s" or "%c"
3363	// and have an extra operand.
3364	if (FormatStr.size() != 2 || FormatStr[0] != '%' || CI->arg_size() != 4)
3365	return nullptr;
3366
3367	// Decode the second character of the format string.
3368	if (FormatStr[1] == 'c') {
3369	if (N <= 1) {
3370	// Use an arbitary string of length 1 to transform the call into
3371	// either a nul store (N == 1) or a no-op (N == 0) and fold it
3372	// to one.
3373	StringRef CharStr("*");
3374	return emitSnPrintfMemCpy(CI, StrArg: nullptr, Str: CharStr, N, B);
3375	}
3376
3377	// snprintf(dst, size, "%c", chr) --> *(i8*)dst = chr; *((i8*)dst+1) = 0
3378	if (!CI->getArgOperand(i: 3)->getType()->isIntegerTy())
3379	return nullptr;
3380	Value *V = B.CreateTrunc(V: CI->getArgOperand(i: 3), DestTy: B.getInt8Ty(), Name: "char");
3381	Value *Ptr = DstArg;
3382	B.CreateStore(Val: V, Ptr);
3383	Ptr = B.CreateInBoundsGEP(Ty: B.getInt8Ty(), Ptr, IdxList: B.getInt32(C: 1), Name: "nul");
3384	B.CreateStore(Val: B.getInt8(C: 0), Ptr);
3385	return ConstantInt::get(Ty: CI->getType(), V: 1);
3386	}
3387
3388	if (FormatStr[1] != 's')
3389	return nullptr;
3390
3391	Value *StrArg = CI->getArgOperand(i: 3);
3392	// snprintf(dest, size, "%s", str) to llvm.memcpy(dest, str, len+1, 1)
3393	StringRef Str;
3394	if (!getConstantStringInfo(V: StrArg, Str))
3395	return nullptr;
3396
3397	return emitSnPrintfMemCpy(CI, StrArg, Str, N, B);
3398	}
3399
3400	Value *LibCallSimplifier::optimizeSnPrintF(CallInst *CI, IRBuilderBase &B) {
3401	if (Value *V = optimizeSnPrintFString(CI, B)) {
3402	return V;
3403	}
3404
3405	if (isKnownNonZero(V: CI->getOperand(i_nocapture: 1), Q: DL))
3406	annotateNonNullNoUndefBasedOnAccess(CI, ArgNos: 0);
3407	return nullptr;
3408	}
3409
3410	Value *LibCallSimplifier::optimizeFPrintFString(CallInst *CI,
3411	IRBuilderBase &B) {
3412	optimizeErrorReporting(CI, B, StreamArg: 0);
3413
3414	// All the optimizations depend on the format string.
3415	StringRef FormatStr;
3416	if (!getConstantStringInfo(V: CI->getArgOperand(i: 1), Str&: FormatStr))
3417	return nullptr;
3418
3419	// Do not do any of the following transformations if the fprintf return
3420	// value is used, in general the fprintf return value is not compatible
3421	// with fwrite(), fputc() or fputs().
3422	if (!CI->use_empty())
3423	return nullptr;
3424
3425	// fprintf(F, "foo") --> fwrite("foo", 3, 1, F)
3426	if (CI->arg_size() == 2) {
3427	// Could handle %% -> % if we cared.
3428	if (FormatStr.contains(C: '%'))
3429	return nullptr; // We found a format specifier.
3430
3431	unsigned SizeTBits = TLI->getSizeTSize(M: *CI->getModule());
3432	Type *SizeTTy = IntegerType::get(C&: CI->getContext(), NumBits: SizeTBits);
3433	return copyFlags(
3434	Old: *CI, New: emitFWrite(Ptr: CI->getArgOperand(i: 1),
3435	Size: ConstantInt::get(Ty: SizeTTy, V: FormatStr.size()),
3436	File: CI->getArgOperand(i: 0), B, DL, TLI));
3437	}
3438
3439	// The remaining optimizations require the format string to be "%s" or "%c"
3440	// and have an extra operand.
3441	if (FormatStr.size() != 2 || FormatStr[0] != '%' || CI->arg_size() < 3)
3442	return nullptr;
3443
3444	// Decode the second character of the format string.
3445	if (FormatStr[1] == 'c') {
3446	// fprintf(F, "%c", chr) --> fputc((int)chr, F)
3447	if (!CI->getArgOperand(i: 2)->getType()->isIntegerTy())
3448	return nullptr;
3449	Type *IntTy = B.getIntNTy(N: TLI->getIntSize());
3450	Value *V = B.CreateIntCast(V: CI->getArgOperand(i: 2), DestTy: IntTy, /*isSigned*/ true,
3451	Name: "chari");
3452	return copyFlags(Old: *CI, New: emitFPutC(Char: V, File: CI->getArgOperand(i: 0), B, TLI));
3453	}
3454
3455	if (FormatStr[1] == 's') {
3456	// fprintf(F, "%s", str) --> fputs(str, F)
3457	if (!CI->getArgOperand(i: 2)->getType()->isPointerTy())
3458	return nullptr;
3459	return copyFlags(
3460	Old: *CI, New: emitFPutS(Str: CI->getArgOperand(i: 2), File: CI->getArgOperand(i: 0), B, TLI));
3461	}
3462	return nullptr;
3463	}
3464
3465	Value *LibCallSimplifier::optimizeFPrintF(CallInst *CI, IRBuilderBase &B) {
3466	Module *M = CI->getModule();
3467	Function *Callee = CI->getCalledFunction();
3468	FunctionType *FT = Callee->getFunctionType();
3469	if (Value *V = optimizeFPrintFString(CI, B)) {
3470	return V;
3471	}
3472
3473	// fprintf(stream, format, ...) -> fiprintf(stream, format, ...) if no
3474	// floating point arguments.
3475	if (isLibFuncEmittable(M, TLI, TheLibFunc: LibFunc_fiprintf) &&
3476	!callHasFloatingPointArgument(CI)) {
3477	FunctionCallee FIPrintFFn = getOrInsertLibFunc(M, TLI: *TLI, TheLibFunc: LibFunc_fiprintf,
3478	T: FT, AttributeList: Callee->getAttributes());
3479	CallInst *New = cast<CallInst>(Val: CI->clone());
3480	New->setCalledFunction(FIPrintFFn);
3481	B.Insert(I: New);
3482	return New;
3483	}
3484
3485	// fprintf(stream, format, ...) -> __small_fprintf(stream, format, ...) if no
3486	// 128-bit floating point arguments.
3487	if (isLibFuncEmittable(M, TLI, TheLibFunc: LibFunc_small_fprintf) &&
3488	!callHasFP128Argument(CI)) {
3489	auto SmallFPrintFFn =
3490	getOrInsertLibFunc(M, TLI: *TLI, TheLibFunc: LibFunc_small_fprintf, T: FT,
3491	AttributeList: Callee->getAttributes());
3492	CallInst *New = cast<CallInst>(Val: CI->clone());
3493	New->setCalledFunction(SmallFPrintFFn);
3494	B.Insert(I: New);
3495	return New;
3496	}
3497
3498	return nullptr;
3499	}
3500
3501	Value *LibCallSimplifier::optimizeFWrite(CallInst *CI, IRBuilderBase &B) {
3502	optimizeErrorReporting(CI, B, StreamArg: 3);
3503
3504	// Get the element size and count.
3505	ConstantInt *SizeC = dyn_cast<ConstantInt>(Val: CI->getArgOperand(i: 1));
3506	ConstantInt *CountC = dyn_cast<ConstantInt>(Val: CI->getArgOperand(i: 2));
3507	if (SizeC && CountC) {
3508	uint64_t Bytes = SizeC->getZExtValue() * CountC->getZExtValue();
3509
3510	// If this is writing zero records, remove the call (it's a noop).
3511	if (Bytes == 0)
3512	return ConstantInt::get(Ty: CI->getType(), V: 0);
3513
3514	// If this is writing one byte, turn it into fputc.
3515	// This optimisation is only valid, if the return value is unused.
3516	if (Bytes == 1 && CI->use_empty()) { // fwrite(S,1,1,F) -> fputc(S[0],F)
3517	Value *Char = B.CreateLoad(Ty: B.getInt8Ty(), Ptr: CI->getArgOperand(i: 0), Name: "char");
3518	Type *IntTy = B.getIntNTy(N: TLI->getIntSize());
3519	Value *Cast = B.CreateIntCast(V: Char, DestTy: IntTy, /*isSigned*/ true, Name: "chari");
3520	Value *NewCI = emitFPutC(Char: Cast, File: CI->getArgOperand(i: 3), B, TLI);
3521	return NewCI ? ConstantInt::get(Ty: CI->getType(), V: 1) : nullptr;
3522	}
3523	}
3524
3525	return nullptr;
3526	}
3527
3528	Value *LibCallSimplifier::optimizeFPuts(CallInst *CI, IRBuilderBase &B) {
3529	optimizeErrorReporting(CI, B, StreamArg: 1);
3530
3531	// Don't rewrite fputs to fwrite when optimising for size because fwrite
3532	// requires more arguments and thus extra MOVs are required.
3533	bool OptForSize = CI->getFunction()->hasOptSize() ||
3534	llvm::shouldOptimizeForSize(BB: CI->getParent(), PSI, BFI,
3535	QueryType: PGSOQueryType::IRPass);
3536	if (OptForSize)
3537	return nullptr;
3538
3539	// We can't optimize if return value is used.
3540	if (!CI->use_empty())
3541	return nullptr;
3542
3543	// fputs(s,F) --> fwrite(s,strlen(s),1,F)
3544	uint64_t Len = GetStringLength(V: CI->getArgOperand(i: 0));
3545	if (!Len)
3546	return nullptr;
3547
3548	// Known to have no uses (see above).
3549	unsigned SizeTBits = TLI->getSizeTSize(M: *CI->getModule());
3550	Type *SizeTTy = IntegerType::get(C&: CI->getContext(), NumBits: SizeTBits);
3551	return copyFlags(
3552	Old: *CI,
3553	New: emitFWrite(Ptr: CI->getArgOperand(i: 0),
3554	Size: ConstantInt::get(Ty: SizeTTy, V: Len - 1),
3555	File: CI->getArgOperand(i: 1), B, DL, TLI));
3556	}
3557
3558	Value *LibCallSimplifier::optimizePuts(CallInst *CI, IRBuilderBase &B) {
3559	annotateNonNullNoUndefBasedOnAccess(CI, ArgNos: 0);
3560	if (!CI->use_empty())
3561	return nullptr;
3562
3563	// Check for a constant string.
3564	// puts("") -> putchar('\n')
3565	StringRef Str;
3566	if (getConstantStringInfo(V: CI->getArgOperand(i: 0), Str) && Str.empty()) {
3567	// putchar takes an argument of the same type as puts returns, i.e.,
3568	// int, which need not be 32 bits wide.
3569	Type *IntTy = CI->getType();
3570	return copyFlags(Old: *CI, New: emitPutChar(Char: ConstantInt::get(Ty: IntTy, V: '\n'), B, TLI));
3571	}
3572
3573	return nullptr;
3574	}
3575
3576	Value *LibCallSimplifier::optimizeBCopy(CallInst *CI, IRBuilderBase &B) {
3577	// bcopy(src, dst, n) -> llvm.memmove(dst, src, n)
3578	return copyFlags(Old: *CI, New: B.CreateMemMove(Dst: CI->getArgOperand(i: 1), DstAlign: Align(1),
3579	Src: CI->getArgOperand(i: 0), SrcAlign: Align(1),
3580	Size: CI->getArgOperand(i: 2)));
3581	}
3582
3583	bool LibCallSimplifier::hasFloatVersion(const Module *M, StringRef FuncName) {
3584	SmallString<20> FloatFuncName = FuncName;
3585	FloatFuncName += 'f';
3586	return isLibFuncEmittable(M, TLI, Name: FloatFuncName);
3587	}
3588
3589	Value *LibCallSimplifier::optimizeStringMemoryLibCall(CallInst *CI,
3590	IRBuilderBase &Builder) {
3591	Module *M = CI->getModule();
3592	LibFunc Func;
3593	Function *Callee = CI->getCalledFunction();
3594	// Check for string/memory library functions.
3595	if (TLI->getLibFunc(FDecl: *Callee, F&: Func) && isLibFuncEmittable(M, TLI, TheLibFunc: Func)) {
3596	// Make sure we never change the calling convention.
3597	assert(
3598	(ignoreCallingConv(Func) ||
3599	TargetLibraryInfoImpl::isCallingConvCCompatible(CI)) &&
3600	"Optimizing string/memory libcall would change the calling convention");
3601	switch (Func) {
3602	case LibFunc_strcat:
3603	return optimizeStrCat(CI, B&: Builder);
3604	case LibFunc_strncat:
3605	return optimizeStrNCat(CI, B&: Builder);
3606	case LibFunc_strchr:
3607	return optimizeStrChr(CI, B&: Builder);
3608	case LibFunc_strrchr:
3609	return optimizeStrRChr(CI, B&: Builder);
3610	case LibFunc_strcmp:
3611	return optimizeStrCmp(CI, B&: Builder);
3612	case LibFunc_strncmp:
3613	return optimizeStrNCmp(CI, B&: Builder);
3614	case LibFunc_strcpy:
3615	return optimizeStrCpy(CI, B&: Builder);
3616	case LibFunc_stpcpy:
3617	return optimizeStpCpy(CI, B&: Builder);
3618	case LibFunc_strlcpy:
3619	return optimizeStrLCpy(CI, B&: Builder);
3620	case LibFunc_stpncpy:
3621	return optimizeStringNCpy(CI, /*RetEnd=*/true, B&: Builder);
3622	case LibFunc_strncpy:
3623	return optimizeStringNCpy(CI, /*RetEnd=*/false, B&: Builder);
3624	case LibFunc_strlen:
3625	return optimizeStrLen(CI, B&: Builder);
3626	case LibFunc_strnlen:
3627	return optimizeStrNLen(CI, B&: Builder);
3628	case LibFunc_strpbrk:
3629	return optimizeStrPBrk(CI, B&: Builder);
3630	case LibFunc_strndup:
3631	return optimizeStrNDup(CI, B&: Builder);
3632	case LibFunc_strtol:
3633	case LibFunc_strtod:
3634	case LibFunc_strtof:
3635	case LibFunc_strtoul:
3636	case LibFunc_strtoll:
3637	case LibFunc_strtold:
3638	case LibFunc_strtoull:
3639	return optimizeStrTo(CI, B&: Builder);
3640	case LibFunc_strspn:
3641	return optimizeStrSpn(CI, B&: Builder);
3642	case LibFunc_strcspn:
3643	return optimizeStrCSpn(CI, B&: Builder);
3644	case LibFunc_strstr:
3645	return optimizeStrStr(CI, B&: Builder);
3646	case LibFunc_memchr:
3647	return optimizeMemChr(CI, B&: Builder);
3648	case LibFunc_memrchr:
3649	return optimizeMemRChr(CI, B&: Builder);
3650	case LibFunc_bcmp:
3651	return optimizeBCmp(CI, B&: Builder);
3652	case LibFunc_memcmp:
3653	return optimizeMemCmp(CI, B&: Builder);
3654	case LibFunc_memcpy:
3655	return optimizeMemCpy(CI, B&: Builder);
3656	case LibFunc_memccpy:
3657	return optimizeMemCCpy(CI, B&: Builder);
3658	case LibFunc_mempcpy:
3659	return optimizeMemPCpy(CI, B&: Builder);
3660	case LibFunc_memmove:
3661	return optimizeMemMove(CI, B&: Builder);
3662	case LibFunc_memset:
3663	return optimizeMemSet(CI, B&: Builder);
3664	case LibFunc_realloc:
3665	return optimizeRealloc(CI, B&: Builder);
3666	case LibFunc_wcslen:
3667	return optimizeWcslen(CI, B&: Builder);
3668	case LibFunc_bcopy:
3669	return optimizeBCopy(CI, B&: Builder);
3670	case LibFunc_Znwm:
3671	case LibFunc_ZnwmRKSt9nothrow_t:
3672	case LibFunc_ZnwmSt11align_val_t:
3673	case LibFunc_ZnwmSt11align_val_tRKSt9nothrow_t:
3674	case LibFunc_Znam:
3675	case LibFunc_ZnamRKSt9nothrow_t:
3676	case LibFunc_ZnamSt11align_val_t:
3677	case LibFunc_ZnamSt11align_val_tRKSt9nothrow_t:
3678	return optimizeNew(CI, B&: Builder, Func);
3679	default:
3680	break;
3681	}
3682	}
3683	return nullptr;
3684	}
3685
3686	Value *LibCallSimplifier::optimizeFloatingPointLibCall(CallInst *CI,
3687	LibFunc Func,
3688	IRBuilderBase &Builder) {
3689	const Module *M = CI->getModule();
3690
3691	// Don't optimize calls that require strict floating point semantics.
3692	if (CI->isStrictFP())
3693	return nullptr;
3694
3695	if (Value *V = optimizeSymmetric(CI, Func, B&: Builder))
3696	return V;
3697
3698	switch (Func) {
3699	case LibFunc_sinpif:
3700	case LibFunc_sinpi:
3701	return optimizeSinCosPi(CI, /*IsSin*/true, B&: Builder);
3702	case LibFunc_cospif:
3703	case LibFunc_cospi:
3704	return optimizeSinCosPi(CI, /*IsSin*/false, B&: Builder);
3705	case LibFunc_powf:
3706	case LibFunc_pow:
3707	case LibFunc_powl:
3708	return optimizePow(Pow: CI, B&: Builder);
3709	case LibFunc_exp2l:
3710	case LibFunc_exp2:
3711	case LibFunc_exp2f:
3712	return optimizeExp2(CI, B&: Builder);
3713	case LibFunc_fabsf:
3714	case LibFunc_fabs:
3715	case LibFunc_fabsl:
3716	return replaceUnaryCall(CI, Builder, Intrinsic::fabs);
3717	case LibFunc_sqrtf:
3718	case LibFunc_sqrt:
3719	case LibFunc_sqrtl:
3720	return optimizeSqrt(CI, B&: Builder);
3721	case LibFunc_logf:
3722	case LibFunc_log:
3723	case LibFunc_logl:
3724	case LibFunc_log10f:
3725	case LibFunc_log10:
3726	case LibFunc_log10l:
3727	case LibFunc_log1pf:
3728	case LibFunc_log1p:
3729	case LibFunc_log1pl:
3730	case LibFunc_log2f:
3731	case LibFunc_log2:
3732	case LibFunc_log2l:
3733	case LibFunc_logbf:
3734	case LibFunc_logb:
3735	case LibFunc_logbl:
3736	return optimizeLog(Log: CI, B&: Builder);
3737	case LibFunc_tan:
3738	case LibFunc_tanf:
3739	case LibFunc_tanl:
3740	case LibFunc_sinh:
3741	case LibFunc_sinhf:
3742	case LibFunc_sinhl:
3743	case LibFunc_asinh:
3744	case LibFunc_asinhf:
3745	case LibFunc_asinhl:
3746	case LibFunc_cosh:
3747	case LibFunc_coshf:
3748	case LibFunc_coshl:
3749	case LibFunc_atanh:
3750	case LibFunc_atanhf:
3751	case LibFunc_atanhl:
3752	return optimizeTrigInversionPairs(CI, B&: Builder);
3753	case LibFunc_ceil:
3754	return replaceUnaryCall(CI, Builder, Intrinsic::ceil);
3755	case LibFunc_floor:
3756	return replaceUnaryCall(CI, Builder, Intrinsic::floor);
3757	case LibFunc_round:
3758	return replaceUnaryCall(CI, Builder, Intrinsic::round);
3759	case LibFunc_roundeven:
3760	return replaceUnaryCall(CI, Builder, Intrinsic::roundeven);
3761	case LibFunc_nearbyint:
3762	return replaceUnaryCall(CI, Builder, Intrinsic::nearbyint);
3763	case LibFunc_rint:
3764	return replaceUnaryCall(CI, Builder, Intrinsic::rint);
3765	case LibFunc_trunc:
3766	return replaceUnaryCall(CI, Builder, Intrinsic::trunc);
3767	case LibFunc_acos:
3768	case LibFunc_acosh:
3769	case LibFunc_asin:
3770	case LibFunc_atan:
3771	case LibFunc_cbrt:
3772	case LibFunc_exp:
3773	case LibFunc_exp10:
3774	case LibFunc_expm1:
3775	case LibFunc_cos:
3776	case LibFunc_sin:
3777	case LibFunc_tanh:
3778	if (UnsafeFPShrink && hasFloatVersion(M, FuncName: CI->getCalledFunction()->getName()))
3779	return optimizeUnaryDoubleFP(CI, B&: Builder, TLI, isPrecise: true);
3780	return nullptr;
3781	case LibFunc_copysign:
3782	if (hasFloatVersion(M, FuncName: CI->getCalledFunction()->getName()))
3783	return optimizeBinaryDoubleFP(CI, B&: Builder, TLI);
3784	return nullptr;
3785	case LibFunc_fminf:
3786	case LibFunc_fmin:
3787	case LibFunc_fminl:
3788	case LibFunc_fmaxf:
3789	case LibFunc_fmax:
3790	case LibFunc_fmaxl:
3791	return optimizeFMinFMax(CI, B&: Builder);
3792	case LibFunc_cabs:
3793	case LibFunc_cabsf:
3794	case LibFunc_cabsl:
3795	return optimizeCAbs(CI, B&: Builder);
3796	default:
3797	return nullptr;
3798	}
3799	}
3800
3801	Value *LibCallSimplifier::optimizeCall(CallInst *CI, IRBuilderBase &Builder) {
3802	Module *M = CI->getModule();
3803	assert(!CI->isMustTailCall() && "These transforms aren't musttail safe.");
3804
3805	// TODO: Split out the code below that operates on FP calls so that
3806	// we can all non-FP calls with the StrictFP attribute to be
3807	// optimized.
3808	if (CI->isNoBuiltin())
3809	return nullptr;
3810
3811	LibFunc Func;
3812	Function *Callee = CI->getCalledFunction();
3813	bool IsCallingConvC = TargetLibraryInfoImpl::isCallingConvCCompatible(CI);
3814
3815	SmallVector<OperandBundleDef, 2> OpBundles;
3816	CI->getOperandBundlesAsDefs(Defs&: OpBundles);
3817
3818	IRBuilderBase::OperandBundlesGuard Guard(Builder);
3819	Builder.setDefaultOperandBundles(OpBundles);
3820
3821	// Command-line parameter overrides instruction attribute.
3822	// This can't be moved to optimizeFloatingPointLibCall() because it may be
3823	// used by the intrinsic optimizations.
3824	if (EnableUnsafeFPShrink.getNumOccurrences() > 0)
3825	UnsafeFPShrink = EnableUnsafeFPShrink;
3826	else if (isa<FPMathOperator>(Val: CI) && CI->isFast())
3827	UnsafeFPShrink = true;
3828
3829	// First, check for intrinsics.
3830	if (IntrinsicInst *II = dyn_cast<IntrinsicInst>(Val: CI)) {
3831	if (!IsCallingConvC)
3832	return nullptr;
3833	// The FP intrinsics have corresponding constrained versions so we don't
3834	// need to check for the StrictFP attribute here.
3835	switch (II->getIntrinsicID()) {
3836	case Intrinsic::pow:
3837	return optimizePow(Pow: CI, B&: Builder);
3838	case Intrinsic::exp2:
3839	return optimizeExp2(CI, B&: Builder);
3840	case Intrinsic::log:
3841	case Intrinsic::log2:
3842	case Intrinsic::log10:
3843	return optimizeLog(Log: CI, B&: Builder);
3844	case Intrinsic::sqrt:
3845	return optimizeSqrt(CI, B&: Builder);
3846	case Intrinsic::memset:
3847	return optimizeMemSet(CI, B&: Builder);
3848	case Intrinsic::memcpy:
3849	return optimizeMemCpy(CI, B&: Builder);
3850	case Intrinsic::memmove:
3851	return optimizeMemMove(CI, B&: Builder);
3852	default:
3853	return nullptr;
3854	}
3855	}
3856
3857	// Also try to simplify calls to fortified library functions.
3858	if (Value *SimplifiedFortifiedCI =
3859	FortifiedSimplifier.optimizeCall(CI, B&: Builder))
3860	return SimplifiedFortifiedCI;
3861
3862	// Then check for known library functions.
3863	if (TLI->getLibFunc(FDecl: *Callee, F&: Func) && isLibFuncEmittable(M, TLI, TheLibFunc: Func)) {
3864	// We never change the calling convention.
3865	if (!ignoreCallingConv(Func) && !IsCallingConvC)
3866	return nullptr;
3867	if (Value *V = optimizeStringMemoryLibCall(CI, Builder))
3868	return V;
3869	if (Value *V = optimizeFloatingPointLibCall(CI, Func, Builder))
3870	return V;
3871	switch (Func) {
3872	case LibFunc_ffs:
3873	case LibFunc_ffsl:
3874	case LibFunc_ffsll:
3875	return optimizeFFS(CI, B&: Builder);
3876	case LibFunc_fls:
3877	case LibFunc_flsl:
3878	case LibFunc_flsll:
3879	return optimizeFls(CI, B&: Builder);
3880	case LibFunc_abs:
3881	case LibFunc_labs:
3882	case LibFunc_llabs:
3883	return optimizeAbs(CI, B&: Builder);
3884	case LibFunc_isdigit:
3885	return optimizeIsDigit(CI, B&: Builder);
3886	case LibFunc_isascii:
3887	return optimizeIsAscii(CI, B&: Builder);
3888	case LibFunc_toascii:
3889	return optimizeToAscii(CI, B&: Builder);
3890	case LibFunc_atoi:
3891	case LibFunc_atol:
3892	case LibFunc_atoll:
3893	return optimizeAtoi(CI, B&: Builder);
3894	case LibFunc_strtol:
3895	case LibFunc_strtoll:
3896	return optimizeStrToInt(CI, B&: Builder, /*AsSigned=*/true);
3897	case LibFunc_strtoul:
3898	case LibFunc_strtoull:
3899	return optimizeStrToInt(CI, B&: Builder, /*AsSigned=*/false);
3900	case LibFunc_printf:
3901	return optimizePrintF(CI, B&: Builder);
3902	case LibFunc_sprintf:
3903	return optimizeSPrintF(CI, B&: Builder);
3904	case LibFunc_snprintf:
3905	return optimizeSnPrintF(CI, B&: Builder);
3906	case LibFunc_fprintf:
3907	return optimizeFPrintF(CI, B&: Builder);
3908	case LibFunc_fwrite:
3909	return optimizeFWrite(CI, B&: Builder);
3910	case LibFunc_fputs:
3911	return optimizeFPuts(CI, B&: Builder);
3912	case LibFunc_puts:
3913	return optimizePuts(CI, B&: Builder);
3914	case LibFunc_perror:
3915	return optimizeErrorReporting(CI, B&: Builder);
3916	case LibFunc_vfprintf:
3917	case LibFunc_fiprintf:
3918	return optimizeErrorReporting(CI, B&: Builder, StreamArg: 0);
3919	default:
3920	return nullptr;
3921	}
3922	}
3923	return nullptr;
3924	}
3925
3926	LibCallSimplifier::LibCallSimplifier(
3927	const DataLayout &DL, const TargetLibraryInfo *TLI, AssumptionCache *AC,
3928	OptimizationRemarkEmitter &ORE, BlockFrequencyInfo *BFI,
3929	ProfileSummaryInfo *PSI,
3930	function_ref<void(Instruction *, Value *)> Replacer,
3931	function_ref<void(Instruction *)> Eraser)
3932	: FortifiedSimplifier(TLI), DL(DL), TLI(TLI), AC(AC), ORE(ORE), BFI(BFI),
3933	PSI(PSI), Replacer(Replacer), Eraser(Eraser) {}
3934
3935	void LibCallSimplifier::replaceAllUsesWith(Instruction *I, Value *With) {
3936	// Indirect through the replacer used in this instance.
3937	Replacer(I, With);
3938	}
3939
3940	void LibCallSimplifier::eraseFromParent(Instruction *I) {
3941	Eraser(I);
3942	}
3943
3944	// TODO:
3945	// Additional cases that we need to add to this file:
3946	//
3947	// cbrt:
3948	// * cbrt(expN(X)) -> expN(x/3)
3949	// * cbrt(sqrt(x)) -> pow(x,1/6)
3950	// * cbrt(cbrt(x)) -> pow(x,1/9)
3951	//
3952	// exp, expf, expl:
3953	// * exp(log(x)) -> x
3954	//
3955	// log, logf, logl:
3956	// * log(exp(x)) -> x
3957	// * log(exp(y)) -> y*log(e)
3958	// * log(exp10(y)) -> y*log(10)
3959	// * log(sqrt(x)) -> 0.5*log(x)
3960	//
3961	// pow, powf, powl:
3962	// * pow(sqrt(x),y) -> pow(x,y*0.5)
3963	// * pow(pow(x,y),z)-> pow(x,y*z)
3964	//
3965	// signbit:
3966	// * signbit(cnst) -> cnst'
3967	// * signbit(nncst) -> 0 (if pstv is a non-negative constant)
3968	//
3969	// sqrt, sqrtf, sqrtl:
3970	// * sqrt(expN(x)) -> expN(x*0.5)
3971	// * sqrt(Nroot(x)) -> pow(x,1/(2*N))
3972	// * sqrt(pow(x,y)) -> pow(|x|,y*0.5)
3973	//
3974
3975	//===----------------------------------------------------------------------===//
3976	// Fortified Library Call Optimizations
3977	//===----------------------------------------------------------------------===//
3978
3979	bool FortifiedLibCallSimplifier::isFortifiedCallFoldable(
3980	CallInst *CI, unsigned ObjSizeOp, std::optional<unsigned> SizeOp,
3981	std::optional<unsigned> StrOp, std::optional<unsigned> FlagOp) {
3982	// If this function takes a flag argument, the implementation may use it to
3983	// perform extra checks. Don't fold into the non-checking variant.
3984	if (FlagOp) {
3985	ConstantInt *Flag = dyn_cast<ConstantInt>(Val: CI->getArgOperand(i: *FlagOp));
3986	if (!Flag || !Flag->isZero())
3987	return false;
3988	}
3989
3990	if (SizeOp && CI->getArgOperand(i: ObjSizeOp) == CI->getArgOperand(i: *SizeOp))
3991	return true;
3992
3993	if (ConstantInt *ObjSizeCI =
3994	dyn_cast<ConstantInt>(Val: CI->getArgOperand(i: ObjSizeOp))) {
3995	if (ObjSizeCI->isMinusOne())
3996	return true;
3997	// If the object size wasn't -1 (unknown), bail out if we were asked to.
3998	if (OnlyLowerUnknownSize)
3999	return false;
4000	if (StrOp) {
4001	uint64_t Len = GetStringLength(V: CI->getArgOperand(i: *StrOp));
4002	// If the length is 0 we don't know how long it is and so we can't
4003	// remove the check.
4004	if (Len)
4005	annotateDereferenceableBytes(CI, ArgNos: *StrOp, DereferenceableBytes: Len);
4006	else
4007	return false;
4008	return ObjSizeCI->getZExtValue() >= Len;
4009	}
4010
4011	if (SizeOp) {
4012	if (ConstantInt *SizeCI =
4013	dyn_cast<ConstantInt>(Val: CI->getArgOperand(i: *SizeOp)))
4014	return ObjSizeCI->getZExtValue() >= SizeCI->getZExtValue();
4015	}
4016	}
4017	return false;
4018	}
4019
4020	Value *FortifiedLibCallSimplifier::optimizeMemCpyChk(CallInst *CI,
4021	IRBuilderBase &B) {
4022	if (isFortifiedCallFoldable(CI, ObjSizeOp: 3, SizeOp: 2)) {
4023	CallInst *NewCI =
4024	B.CreateMemCpy(Dst: CI->getArgOperand(i: 0), DstAlign: Align(1), Src: CI->getArgOperand(i: 1),
4025	SrcAlign: Align(1), Size: CI->getArgOperand(i: 2));
4026	mergeAttributesAndFlags(NewCI, Old: *CI);
4027	return CI->getArgOperand(i: 0);
4028	}
4029	return nullptr;
4030	}
4031
4032	Value *FortifiedLibCallSimplifier::optimizeMemMoveChk(CallInst *CI,
4033	IRBuilderBase &B) {
4034	if (isFortifiedCallFoldable(CI, ObjSizeOp: 3, SizeOp: 2)) {
4035	CallInst *NewCI =
4036	B.CreateMemMove(Dst: CI->getArgOperand(i: 0), DstAlign: Align(1), Src: CI->getArgOperand(i: 1),
4037	SrcAlign: Align(1), Size: CI->getArgOperand(i: 2));
4038	mergeAttributesAndFlags(NewCI, Old: *CI);
4039	return CI->getArgOperand(i: 0);
4040	}
4041	return nullptr;
4042	}
4043
4044	Value *FortifiedLibCallSimplifier::optimizeMemSetChk(CallInst *CI,
4045	IRBuilderBase &B) {
4046	if (isFortifiedCallFoldable(CI, ObjSizeOp: 3, SizeOp: 2)) {
4047	Value *Val = B.CreateIntCast(V: CI->getArgOperand(i: 1), DestTy: B.getInt8Ty(), isSigned: false);
4048	CallInst *NewCI = B.CreateMemSet(Ptr: CI->getArgOperand(i: 0), Val,
4049	Size: CI->getArgOperand(i: 2), Align: Align(1));
4050	mergeAttributesAndFlags(NewCI, Old: *CI);
4051	return CI->getArgOperand(i: 0);
4052	}
4053	return nullptr;
4054	}
4055
4056	Value *FortifiedLibCallSimplifier::optimizeMemPCpyChk(CallInst *CI,
4057	IRBuilderBase &B) {
4058	const DataLayout &DL = CI->getModule()->getDataLayout();
4059	if (isFortifiedCallFoldable(CI, ObjSizeOp: 3, SizeOp: 2))
4060	if (Value *Call = emitMemPCpy(Dst: CI->getArgOperand(i: 0), Src: CI->getArgOperand(i: 1),
4061	Len: CI->getArgOperand(i: 2), B, DL, TLI)) {
4062	return mergeAttributesAndFlags(NewCI: cast<CallInst>(Val: Call), Old: *CI);
4063	}
4064	return nullptr;
4065	}
4066
4067	Value *FortifiedLibCallSimplifier::optimizeStrpCpyChk(CallInst *CI,
4068	IRBuilderBase &B,
4069	LibFunc Func) {
4070	const DataLayout &DL = CI->getModule()->getDataLayout();
4071	Value *Dst = CI->getArgOperand(i: 0), *Src = CI->getArgOperand(i: 1),
4072	*ObjSize = CI->getArgOperand(i: 2);
4073
4074	// __stpcpy_chk(x,x,...) -> x+strlen(x)
4075	if (Func == LibFunc_stpcpy_chk && !OnlyLowerUnknownSize && Dst == Src) {
4076	Value *StrLen = emitStrLen(Ptr: Src, B, DL, TLI);
4077	return StrLen ? B.CreateInBoundsGEP(Ty: B.getInt8Ty(), Ptr: Dst, IdxList: StrLen) : nullptr;
4078	}
4079
4080	// If a) we don't have any length information, or b) we know this will
4081	// fit then just lower to a plain st[rp]cpy. Otherwise we'll keep our
4082	// st[rp]cpy_chk call which may fail at runtime if the size is too long.
4083	// TODO: It might be nice to get a maximum length out of the possible
4084	// string lengths for varying.
4085	if (isFortifiedCallFoldable(CI, ObjSizeOp: 2, SizeOp: std::nullopt, StrOp: 1)) {
4086	if (Func == LibFunc_strcpy_chk)
4087	return copyFlags(Old: *CI, New: emitStrCpy(Dst, Src, B, TLI));
4088	else
4089	return copyFlags(Old: *CI, New: emitStpCpy(Dst, Src, B, TLI));
4090	}
4091
4092	if (OnlyLowerUnknownSize)
4093	return nullptr;
4094
4095	// Maybe we can stil fold __st[rp]cpy_chk to __memcpy_chk.
4096	uint64_t Len = GetStringLength(V: Src);
4097	if (Len)
4098	annotateDereferenceableBytes(CI, ArgNos: 1, DereferenceableBytes: Len);
4099	else
4100	return nullptr;
4101
4102	unsigned SizeTBits = TLI->getSizeTSize(M: *CI->getModule());
4103	Type *SizeTTy = IntegerType::get(C&: CI->getContext(), NumBits: SizeTBits);
4104	Value *LenV = ConstantInt::get(Ty: SizeTTy, V: Len);
4105	Value *Ret = emitMemCpyChk(Dst, Src, Len: LenV, ObjSize, B, DL, TLI);
4106	// If the function was an __stpcpy_chk, and we were able to fold it into
4107	// a __memcpy_chk, we still need to return the correct end pointer.
4108	if (Ret && Func == LibFunc_stpcpy_chk)
4109	return B.CreateInBoundsGEP(Ty: B.getInt8Ty(), Ptr: Dst,
4110	IdxList: ConstantInt::get(Ty: SizeTTy, V: Len - 1));
4111	return copyFlags(Old: *CI, New: cast<CallInst>(Val: Ret));
4112	}
4113
4114	Value *FortifiedLibCallSimplifier::optimizeStrLenChk(CallInst *CI,
4115	IRBuilderBase &B) {
4116	if (isFortifiedCallFoldable(CI, ObjSizeOp: 1, SizeOp: std::nullopt, StrOp: 0))
4117	return copyFlags(Old: *CI, New: emitStrLen(Ptr: CI->getArgOperand(i: 0), B,
4118	DL: CI->getModule()->getDataLayout(), TLI));
4119	return nullptr;
4120	}
4121
4122	Value *FortifiedLibCallSimplifier::optimizeStrpNCpyChk(CallInst *CI,
4123	IRBuilderBase &B,
4124	LibFunc Func) {
4125	if (isFortifiedCallFoldable(CI, ObjSizeOp: 3, SizeOp: 2)) {
4126	if (Func == LibFunc_strncpy_chk)
4127	return copyFlags(Old: *CI,
4128	New: emitStrNCpy(Dst: CI->getArgOperand(i: 0), Src: CI->getArgOperand(i: 1),
4129	Len: CI->getArgOperand(i: 2), B, TLI));
4130	else
4131	return copyFlags(Old: *CI,
4132	New: emitStpNCpy(Dst: CI->getArgOperand(i: 0), Src: CI->getArgOperand(i: 1),
4133	Len: CI->getArgOperand(i: 2), B, TLI));
4134	}
4135
4136	return nullptr;
4137	}
4138
4139	Value *FortifiedLibCallSimplifier::optimizeMemCCpyChk(CallInst *CI,
4140	IRBuilderBase &B) {
4141	if (isFortifiedCallFoldable(CI, ObjSizeOp: 4, SizeOp: 3))
4142	return copyFlags(
4143	Old: *CI, New: emitMemCCpy(Ptr1: CI->getArgOperand(i: 0), Ptr2: CI->getArgOperand(i: 1),
4144	Val: CI->getArgOperand(i: 2), Len: CI->getArgOperand(i: 3), B, TLI));
4145
4146	return nullptr;
4147	}
4148
4149	Value *FortifiedLibCallSimplifier::optimizeSNPrintfChk(CallInst *CI,
4150	IRBuilderBase &B) {
4151	if (isFortifiedCallFoldable(CI, ObjSizeOp: 3, SizeOp: 1, StrOp: std::nullopt, FlagOp: 2)) {
4152	SmallVector<Value *, 8> VariadicArgs(drop_begin(RangeOrContainer: CI->args(), N: 5));
4153	return copyFlags(Old: *CI,
4154	New: emitSNPrintf(Dest: CI->getArgOperand(i: 0), Size: CI->getArgOperand(i: 1),
4155	Fmt: CI->getArgOperand(i: 4), Args: VariadicArgs, B, TLI));
4156	}
4157
4158	return nullptr;
4159	}
4160
4161	Value *FortifiedLibCallSimplifier::optimizeSPrintfChk(CallInst *CI,
4162	IRBuilderBase &B) {
4163	if (isFortifiedCallFoldable(CI, ObjSizeOp: 2, SizeOp: std::nullopt, StrOp: std::nullopt, FlagOp: 1)) {
4164	SmallVector<Value *, 8> VariadicArgs(drop_begin(RangeOrContainer: CI->args(), N: 4));
4165	return copyFlags(Old: *CI,
4166	New: emitSPrintf(Dest: CI->getArgOperand(i: 0), Fmt: CI->getArgOperand(i: 3),
4167	VariadicArgs, B, TLI));
4168	}
4169
4170	return nullptr;
4171	}
4172
4173	Value *FortifiedLibCallSimplifier::optimizeStrCatChk(CallInst *CI,
4174	IRBuilderBase &B) {
4175	if (isFortifiedCallFoldable(CI, ObjSizeOp: 2))
4176	return copyFlags(
4177	Old: *CI, New: emitStrCat(Dest: CI->getArgOperand(i: 0), Src: CI->getArgOperand(i: 1), B, TLI));
4178
4179	return nullptr;
4180	}
4181
4182	Value *FortifiedLibCallSimplifier::optimizeStrLCat(CallInst *CI,
4183	IRBuilderBase &B) {
4184	if (isFortifiedCallFoldable(CI, ObjSizeOp: 3))
4185	return copyFlags(Old: *CI,
4186	New: emitStrLCat(Dest: CI->getArgOperand(i: 0), Src: CI->getArgOperand(i: 1),
4187	Size: CI->getArgOperand(i: 2), B, TLI));
4188
4189	return nullptr;
4190	}
4191
4192	Value *FortifiedLibCallSimplifier::optimizeStrNCatChk(CallInst *CI,
4193	IRBuilderBase &B) {
4194	if (isFortifiedCallFoldable(CI, ObjSizeOp: 3))
4195	return copyFlags(Old: *CI,
4196	New: emitStrNCat(Dest: CI->getArgOperand(i: 0), Src: CI->getArgOperand(i: 1),
4197	Size: CI->getArgOperand(i: 2), B, TLI));
4198
4199	return nullptr;
4200	}
4201
4202	Value *FortifiedLibCallSimplifier::optimizeStrLCpyChk(CallInst *CI,
4203	IRBuilderBase &B) {
4204	if (isFortifiedCallFoldable(CI, ObjSizeOp: 3))
4205	return copyFlags(Old: *CI,
4206	New: emitStrLCpy(Dest: CI->getArgOperand(i: 0), Src: CI->getArgOperand(i: 1),
4207	Size: CI->getArgOperand(i: 2), B, TLI));
4208
4209	return nullptr;
4210	}
4211
4212	Value *FortifiedLibCallSimplifier::optimizeVSNPrintfChk(CallInst *CI,
4213	IRBuilderBase &B) {
4214	if (isFortifiedCallFoldable(CI, ObjSizeOp: 3, SizeOp: 1, StrOp: std::nullopt, FlagOp: 2))
4215	return copyFlags(
4216	Old: *CI, New: emitVSNPrintf(Dest: CI->getArgOperand(i: 0), Size: CI->getArgOperand(i: 1),
4217	Fmt: CI->getArgOperand(i: 4), VAList: CI->getArgOperand(i: 5), B, TLI));
4218
4219	return nullptr;
4220	}
4221
4222	Value *FortifiedLibCallSimplifier::optimizeVSPrintfChk(CallInst *CI,
4223	IRBuilderBase &B) {
4224	if (isFortifiedCallFoldable(CI, ObjSizeOp: 2, SizeOp: std::nullopt, StrOp: std::nullopt, FlagOp: 1))
4225	return copyFlags(Old: *CI,
4226	New: emitVSPrintf(Dest: CI->getArgOperand(i: 0), Fmt: CI->getArgOperand(i: 3),
4227	VAList: CI->getArgOperand(i: 4), B, TLI));
4228
4229	return nullptr;
4230	}
4231
4232	Value *FortifiedLibCallSimplifier::optimizeCall(CallInst *CI,
4233	IRBuilderBase &Builder) {
4234	// FIXME: We shouldn't be changing "nobuiltin" or TLI unavailable calls here.
4235	// Some clang users checked for _chk libcall availability using:
4236	// __has_builtin(__builtin___memcpy_chk)
4237	// When compiling with -fno-builtin, this is always true.
4238	// When passing -ffreestanding/-mkernel, which both imply -fno-builtin, we
4239	// end up with fortified libcalls, which isn't acceptable in a freestanding
4240	// environment which only provides their non-fortified counterparts.
4241	//
4242	// Until we change clang and/or teach external users to check for availability
4243	// differently, disregard the "nobuiltin" attribute and TLI::has.
4244	//
4245	// PR23093.
4246
4247	LibFunc Func;
4248	Function *Callee = CI->getCalledFunction();
4249	bool IsCallingConvC = TargetLibraryInfoImpl::isCallingConvCCompatible(CI);
4250
4251	SmallVector<OperandBundleDef, 2> OpBundles;
4252	CI->getOperandBundlesAsDefs(Defs&: OpBundles);
4253
4254	IRBuilderBase::OperandBundlesGuard Guard(Builder);
4255	Builder.setDefaultOperandBundles(OpBundles);
4256
4257	// First, check that this is a known library functions and that the prototype
4258	// is correct.
4259	if (!TLI->getLibFunc(FDecl: *Callee, F&: Func))
4260	return nullptr;
4261
4262	// We never change the calling convention.
4263	if (!ignoreCallingConv(Func) && !IsCallingConvC)
4264	return nullptr;
4265
4266	switch (Func) {
4267	case LibFunc_memcpy_chk:
4268	return optimizeMemCpyChk(CI, B&: Builder);
4269	case LibFunc_mempcpy_chk:
4270	return optimizeMemPCpyChk(CI, B&: Builder);
4271	case LibFunc_memmove_chk:
4272	return optimizeMemMoveChk(CI, B&: Builder);
4273	case LibFunc_memset_chk:
4274	return optimizeMemSetChk(CI, B&: Builder);
4275	case LibFunc_stpcpy_chk:
4276	case LibFunc_strcpy_chk:
4277	return optimizeStrpCpyChk(CI, B&: Builder, Func);
4278	case LibFunc_strlen_chk:
4279	return optimizeStrLenChk(CI, B&: Builder);
4280	case LibFunc_stpncpy_chk:
4281	case LibFunc_strncpy_chk:
4282	return optimizeStrpNCpyChk(CI, B&: Builder, Func);
4283	case LibFunc_memccpy_chk:
4284	return optimizeMemCCpyChk(CI, B&: Builder);
4285	case LibFunc_snprintf_chk:
4286	return optimizeSNPrintfChk(CI, B&: Builder);
4287	case LibFunc_sprintf_chk:
4288	return optimizeSPrintfChk(CI, B&: Builder);
4289	case LibFunc_strcat_chk:
4290	return optimizeStrCatChk(CI, B&: Builder);
4291	case LibFunc_strlcat_chk:
4292	return optimizeStrLCat(CI, B&: Builder);
4293	case LibFunc_strncat_chk:
4294	return optimizeStrNCatChk(CI, B&: Builder);
4295	case LibFunc_strlcpy_chk:
4296	return optimizeStrLCpyChk(CI, B&: Builder);
4297	case LibFunc_vsnprintf_chk:
4298	return optimizeVSNPrintfChk(CI, B&: Builder);
4299	case LibFunc_vsprintf_chk:
4300	return optimizeVSPrintfChk(CI, B&: Builder);
4301	default:
4302	break;
4303	}
4304	return nullptr;
4305	}
4306
4307	FortifiedLibCallSimplifier::FortifiedLibCallSimplifier(
4308	const TargetLibraryInfo *TLI, bool OnlyLowerUnknownSize)
4309	: TLI(TLI), OnlyLowerUnknownSize(OnlyLowerUnknownSize) {}
4310