1	//===- InstructionSimplify.cpp - Fold instruction operands ----------------===//
2	//
3	// Part of the LLVM Project, under the Apache License v2.0 with LLVM Exceptions.
4	// See https://llvm.org/LICENSE.txt for license information.
5	// SPDX-License-Identifier: Apache-2.0 WITH LLVM-exception
6	//
7	//===----------------------------------------------------------------------===//
8	//
9	// This file implements routines for folding instructions into simpler forms
10	// that do not require creating new instructions. This does constant folding
11	// ("add i32 1, 1" -> "2") but can also handle non-constant operands, either
12	// returning a constant ("and i32 %x, 0" -> "0") or an already existing value
13	// ("and i32 %x, %x" -> "%x"). All operands are assumed to have already been
14	// simplified: This is usually true and assuming it simplifies the logic (if
15	// they have not been simplified then results are correct but maybe suboptimal).
16	//
17	//===----------------------------------------------------------------------===//
18
19	#include "llvm/Analysis/InstructionSimplify.h"
20
21	#include "llvm/ADT/STLExtras.h"
22	#include "llvm/ADT/SetVector.h"
23	#include "llvm/ADT/Statistic.h"
24	#include "llvm/Analysis/AliasAnalysis.h"
25	#include "llvm/Analysis/AssumptionCache.h"
26	#include "llvm/Analysis/CaptureTracking.h"
27	#include "llvm/Analysis/CmpInstAnalysis.h"
28	#include "llvm/Analysis/ConstantFolding.h"
29	#include "llvm/Analysis/InstSimplifyFolder.h"
30	#include "llvm/Analysis/LoopAnalysisManager.h"
31	#include "llvm/Analysis/MemoryBuiltins.h"
32	#include "llvm/Analysis/OverflowInstAnalysis.h"
33	#include "llvm/Analysis/ValueTracking.h"
34	#include "llvm/Analysis/VectorUtils.h"
35	#include "llvm/IR/ConstantRange.h"
36	#include "llvm/IR/DataLayout.h"
37	#include "llvm/IR/Dominators.h"
38	#include "llvm/IR/InstrTypes.h"
39	#include "llvm/IR/Instructions.h"
40	#include "llvm/IR/Operator.h"
41	#include "llvm/IR/PatternMatch.h"
42	#include "llvm/IR/Statepoint.h"
43	#include "llvm/Support/KnownBits.h"
44	#include <algorithm>
45	#include <optional>
46	using namespace llvm;
47	using namespace llvm::PatternMatch;
48
49	#define DEBUG_TYPE "instsimplify"
50
51	enum { RecursionLimit = 3 };
52
53	STATISTIC(NumExpand, "Number of expansions");
54	STATISTIC(NumReassoc, "Number of reassociations");
55
56	static Value *simplifyAndInst(Value *, Value *, const SimplifyQuery &,
57	unsigned);
58	static Value *simplifyUnOp(unsigned, Value *, const SimplifyQuery &, unsigned);
59	static Value *simplifyFPUnOp(unsigned, Value *, const FastMathFlags &,
60	const SimplifyQuery &, unsigned);
61	static Value *simplifyBinOp(unsigned, Value *, Value *, const SimplifyQuery &,
62	unsigned);
63	static Value *simplifyBinOp(unsigned, Value *, Value *, const FastMathFlags &,
64	const SimplifyQuery &, unsigned);
65	static Value *simplifyCmpInst(unsigned, Value *, Value *, const SimplifyQuery &,
66	unsigned);
67	static Value *simplifyICmpInst(unsigned Predicate, Value *LHS, Value *RHS,
68	const SimplifyQuery &Q, unsigned MaxRecurse);
69	static Value *simplifyOrInst(Value *, Value *, const SimplifyQuery &, unsigned);
70	static Value *simplifyXorInst(Value *, Value *, const SimplifyQuery &,
71	unsigned);
72	static Value *simplifyCastInst(unsigned, Value *, Type *, const SimplifyQuery &,
73	unsigned);
74	static Value *simplifyGEPInst(Type *, Value *, ArrayRef<Value *>, bool,
75	const SimplifyQuery &, unsigned);
76	static Value *simplifySelectInst(Value *, Value *, Value *,
77	const SimplifyQuery &, unsigned);
78	static Value *simplifyInstructionWithOperands(Instruction *I,
79	ArrayRef<Value *> NewOps,
80	const SimplifyQuery &SQ,
81	unsigned MaxRecurse);
82
83	static Value *foldSelectWithBinaryOp(Value *Cond, Value *TrueVal,
84	Value *FalseVal) {
85	BinaryOperator::BinaryOps BinOpCode;
86	if (auto *BO = dyn_cast<BinaryOperator>(Val: Cond))
87	BinOpCode = BO->getOpcode();
88	else
89	return nullptr;
90
91	CmpInst::Predicate ExpectedPred, Pred1, Pred2;
92	if (BinOpCode == BinaryOperator::Or) {
93	ExpectedPred = ICmpInst::ICMP_NE;
94	} else if (BinOpCode == BinaryOperator::And) {
95	ExpectedPred = ICmpInst::ICMP_EQ;
96	} else
97	return nullptr;
98
99	// %A = icmp eq %TV, %FV
100	// %B = icmp eq %X, %Y (and one of these is a select operand)
101	// %C = and %A, %B
102	// %D = select %C, %TV, %FV
103	// -->
104	// %FV
105
106	// %A = icmp ne %TV, %FV
107	// %B = icmp ne %X, %Y (and one of these is a select operand)
108	// %C = or %A, %B
109	// %D = select %C, %TV, %FV
110	// -->
111	// %TV
112	Value *X, *Y;
113	if (!match(V: Cond, P: m_c_BinOp(L: m_c_ICmp(Pred&: Pred1, L: m_Specific(V: TrueVal),
114	R: m_Specific(V: FalseVal)),
115	R: m_ICmp(Pred&: Pred2, L: m_Value(V&: X), R: m_Value(V&: Y)))) ||
116	Pred1 != Pred2 || Pred1 != ExpectedPred)
117	return nullptr;
118
119	if (X == TrueVal || X == FalseVal || Y == TrueVal || Y == FalseVal)
120	return BinOpCode == BinaryOperator::Or ? TrueVal : FalseVal;
121
122	return nullptr;
123	}
124
125	/// For a boolean type or a vector of boolean type, return false or a vector
126	/// with every element false.
127	static Constant *getFalse(Type *Ty) { return ConstantInt::getFalse(Ty); }
128
129	/// For a boolean type or a vector of boolean type, return true or a vector
130	/// with every element true.
131	static Constant *getTrue(Type *Ty) { return ConstantInt::getTrue(Ty); }
132
133	/// isSameCompare - Is V equivalent to the comparison "LHS Pred RHS"?
134	static bool isSameCompare(Value *V, CmpInst::Predicate Pred, Value *LHS,
135	Value *RHS) {
136	CmpInst *Cmp = dyn_cast<CmpInst>(Val: V);
137	if (!Cmp)
138	return false;
139	CmpInst::Predicate CPred = Cmp->getPredicate();
140	Value *CLHS = Cmp->getOperand(i_nocapture: 0), *CRHS = Cmp->getOperand(i_nocapture: 1);
141	if (CPred == Pred && CLHS == LHS && CRHS == RHS)
142	return true;
143	return CPred == CmpInst::getSwappedPredicate(pred: Pred) && CLHS == RHS &&
144	CRHS == LHS;
145	}
146
147	/// Simplify comparison with true or false branch of select:
148	/// %sel = select i1 %cond, i32 %tv, i32 %fv
149	/// %cmp = icmp sle i32 %sel, %rhs
150	/// Compose new comparison by substituting %sel with either %tv or %fv
151	/// and see if it simplifies.
152	static Value *simplifyCmpSelCase(CmpInst::Predicate Pred, Value *LHS,
153	Value *RHS, Value *Cond,
154	const SimplifyQuery &Q, unsigned MaxRecurse,
155	Constant *TrueOrFalse) {
156	Value *SimplifiedCmp = simplifyCmpInst(Pred, LHS, RHS, Q, MaxRecurse);
157	if (SimplifiedCmp == Cond) {
158	// %cmp simplified to the select condition (%cond).
159	return TrueOrFalse;
160	} else if (!SimplifiedCmp && isSameCompare(V: Cond, Pred, LHS, RHS)) {
161	// It didn't simplify. However, if composed comparison is equivalent
162	// to the select condition (%cond) then we can replace it.
163	return TrueOrFalse;
164	}
165	return SimplifiedCmp;
166	}
167
168	/// Simplify comparison with true branch of select
169	static Value *simplifyCmpSelTrueCase(CmpInst::Predicate Pred, Value *LHS,
170	Value *RHS, Value *Cond,
171	const SimplifyQuery &Q,
172	unsigned MaxRecurse) {
173	return simplifyCmpSelCase(Pred, LHS, RHS, Cond, Q, MaxRecurse,
174	TrueOrFalse: getTrue(Ty: Cond->getType()));
175	}
176
177	/// Simplify comparison with false branch of select
178	static Value *simplifyCmpSelFalseCase(CmpInst::Predicate Pred, Value *LHS,
179	Value *RHS, Value *Cond,
180	const SimplifyQuery &Q,
181	unsigned MaxRecurse) {
182	return simplifyCmpSelCase(Pred, LHS, RHS, Cond, Q, MaxRecurse,
183	TrueOrFalse: getFalse(Ty: Cond->getType()));
184	}
185
186	/// We know comparison with both branches of select can be simplified, but they
187	/// are not equal. This routine handles some logical simplifications.
188	static Value *handleOtherCmpSelSimplifications(Value *TCmp, Value *FCmp,
189	Value *Cond,
190	const SimplifyQuery &Q,
191	unsigned MaxRecurse) {
192	// If the false value simplified to false, then the result of the compare
193	// is equal to "Cond && TCmp". This also catches the case when the false
194	// value simplified to false and the true value to true, returning "Cond".
195	// Folding select to and/or isn't poison-safe in general; impliesPoison
196	// checks whether folding it does not convert a well-defined value into
197	// poison.
198	if (match(V: FCmp, P: m_Zero()) && impliesPoison(ValAssumedPoison: TCmp, V: Cond))
199	if (Value *V = simplifyAndInst(Cond, TCmp, Q, MaxRecurse))
200	return V;
201	// If the true value simplified to true, then the result of the compare
202	// is equal to "Cond || FCmp".
203	if (match(V: TCmp, P: m_One()) && impliesPoison(ValAssumedPoison: FCmp, V: Cond))
204	if (Value *V = simplifyOrInst(Cond, FCmp, Q, MaxRecurse))
205	return V;
206	// Finally, if the false value simplified to true and the true value to
207	// false, then the result of the compare is equal to "!Cond".
208	if (match(V: FCmp, P: m_One()) && match(V: TCmp, P: m_Zero()))
209	if (Value *V = simplifyXorInst(
210	Cond, Constant::getAllOnesValue(Ty: Cond->getType()), Q, MaxRecurse))
211	return V;
212	return nullptr;
213	}
214
215	/// Does the given value dominate the specified phi node?
216	static bool valueDominatesPHI(Value *V, PHINode *P, const DominatorTree *DT) {
217	Instruction *I = dyn_cast<Instruction>(Val: V);
218	if (!I)
219	// Arguments and constants dominate all instructions.
220	return true;
221
222	// If we have a DominatorTree then do a precise test.
223	if (DT)
224	return DT->dominates(Def: I, User: P);
225
226	// Otherwise, if the instruction is in the entry block and is not an invoke,
227	// then it obviously dominates all phi nodes.
228	if (I->getParent()->isEntryBlock() && !isa<InvokeInst>(Val: I) &&
229	!isa<CallBrInst>(Val: I))
230	return true;
231
232	return false;
233	}
234
235	/// Try to simplify a binary operator of form "V op OtherOp" where V is
236	/// "(B0 opex B1)" by distributing 'op' across 'opex' as
237	/// "(B0 op OtherOp) opex (B1 op OtherOp)".
238	static Value *expandBinOp(Instruction::BinaryOps Opcode, Value *V,
239	Value *OtherOp, Instruction::BinaryOps OpcodeToExpand,
240	const SimplifyQuery &Q, unsigned MaxRecurse) {
241	auto *B = dyn_cast<BinaryOperator>(Val: V);
242	if (!B || B->getOpcode() != OpcodeToExpand)
243	return nullptr;
244	Value *B0 = B->getOperand(i_nocapture: 0), *B1 = B->getOperand(i_nocapture: 1);
245	Value *L =
246	simplifyBinOp(Opcode, B0, OtherOp, Q.getWithoutUndef(), MaxRecurse);
247	if (!L)
248	return nullptr;
249	Value *R =
250	simplifyBinOp(Opcode, B1, OtherOp, Q.getWithoutUndef(), MaxRecurse);
251	if (!R)
252	return nullptr;
253
254	// Does the expanded pair of binops simplify to the existing binop?
255	if ((L == B0 && R == B1) ||
256	(Instruction::isCommutative(Opcode: OpcodeToExpand) && L == B1 && R == B0)) {
257	++NumExpand;
258	return B;
259	}
260
261	// Otherwise, return "L op' R" if it simplifies.
262	Value *S = simplifyBinOp(OpcodeToExpand, L, R, Q, MaxRecurse);
263	if (!S)
264	return nullptr;
265
266	++NumExpand;
267	return S;
268	}
269
270	/// Try to simplify binops of form "A op (B op' C)" or the commuted variant by
271	/// distributing op over op'.
272	static Value *expandCommutativeBinOp(Instruction::BinaryOps Opcode, Value *L,
273	Value *R,
274	Instruction::BinaryOps OpcodeToExpand,
275	const SimplifyQuery &Q,
276	unsigned MaxRecurse) {
277	// Recursion is always used, so bail out at once if we already hit the limit.
278	if (!MaxRecurse--)
279	return nullptr;
280
281	if (Value *V = expandBinOp(Opcode, V: L, OtherOp: R, OpcodeToExpand, Q, MaxRecurse))
282	return V;
283	if (Value *V = expandBinOp(Opcode, V: R, OtherOp: L, OpcodeToExpand, Q, MaxRecurse))
284	return V;
285	return nullptr;
286	}
287
288	/// Generic simplifications for associative binary operations.
289	/// Returns the simpler value, or null if none was found.
290	static Value *simplifyAssociativeBinOp(Instruction::BinaryOps Opcode,
291	Value *LHS, Value *RHS,
292	const SimplifyQuery &Q,
293	unsigned MaxRecurse) {
294	assert(Instruction::isAssociative(Opcode) && "Not an associative operation!");
295
296	// Recursion is always used, so bail out at once if we already hit the limit.
297	if (!MaxRecurse--)
298	return nullptr;
299
300	BinaryOperator *Op0 = dyn_cast<BinaryOperator>(Val: LHS);
301	BinaryOperator *Op1 = dyn_cast<BinaryOperator>(Val: RHS);
302
303	// Transform: "(A op B) op C" ==> "A op (B op C)" if it simplifies completely.
304	if (Op0 && Op0->getOpcode() == Opcode) {
305	Value *A = Op0->getOperand(i_nocapture: 0);
306	Value *B = Op0->getOperand(i_nocapture: 1);
307	Value *C = RHS;
308
309	// Does "B op C" simplify?
310	if (Value *V = simplifyBinOp(Opcode, B, C, Q, MaxRecurse)) {
311	// It does! Return "A op V" if it simplifies or is already available.
312	// If V equals B then "A op V" is just the LHS.
313	if (V == B)
314	return LHS;
315	// Otherwise return "A op V" if it simplifies.
316	if (Value *W = simplifyBinOp(Opcode, A, V, Q, MaxRecurse)) {
317	++NumReassoc;
318	return W;
319	}
320	}
321	}
322
323	// Transform: "A op (B op C)" ==> "(A op B) op C" if it simplifies completely.
324	if (Op1 && Op1->getOpcode() == Opcode) {
325	Value *A = LHS;
326	Value *B = Op1->getOperand(i_nocapture: 0);
327	Value *C = Op1->getOperand(i_nocapture: 1);
328
329	// Does "A op B" simplify?
330	if (Value *V = simplifyBinOp(Opcode, A, B, Q, MaxRecurse)) {
331	// It does! Return "V op C" if it simplifies or is already available.
332	// If V equals B then "V op C" is just the RHS.
333	if (V == B)
334	return RHS;
335	// Otherwise return "V op C" if it simplifies.
336	if (Value *W = simplifyBinOp(Opcode, V, C, Q, MaxRecurse)) {
337	++NumReassoc;
338	return W;
339	}
340	}
341	}
342
343	// The remaining transforms require commutativity as well as associativity.
344	if (!Instruction::isCommutative(Opcode))
345	return nullptr;
346
347	// Transform: "(A op B) op C" ==> "(C op A) op B" if it simplifies completely.
348	if (Op0 && Op0->getOpcode() == Opcode) {
349	Value *A = Op0->getOperand(i_nocapture: 0);
350	Value *B = Op0->getOperand(i_nocapture: 1);
351	Value *C = RHS;
352
353	// Does "C op A" simplify?
354	if (Value *V = simplifyBinOp(Opcode, C, A, Q, MaxRecurse)) {
355	// It does! Return "V op B" if it simplifies or is already available.
356	// If V equals A then "V op B" is just the LHS.
357	if (V == A)
358	return LHS;
359	// Otherwise return "V op B" if it simplifies.
360	if (Value *W = simplifyBinOp(Opcode, V, B, Q, MaxRecurse)) {
361	++NumReassoc;
362	return W;
363	}
364	}
365	}
366
367	// Transform: "A op (B op C)" ==> "B op (C op A)" if it simplifies completely.
368	if (Op1 && Op1->getOpcode() == Opcode) {
369	Value *A = LHS;
370	Value *B = Op1->getOperand(i_nocapture: 0);
371	Value *C = Op1->getOperand(i_nocapture: 1);
372
373	// Does "C op A" simplify?
374	if (Value *V = simplifyBinOp(Opcode, C, A, Q, MaxRecurse)) {
375	// It does! Return "B op V" if it simplifies or is already available.
376	// If V equals C then "B op V" is just the RHS.
377	if (V == C)
378	return RHS;
379	// Otherwise return "B op V" if it simplifies.
380	if (Value *W = simplifyBinOp(Opcode, B, V, Q, MaxRecurse)) {
381	++NumReassoc;
382	return W;
383	}
384	}
385	}
386
387	return nullptr;
388	}
389
390	/// In the case of a binary operation with a select instruction as an operand,
391	/// try to simplify the binop by seeing whether evaluating it on both branches
392	/// of the select results in the same value. Returns the common value if so,
393	/// otherwise returns null.
394	static Value *threadBinOpOverSelect(Instruction::BinaryOps Opcode, Value *LHS,
395	Value *RHS, const SimplifyQuery &Q,
396	unsigned MaxRecurse) {
397	// Recursion is always used, so bail out at once if we already hit the limit.
398	if (!MaxRecurse--)
399	return nullptr;
400
401	SelectInst *SI;
402	if (isa<SelectInst>(Val: LHS)) {
403	SI = cast<SelectInst>(Val: LHS);
404	} else {
405	assert(isa<SelectInst>(RHS) && "No select instruction operand!");
406	SI = cast<SelectInst>(Val: RHS);
407	}
408
409	// Evaluate the BinOp on the true and false branches of the select.
410	Value *TV;
411	Value *FV;
412	if (SI == LHS) {
413	TV = simplifyBinOp(Opcode, SI->getTrueValue(), RHS, Q, MaxRecurse);
414	FV = simplifyBinOp(Opcode, SI->getFalseValue(), RHS, Q, MaxRecurse);
415	} else {
416	TV = simplifyBinOp(Opcode, LHS, SI->getTrueValue(), Q, MaxRecurse);
417	FV = simplifyBinOp(Opcode, LHS, SI->getFalseValue(), Q, MaxRecurse);
418	}
419
420	// If they simplified to the same value, then return the common value.
421	// If they both failed to simplify then return null.
422	if (TV == FV)
423	return TV;
424
425	// If one branch simplified to undef, return the other one.
426	if (TV && Q.isUndefValue(V: TV))
427	return FV;
428	if (FV && Q.isUndefValue(V: FV))
429	return TV;
430
431	// If applying the operation did not change the true and false select values,
432	// then the result of the binop is the select itself.
433	if (TV == SI->getTrueValue() && FV == SI->getFalseValue())
434	return SI;
435
436	// If one branch simplified and the other did not, and the simplified
437	// value is equal to the unsimplified one, return the simplified value.
438	// For example, select (cond, X, X & Z) & Z -> X & Z.
439	if ((FV && !TV) || (TV && !FV)) {
440	// Check that the simplified value has the form "X op Y" where "op" is the
441	// same as the original operation.
442	Instruction *Simplified = dyn_cast<Instruction>(Val: FV ? FV : TV);
443	if (Simplified && Simplified->getOpcode() == unsigned(Opcode) &&
444	!Simplified->hasPoisonGeneratingFlags()) {
445	// The value that didn't simplify is "UnsimplifiedLHS op UnsimplifiedRHS".
446	// We already know that "op" is the same as for the simplified value. See
447	// if the operands match too. If so, return the simplified value.
448	Value *UnsimplifiedBranch = FV ? SI->getTrueValue() : SI->getFalseValue();
449	Value *UnsimplifiedLHS = SI == LHS ? UnsimplifiedBranch : LHS;
450	Value *UnsimplifiedRHS = SI == LHS ? RHS : UnsimplifiedBranch;
451	if (Simplified->getOperand(i: 0) == UnsimplifiedLHS &&
452	Simplified->getOperand(i: 1) == UnsimplifiedRHS)
453	return Simplified;
454	if (Simplified->isCommutative() &&
455	Simplified->getOperand(i: 1) == UnsimplifiedLHS &&
456	Simplified->getOperand(i: 0) == UnsimplifiedRHS)
457	return Simplified;
458	}
459	}
460
461	return nullptr;
462	}
463
464	/// In the case of a comparison with a select instruction, try to simplify the
465	/// comparison by seeing whether both branches of the select result in the same
466	/// value. Returns the common value if so, otherwise returns null.
467	/// For example, if we have:
468	/// %tmp = select i1 %cmp, i32 1, i32 2
469	/// %cmp1 = icmp sle i32 %tmp, 3
470	/// We can simplify %cmp1 to true, because both branches of select are
471	/// less than 3. We compose new comparison by substituting %tmp with both
472	/// branches of select and see if it can be simplified.
473	static Value *threadCmpOverSelect(CmpInst::Predicate Pred, Value *LHS,
474	Value *RHS, const SimplifyQuery &Q,
475	unsigned MaxRecurse) {
476	// Recursion is always used, so bail out at once if we already hit the limit.
477	if (!MaxRecurse--)
478	return nullptr;
479
480	// Make sure the select is on the LHS.
481	if (!isa<SelectInst>(Val: LHS)) {
482	std::swap(a&: LHS, b&: RHS);
483	Pred = CmpInst::getSwappedPredicate(pred: Pred);
484	}
485	assert(isa<SelectInst>(LHS) && "Not comparing with a select instruction!");
486	SelectInst *SI = cast<SelectInst>(Val: LHS);
487	Value *Cond = SI->getCondition();
488	Value *TV = SI->getTrueValue();
489	Value *FV = SI->getFalseValue();
490
491	// Now that we have "cmp select(Cond, TV, FV), RHS", analyse it.
492	// Does "cmp TV, RHS" simplify?
493	Value *TCmp = simplifyCmpSelTrueCase(Pred, LHS: TV, RHS, Cond, Q, MaxRecurse);
494	if (!TCmp)
495	return nullptr;
496
497	// Does "cmp FV, RHS" simplify?
498	Value *FCmp = simplifyCmpSelFalseCase(Pred, LHS: FV, RHS, Cond, Q, MaxRecurse);
499	if (!FCmp)
500	return nullptr;
501
502	// If both sides simplified to the same value, then use it as the result of
503	// the original comparison.
504	if (TCmp == FCmp)
505	return TCmp;
506
507	// The remaining cases only make sense if the select condition has the same
508	// type as the result of the comparison, so bail out if this is not so.
509	if (Cond->getType()->isVectorTy() == RHS->getType()->isVectorTy())
510	return handleOtherCmpSelSimplifications(TCmp, FCmp, Cond, Q, MaxRecurse);
511
512	return nullptr;
513	}
514
515	/// In the case of a binary operation with an operand that is a PHI instruction,
516	/// try to simplify the binop by seeing whether evaluating it on the incoming
517	/// phi values yields the same result for every value. If so returns the common
518	/// value, otherwise returns null.
519	static Value *threadBinOpOverPHI(Instruction::BinaryOps Opcode, Value *LHS,
520	Value *RHS, const SimplifyQuery &Q,
521	unsigned MaxRecurse) {
522	// Recursion is always used, so bail out at once if we already hit the limit.
523	if (!MaxRecurse--)
524	return nullptr;
525
526	PHINode *PI;
527	if (isa<PHINode>(Val: LHS)) {
528	PI = cast<PHINode>(Val: LHS);
529	// Bail out if RHS and the phi may be mutually interdependent due to a loop.
530	if (!valueDominatesPHI(V: RHS, P: PI, DT: Q.DT))
531	return nullptr;
532	} else {
533	assert(isa<PHINode>(RHS) && "No PHI instruction operand!");
534	PI = cast<PHINode>(Val: RHS);
535	// Bail out if LHS and the phi may be mutually interdependent due to a loop.
536	if (!valueDominatesPHI(V: LHS, P: PI, DT: Q.DT))
537	return nullptr;
538	}
539
540	// Evaluate the BinOp on the incoming phi values.
541	Value *CommonValue = nullptr;
542	for (Use &Incoming : PI->incoming_values()) {
543	// If the incoming value is the phi node itself, it can safely be skipped.
544	if (Incoming == PI)
545	continue;
546	Instruction *InTI = PI->getIncomingBlock(U: Incoming)->getTerminator();
547	Value *V = PI == LHS
548	? simplifyBinOp(Opcode, Incoming, RHS,
549	Q.getWithInstruction(I: InTI), MaxRecurse)
550	: simplifyBinOp(Opcode, LHS, Incoming,
551	Q.getWithInstruction(I: InTI), MaxRecurse);
552	// If the operation failed to simplify, or simplified to a different value
553	// to previously, then give up.
554	if (!V || (CommonValue && V != CommonValue))
555	return nullptr;
556	CommonValue = V;
557	}
558
559	return CommonValue;
560	}
561
562	/// In the case of a comparison with a PHI instruction, try to simplify the
563	/// comparison by seeing whether comparing with all of the incoming phi values
564	/// yields the same result every time. If so returns the common result,
565	/// otherwise returns null.
566	static Value *threadCmpOverPHI(CmpInst::Predicate Pred, Value *LHS, Value *RHS,
567	const SimplifyQuery &Q, unsigned MaxRecurse) {
568	// Recursion is always used, so bail out at once if we already hit the limit.
569	if (!MaxRecurse--)
570	return nullptr;
571
572	// Make sure the phi is on the LHS.
573	if (!isa<PHINode>(Val: LHS)) {
574	std::swap(a&: LHS, b&: RHS);
575	Pred = CmpInst::getSwappedPredicate(pred: Pred);
576	}
577	assert(isa<PHINode>(LHS) && "Not comparing with a phi instruction!");
578	PHINode *PI = cast<PHINode>(Val: LHS);
579
580	// Bail out if RHS and the phi may be mutually interdependent due to a loop.
581	if (!valueDominatesPHI(V: RHS, P: PI, DT: Q.DT))
582	return nullptr;
583
584	// Evaluate the BinOp on the incoming phi values.
585	Value *CommonValue = nullptr;
586	for (unsigned u = 0, e = PI->getNumIncomingValues(); u < e; ++u) {
587	Value *Incoming = PI->getIncomingValue(i: u);
588	Instruction *InTI = PI->getIncomingBlock(i: u)->getTerminator();
589	// If the incoming value is the phi node itself, it can safely be skipped.
590	if (Incoming == PI)
591	continue;
592	// Change the context instruction to the "edge" that flows into the phi.
593	// This is important because that is where incoming is actually "evaluated"
594	// even though it is used later somewhere else.
595	Value *V = simplifyCmpInst(Pred, Incoming, RHS, Q.getWithInstruction(I: InTI),
596	MaxRecurse);
597	// If the operation failed to simplify, or simplified to a different value
598	// to previously, then give up.
599	if (!V || (CommonValue && V != CommonValue))
600	return nullptr;
601	CommonValue = V;
602	}
603
604	return CommonValue;
605	}
606
607	static Constant *foldOrCommuteConstant(Instruction::BinaryOps Opcode,
608	Value *&Op0, Value *&Op1,
609	const SimplifyQuery &Q) {
610	if (auto *CLHS = dyn_cast<Constant>(Val: Op0)) {
611	if (auto *CRHS = dyn_cast<Constant>(Val: Op1)) {
612	switch (Opcode) {
613	default:
614	break;
615	case Instruction::FAdd:
616	case Instruction::FSub:
617	case Instruction::FMul:
618	case Instruction::FDiv:
619	case Instruction::FRem:
620	if (Q.CxtI != nullptr)
621	return ConstantFoldFPInstOperands(Opcode, LHS: CLHS, RHS: CRHS, DL: Q.DL, I: Q.CxtI);
622	}
623	return ConstantFoldBinaryOpOperands(Opcode, LHS: CLHS, RHS: CRHS, DL: Q.DL);
624	}
625
626	// Canonicalize the constant to the RHS if this is a commutative operation.
627	if (Instruction::isCommutative(Opcode))
628	std::swap(a&: Op0, b&: Op1);
629	}
630	return nullptr;
631	}
632
633	/// Given operands for an Add, see if we can fold the result.
634	/// If not, this returns null.
635	static Value *simplifyAddInst(Value *Op0, Value *Op1, bool IsNSW, bool IsNUW,
636	const SimplifyQuery &Q, unsigned MaxRecurse) {
637	if (Constant *C = foldOrCommuteConstant(Opcode: Instruction::Add, Op0, Op1, Q))
638	return C;
639
640	// X + poison -> poison
641	if (isa<PoisonValue>(Val: Op1))
642	return Op1;
643
644	// X + undef -> undef
645	if (Q.isUndefValue(V: Op1))
646	return Op1;
647
648	// X + 0 -> X
649	if (match(V: Op1, P: m_Zero()))
650	return Op0;
651
652	// If two operands are negative, return 0.
653	if (isKnownNegation(X: Op0, Y: Op1))
654	return Constant::getNullValue(Ty: Op0->getType());
655
656	// X + (Y - X) -> Y
657	// (Y - X) + X -> Y
658	// Eg: X + -X -> 0
659	Value *Y = nullptr;
660	if (match(V: Op1, P: m_Sub(L: m_Value(V&: Y), R: m_Specific(V: Op0))) ||
661	match(V: Op0, P: m_Sub(L: m_Value(V&: Y), R: m_Specific(V: Op1))))
662	return Y;
663
664	// X + ~X -> -1 since ~X = -X-1
665	Type *Ty = Op0->getType();
666	if (match(V: Op0, P: m_Not(V: m_Specific(V: Op1))) || match(V: Op1, P: m_Not(V: m_Specific(V: Op0))))
667	return Constant::getAllOnesValue(Ty);
668
669	// add nsw/nuw (xor Y, signmask), signmask --> Y
670	// The no-wrapping add guarantees that the top bit will be set by the add.
671	// Therefore, the xor must be clearing the already set sign bit of Y.
672	if ((IsNSW || IsNUW) && match(V: Op1, P: m_SignMask()) &&
673	match(V: Op0, P: m_Xor(L: m_Value(V&: Y), R: m_SignMask())))
674	return Y;
675
676	// add nuw %x, -1 -> -1, because %x can only be 0.
677	if (IsNUW && match(V: Op1, P: m_AllOnes()))
678	return Op1; // Which is -1.
679
680	/// i1 add -> xor.
681	if (MaxRecurse && Op0->getType()->isIntOrIntVectorTy(BitWidth: 1))
682	if (Value *V = simplifyXorInst(Op0, Op1, Q, MaxRecurse - 1))
683	return V;
684
685	// Try some generic simplifications for associative operations.
686	if (Value *V =
687	simplifyAssociativeBinOp(Opcode: Instruction::Add, LHS: Op0, RHS: Op1, Q, MaxRecurse))
688	return V;
689
690	// Threading Add over selects and phi nodes is pointless, so don't bother.
691	// Threading over the select in "A + select(cond, B, C)" means evaluating
692	// "A+B" and "A+C" and seeing if they are equal; but they are equal if and
693	// only if B and C are equal. If B and C are equal then (since we assume
694	// that operands have already been simplified) "select(cond, B, C)" should
695	// have been simplified to the common value of B and C already. Analysing
696	// "A+B" and "A+C" thus gains nothing, but costs compile time. Similarly
697	// for threading over phi nodes.
698
699	return nullptr;
700	}
701
702	Value *llvm::simplifyAddInst(Value *Op0, Value *Op1, bool IsNSW, bool IsNUW,
703	const SimplifyQuery &Query) {
704	return ::simplifyAddInst(Op0, Op1, IsNSW, IsNUW, Q: Query, MaxRecurse: RecursionLimit);
705	}
706
707	/// Compute the base pointer and cumulative constant offsets for V.
708	///
709	/// This strips all constant offsets off of V, leaving it the base pointer, and
710	/// accumulates the total constant offset applied in the returned constant.
711	/// It returns zero if there are no constant offsets applied.
712	///
713	/// This is very similar to stripAndAccumulateConstantOffsets(), except it
714	/// normalizes the offset bitwidth to the stripped pointer type, not the
715	/// original pointer type.
716	static APInt stripAndComputeConstantOffsets(const DataLayout &DL, Value *&V,
717	bool AllowNonInbounds = false) {
718	assert(V->getType()->isPtrOrPtrVectorTy());
719
720	APInt Offset = APInt::getZero(numBits: DL.getIndexTypeSizeInBits(Ty: V->getType()));
721	V = V->stripAndAccumulateConstantOffsets(DL, Offset, AllowNonInbounds);
722	// As that strip may trace through `addrspacecast`, need to sext or trunc
723	// the offset calculated.
724	return Offset.sextOrTrunc(width: DL.getIndexTypeSizeInBits(Ty: V->getType()));
725	}
726
727	/// Compute the constant difference between two pointer values.
728	/// If the difference is not a constant, returns zero.
729	static Constant *computePointerDifference(const DataLayout &DL, Value *LHS,
730	Value *RHS) {
731	APInt LHSOffset = stripAndComputeConstantOffsets(DL, V&: LHS);
732	APInt RHSOffset = stripAndComputeConstantOffsets(DL, V&: RHS);
733
734	// If LHS and RHS are not related via constant offsets to the same base
735	// value, there is nothing we can do here.
736	if (LHS != RHS)
737	return nullptr;
738
739	// Otherwise, the difference of LHS - RHS can be computed as:
740	// LHS - RHS
741	// = (LHSOffset + Base) - (RHSOffset + Base)
742	// = LHSOffset - RHSOffset
743	Constant *Res = ConstantInt::get(Context&: LHS->getContext(), V: LHSOffset - RHSOffset);
744	if (auto *VecTy = dyn_cast<VectorType>(Val: LHS->getType()))
745	Res = ConstantVector::getSplat(EC: VecTy->getElementCount(), Elt: Res);
746	return Res;
747	}
748
749	/// Test if there is a dominating equivalence condition for the
750	/// two operands. If there is, try to reduce the binary operation
751	/// between the two operands.
752	/// Example: Op0 - Op1 --> 0 when Op0 == Op1
753	static Value *simplifyByDomEq(unsigned Opcode, Value *Op0, Value *Op1,
754	const SimplifyQuery &Q, unsigned MaxRecurse) {
755	// Recursive run it can not get any benefit
756	if (MaxRecurse != RecursionLimit)
757	return nullptr;
758
759	std::optional<bool> Imp =
760	isImpliedByDomCondition(Pred: CmpInst::ICMP_EQ, LHS: Op0, RHS: Op1, ContextI: Q.CxtI, DL: Q.DL);
761	if (Imp && *Imp) {
762	Type *Ty = Op0->getType();
763	switch (Opcode) {
764	case Instruction::Sub:
765	case Instruction::Xor:
766	case Instruction::URem:
767	case Instruction::SRem:
768	return Constant::getNullValue(Ty);
769
770	case Instruction::SDiv:
771	case Instruction::UDiv:
772	return ConstantInt::get(Ty, V: 1);
773
774	case Instruction::And:
775	case Instruction::Or:
776	// Could be either one - choose Op1 since that's more likely a constant.
777	return Op1;
778	default:
779	break;
780	}
781	}
782	return nullptr;
783	}
784
785	/// Given operands for a Sub, see if we can fold the result.
786	/// If not, this returns null.
787	static Value *simplifySubInst(Value *Op0, Value *Op1, bool IsNSW, bool IsNUW,
788	const SimplifyQuery &Q, unsigned MaxRecurse) {
789	if (Constant *C = foldOrCommuteConstant(Opcode: Instruction::Sub, Op0, Op1, Q))
790	return C;
791
792	// X - poison -> poison
793	// poison - X -> poison
794	if (isa<PoisonValue>(Val: Op0) || isa<PoisonValue>(Val: Op1))
795	return PoisonValue::get(T: Op0->getType());
796
797	// X - undef -> undef
798	// undef - X -> undef
799	if (Q.isUndefValue(V: Op0) || Q.isUndefValue(V: Op1))
800	return UndefValue::get(T: Op0->getType());
801
802	// X - 0 -> X
803	if (match(V: Op1, P: m_Zero()))
804	return Op0;
805
806	// X - X -> 0
807	if (Op0 == Op1)
808	return Constant::getNullValue(Ty: Op0->getType());
809
810	// Is this a negation?
811	if (match(V: Op0, P: m_Zero())) {
812	// 0 - X -> 0 if the sub is NUW.
813	if (IsNUW)
814	return Constant::getNullValue(Ty: Op0->getType());
815
816	KnownBits Known = computeKnownBits(V: Op1, /* Depth */ 0, Q);
817	if (Known.Zero.isMaxSignedValue()) {
818	// Op1 is either 0 or the minimum signed value. If the sub is NSW, then
819	// Op1 must be 0 because negating the minimum signed value is undefined.
820	if (IsNSW)
821	return Constant::getNullValue(Ty: Op0->getType());
822
823	// 0 - X -> X if X is 0 or the minimum signed value.
824	return Op1;
825	}
826	}
827
828	// (X + Y) - Z -> X + (Y - Z) or Y + (X - Z) if everything simplifies.
829	// For example, (X + Y) - Y -> X; (Y + X) - Y -> X
830	Value *X = nullptr, *Y = nullptr, *Z = Op1;
831	if (MaxRecurse && match(V: Op0, P: m_Add(L: m_Value(V&: X), R: m_Value(V&: Y)))) { // (X + Y) - Z
832	// See if "V === Y - Z" simplifies.
833	if (Value *V = simplifyBinOp(Instruction::Sub, Y, Z, Q, MaxRecurse - 1))
834	// It does! Now see if "X + V" simplifies.
835	if (Value *W = simplifyBinOp(Instruction::Add, X, V, Q, MaxRecurse - 1)) {
836	// It does, we successfully reassociated!
837	++NumReassoc;
838	return W;
839	}
840	// See if "V === X - Z" simplifies.
841	if (Value *V = simplifyBinOp(Instruction::Sub, X, Z, Q, MaxRecurse - 1))
842	// It does! Now see if "Y + V" simplifies.
843	if (Value *W = simplifyBinOp(Instruction::Add, Y, V, Q, MaxRecurse - 1)) {
844	// It does, we successfully reassociated!
845	++NumReassoc;
846	return W;
847	}
848	}
849
850	// X - (Y + Z) -> (X - Y) - Z or (X - Z) - Y if everything simplifies.
851	// For example, X - (X + 1) -> -1
852	X = Op0;
853	if (MaxRecurse && match(V: Op1, P: m_Add(L: m_Value(V&: Y), R: m_Value(V&: Z)))) { // X - (Y + Z)
854	// See if "V === X - Y" simplifies.
855	if (Value *V = simplifyBinOp(Instruction::Sub, X, Y, Q, MaxRecurse - 1))
856	// It does! Now see if "V - Z" simplifies.
857	if (Value *W = simplifyBinOp(Instruction::Sub, V, Z, Q, MaxRecurse - 1)) {
858	// It does, we successfully reassociated!
859	++NumReassoc;
860	return W;
861	}
862	// See if "V === X - Z" simplifies.
863	if (Value *V = simplifyBinOp(Instruction::Sub, X, Z, Q, MaxRecurse - 1))
864	// It does! Now see if "V - Y" simplifies.
865	if (Value *W = simplifyBinOp(Instruction::Sub, V, Y, Q, MaxRecurse - 1)) {
866	// It does, we successfully reassociated!
867	++NumReassoc;
868	return W;
869	}
870	}
871
872	// Z - (X - Y) -> (Z - X) + Y if everything simplifies.
873	// For example, X - (X - Y) -> Y.
874	Z = Op0;
875	if (MaxRecurse && match(V: Op1, P: m_Sub(L: m_Value(V&: X), R: m_Value(V&: Y)))) // Z - (X - Y)
876	// See if "V === Z - X" simplifies.
877	if (Value *V = simplifyBinOp(Instruction::Sub, Z, X, Q, MaxRecurse - 1))
878	// It does! Now see if "V + Y" simplifies.
879	if (Value *W = simplifyBinOp(Instruction::Add, V, Y, Q, MaxRecurse - 1)) {
880	// It does, we successfully reassociated!
881	++NumReassoc;
882	return W;
883	}
884
885	// trunc(X) - trunc(Y) -> trunc(X - Y) if everything simplifies.
886	if (MaxRecurse && match(V: Op0, P: m_Trunc(Op: m_Value(V&: X))) &&
887	match(V: Op1, P: m_Trunc(Op: m_Value(V&: Y))))
888	if (X->getType() == Y->getType())
889	// See if "V === X - Y" simplifies.
890	if (Value *V = simplifyBinOp(Instruction::Sub, X, Y, Q, MaxRecurse - 1))
891	// It does! Now see if "trunc V" simplifies.
892	if (Value *W = simplifyCastInst(Instruction::Trunc, V, Op0->getType(),
893	Q, MaxRecurse - 1))
894	// It does, return the simplified "trunc V".
895	return W;
896
897	// Variations on GEP(base, I, ...) - GEP(base, i, ...) -> GEP(null, I-i, ...).
898	if (match(V: Op0, P: m_PtrToInt(Op: m_Value(V&: X))) && match(V: Op1, P: m_PtrToInt(Op: m_Value(V&: Y))))
899	if (Constant *Result = computePointerDifference(DL: Q.DL, LHS: X, RHS: Y))
900	return ConstantFoldIntegerCast(C: Result, DestTy: Op0->getType(), /*IsSigned*/ true,
901	DL: Q.DL);
902
903	// i1 sub -> xor.
904	if (MaxRecurse && Op0->getType()->isIntOrIntVectorTy(BitWidth: 1))
905	if (Value *V = simplifyXorInst(Op0, Op1, Q, MaxRecurse - 1))
906	return V;
907
908	// Threading Sub over selects and phi nodes is pointless, so don't bother.
909	// Threading over the select in "A - select(cond, B, C)" means evaluating
910	// "A-B" and "A-C" and seeing if they are equal; but they are equal if and
911	// only if B and C are equal. If B and C are equal then (since we assume
912	// that operands have already been simplified) "select(cond, B, C)" should
913	// have been simplified to the common value of B and C already. Analysing
914	// "A-B" and "A-C" thus gains nothing, but costs compile time. Similarly
915	// for threading over phi nodes.
916
917	if (Value *V = simplifyByDomEq(Opcode: Instruction::Sub, Op0, Op1, Q, MaxRecurse))
918	return V;
919
920	return nullptr;
921	}
922
923	Value *llvm::simplifySubInst(Value *Op0, Value *Op1, bool IsNSW, bool IsNUW,
924	const SimplifyQuery &Q) {
925	return ::simplifySubInst(Op0, Op1, IsNSW, IsNUW, Q, MaxRecurse: RecursionLimit);
926	}
927
928	/// Given operands for a Mul, see if we can fold the result.
929	/// If not, this returns null.
930	static Value *simplifyMulInst(Value *Op0, Value *Op1, bool IsNSW, bool IsNUW,
931	const SimplifyQuery &Q, unsigned MaxRecurse) {
932	if (Constant *C = foldOrCommuteConstant(Opcode: Instruction::Mul, Op0, Op1, Q))
933	return C;
934
935	// X * poison -> poison
936	if (isa<PoisonValue>(Val: Op1))
937	return Op1;
938
939	// X * undef -> 0
940	// X * 0 -> 0
941	if (Q.isUndefValue(V: Op1) || match(V: Op1, P: m_Zero()))
942	return Constant::getNullValue(Ty: Op0->getType());
943
944	// X * 1 -> X
945	if (match(V: Op1, P: m_One()))
946	return Op0;
947
948	// (X / Y) * Y -> X if the division is exact.
949	Value *X = nullptr;
950	if (Q.IIQ.UseInstrInfo &&
951	(match(V: Op0,
952	P: m_Exact(SubPattern: m_IDiv(L: m_Value(V&: X), R: m_Specific(V: Op1)))) || // (X / Y) * Y
953	match(V: Op1, P: m_Exact(SubPattern: m_IDiv(L: m_Value(V&: X), R: m_Specific(V: Op0)))))) // Y * (X / Y)
954	return X;
955
956	if (Op0->getType()->isIntOrIntVectorTy(BitWidth: 1)) {
957	// mul i1 nsw is a special-case because -1 * -1 is poison (+1 is not
958	// representable). All other cases reduce to 0, so just return 0.
959	if (IsNSW)
960	return ConstantInt::getNullValue(Ty: Op0->getType());
961
962	// Treat "mul i1" as "and i1".
963	if (MaxRecurse)
964	if (Value *V = simplifyAndInst(Op0, Op1, Q, MaxRecurse - 1))
965	return V;
966	}
967
968	// Try some generic simplifications for associative operations.
969	if (Value *V =
970	simplifyAssociativeBinOp(Opcode: Instruction::Mul, LHS: Op0, RHS: Op1, Q, MaxRecurse))
971	return V;
972
973	// Mul distributes over Add. Try some generic simplifications based on this.
974	if (Value *V = expandCommutativeBinOp(Opcode: Instruction::Mul, L: Op0, R: Op1,
975	OpcodeToExpand: Instruction::Add, Q, MaxRecurse))
976	return V;
977
978	// If the operation is with the result of a select instruction, check whether
979	// operating on either branch of the select always yields the same value.
980	if (isa<SelectInst>(Val: Op0) || isa<SelectInst>(Val: Op1))
981	if (Value *V =
982	threadBinOpOverSelect(Opcode: Instruction::Mul, LHS: Op0, RHS: Op1, Q, MaxRecurse))
983	return V;
984
985	// If the operation is with the result of a phi instruction, check whether
986	// operating on all incoming values of the phi always yields the same value.
987	if (isa<PHINode>(Val: Op0) || isa<PHINode>(Val: Op1))
988	if (Value *V =
989	threadBinOpOverPHI(Opcode: Instruction::Mul, LHS: Op0, RHS: Op1, Q, MaxRecurse))
990	return V;
991
992	return nullptr;
993	}
994
995	Value *llvm::simplifyMulInst(Value *Op0, Value *Op1, bool IsNSW, bool IsNUW,
996	const SimplifyQuery &Q) {
997	return ::simplifyMulInst(Op0, Op1, IsNSW, IsNUW, Q, MaxRecurse: RecursionLimit);
998	}
999
1000	/// Given a predicate and two operands, return true if the comparison is true.
1001	/// This is a helper for div/rem simplification where we return some other value
1002	/// when we can prove a relationship between the operands.
1003	static bool isICmpTrue(ICmpInst::Predicate Pred, Value *LHS, Value *RHS,
1004	const SimplifyQuery &Q, unsigned MaxRecurse) {
1005	Value *V = simplifyICmpInst(Predicate: Pred, LHS, RHS, Q, MaxRecurse);
1006	Constant *C = dyn_cast_or_null<Constant>(Val: V);
1007	return (C && C->isAllOnesValue());
1008	}
1009
1010	/// Return true if we can simplify X / Y to 0. Remainder can adapt that answer
1011	/// to simplify X % Y to X.
1012	static bool isDivZero(Value *X, Value *Y, const SimplifyQuery &Q,
1013	unsigned MaxRecurse, bool IsSigned) {
1014	// Recursion is always used, so bail out at once if we already hit the limit.
1015	if (!MaxRecurse--)
1016	return false;
1017
1018	if (IsSigned) {
1019	// (X srem Y) sdiv Y --> 0
1020	if (match(V: X, P: m_SRem(L: m_Value(), R: m_Specific(V: Y))))
1021	return true;
1022
1023	// |X| / |Y| --> 0
1024	//
1025	// We require that 1 operand is a simple constant. That could be extended to
1026	// 2 variables if we computed the sign bit for each.
1027	//
1028	// Make sure that a constant is not the minimum signed value because taking
1029	// the abs() of that is undefined.
1030	Type *Ty = X->getType();
1031	const APInt *C;
1032	if (match(V: X, P: m_APInt(Res&: C)) && !C->isMinSignedValue()) {
1033	// Is the variable divisor magnitude always greater than the constant
1034	// dividend magnitude?
1035	// |Y| > |C| --> Y < -abs(C) or Y > abs(C)
1036	Constant *PosDividendC = ConstantInt::get(Ty, V: C->abs());
1037	Constant *NegDividendC = ConstantInt::get(Ty, V: -C->abs());
1038	if (isICmpTrue(Pred: CmpInst::ICMP_SLT, LHS: Y, RHS: NegDividendC, Q, MaxRecurse) ||
1039	isICmpTrue(Pred: CmpInst::ICMP_SGT, LHS: Y, RHS: PosDividendC, Q, MaxRecurse))
1040	return true;
1041	}
1042	if (match(V: Y, P: m_APInt(Res&: C))) {
1043	// Special-case: we can't take the abs() of a minimum signed value. If
1044	// that's the divisor, then all we have to do is prove that the dividend
1045	// is also not the minimum signed value.
1046	if (C->isMinSignedValue())
1047	return isICmpTrue(Pred: CmpInst::ICMP_NE, LHS: X, RHS: Y, Q, MaxRecurse);
1048
1049	// Is the variable dividend magnitude always less than the constant
1050	// divisor magnitude?
1051	// |X| < |C| --> X > -abs(C) and X < abs(C)
1052	Constant *PosDivisorC = ConstantInt::get(Ty, V: C->abs());
1053	Constant *NegDivisorC = ConstantInt::get(Ty, V: -C->abs());
1054	if (isICmpTrue(Pred: CmpInst::ICMP_SGT, LHS: X, RHS: NegDivisorC, Q, MaxRecurse) &&
1055	isICmpTrue(Pred: CmpInst::ICMP_SLT, LHS: X, RHS: PosDivisorC, Q, MaxRecurse))
1056	return true;
1057	}
1058	return false;
1059	}
1060
1061	// IsSigned == false.
1062
1063	// Is the unsigned dividend known to be less than a constant divisor?
1064	// TODO: Convert this (and above) to range analysis
1065	// ("computeConstantRangeIncludingKnownBits")?
1066	const APInt *C;
1067	if (match(V: Y, P: m_APInt(Res&: C)) &&
1068	computeKnownBits(V: X, /* Depth */ 0, Q).getMaxValue().ult(RHS: *C))
1069	return true;
1070
1071	// Try again for any divisor:
1072	// Is the dividend unsigned less than the divisor?
1073	return isICmpTrue(Pred: ICmpInst::ICMP_ULT, LHS: X, RHS: Y, Q, MaxRecurse);
1074	}
1075
1076	/// Check for common or similar folds of integer division or integer remainder.
1077	/// This applies to all 4 opcodes (sdiv/udiv/srem/urem).
1078	static Value *simplifyDivRem(Instruction::BinaryOps Opcode, Value *Op0,
1079	Value *Op1, const SimplifyQuery &Q,
1080	unsigned MaxRecurse) {
1081	bool IsDiv = (Opcode == Instruction::SDiv || Opcode == Instruction::UDiv);
1082	bool IsSigned = (Opcode == Instruction::SDiv || Opcode == Instruction::SRem);
1083
1084	Type *Ty = Op0->getType();
1085
1086	// X / undef -> poison
1087	// X % undef -> poison
1088	if (Q.isUndefValue(V: Op1) || isa<PoisonValue>(Val: Op1))
1089	return PoisonValue::get(T: Ty);
1090
1091	// X / 0 -> poison
1092	// X % 0 -> poison
1093	// We don't need to preserve faults!
1094	if (match(V: Op1, P: m_Zero()))
1095	return PoisonValue::get(T: Ty);
1096
1097	// If any element of a constant divisor fixed width vector is zero or undef
1098	// the behavior is undefined and we can fold the whole op to poison.
1099	auto *Op1C = dyn_cast<Constant>(Val: Op1);
1100	auto *VTy = dyn_cast<FixedVectorType>(Val: Ty);
1101	if (Op1C && VTy) {
1102	unsigned NumElts = VTy->getNumElements();
1103	for (unsigned i = 0; i != NumElts; ++i) {
1104	Constant *Elt = Op1C->getAggregateElement(Elt: i);
1105	if (Elt && (Elt->isNullValue() || Q.isUndefValue(V: Elt)))
1106	return PoisonValue::get(T: Ty);
1107	}
1108	}
1109
1110	// poison / X -> poison
1111	// poison % X -> poison
1112	if (isa<PoisonValue>(Val: Op0))
1113	return Op0;
1114
1115	// undef / X -> 0
1116	// undef % X -> 0
1117	if (Q.isUndefValue(V: Op0))
1118	return Constant::getNullValue(Ty);
1119
1120	// 0 / X -> 0
1121	// 0 % X -> 0
1122	if (match(V: Op0, P: m_Zero()))
1123	return Constant::getNullValue(Ty: Op0->getType());
1124
1125	// X / X -> 1
1126	// X % X -> 0
1127	if (Op0 == Op1)
1128	return IsDiv ? ConstantInt::get(Ty, V: 1) : Constant::getNullValue(Ty);
1129
1130	KnownBits Known = computeKnownBits(V: Op1, /* Depth */ 0, Q);
1131	// X / 0 -> poison
1132	// X % 0 -> poison
1133	// If the divisor is known to be zero, just return poison. This can happen in
1134	// some cases where its provable indirectly the denominator is zero but it's
1135	// not trivially simplifiable (i.e known zero through a phi node).
1136	if (Known.isZero())
1137	return PoisonValue::get(T: Ty);
1138
1139	// X / 1 -> X
1140	// X % 1 -> 0
1141	// If the divisor can only be zero or one, we can't have division-by-zero
1142	// or remainder-by-zero, so assume the divisor is 1.
1143	// e.g. 1, zext (i8 X), sdiv X (Y and 1)
1144	if (Known.countMinLeadingZeros() == Known.getBitWidth() - 1)
1145	return IsDiv ? Op0 : Constant::getNullValue(Ty);
1146
1147	// If X * Y does not overflow, then:
1148	// X * Y / Y -> X
1149	// X * Y % Y -> 0
1150	Value *X;
1151	if (match(V: Op0, P: m_c_Mul(L: m_Value(V&: X), R: m_Specific(V: Op1)))) {
1152	auto *Mul = cast<OverflowingBinaryOperator>(Val: Op0);
1153	// The multiplication can't overflow if it is defined not to, or if
1154	// X == A / Y for some A.
1155	if ((IsSigned && Q.IIQ.hasNoSignedWrap(Op: Mul)) ||
1156	(!IsSigned && Q.IIQ.hasNoUnsignedWrap(Op: Mul)) ||
1157	(IsSigned && match(V: X, P: m_SDiv(L: m_Value(), R: m_Specific(V: Op1)))) ||
1158	(!IsSigned && match(V: X, P: m_UDiv(L: m_Value(), R: m_Specific(V: Op1))))) {
1159	return IsDiv ? X : Constant::getNullValue(Ty: Op0->getType());
1160	}
1161	}
1162
1163	if (isDivZero(X: Op0, Y: Op1, Q, MaxRecurse, IsSigned))
1164	return IsDiv ? Constant::getNullValue(Ty: Op0->getType()) : Op0;
1165
1166	if (Value *V = simplifyByDomEq(Opcode, Op0, Op1, Q, MaxRecurse))
1167	return V;
1168
1169	// If the operation is with the result of a select instruction, check whether
1170	// operating on either branch of the select always yields the same value.
1171	if (isa<SelectInst>(Val: Op0) || isa<SelectInst>(Val: Op1))
1172	if (Value *V = threadBinOpOverSelect(Opcode, LHS: Op0, RHS: Op1, Q, MaxRecurse))
1173	return V;
1174
1175	// If the operation is with the result of a phi instruction, check whether
1176	// operating on all incoming values of the phi always yields the same value.
1177	if (isa<PHINode>(Val: Op0) || isa<PHINode>(Val: Op1))
1178	if (Value *V = threadBinOpOverPHI(Opcode, LHS: Op0, RHS: Op1, Q, MaxRecurse))
1179	return V;
1180
1181	return nullptr;
1182	}
1183
1184	/// These are simplifications common to SDiv and UDiv.
1185	static Value *simplifyDiv(Instruction::BinaryOps Opcode, Value *Op0, Value *Op1,
1186	bool IsExact, const SimplifyQuery &Q,
1187	unsigned MaxRecurse) {
1188	if (Constant *C = foldOrCommuteConstant(Opcode, Op0, Op1, Q))
1189	return C;
1190
1191	if (Value *V = simplifyDivRem(Opcode, Op0, Op1, Q, MaxRecurse))
1192	return V;
1193
1194	const APInt *DivC;
1195	if (IsExact && match(V: Op1, P: m_APInt(Res&: DivC))) {
1196	// If this is an exact divide by a constant, then the dividend (Op0) must
1197	// have at least as many trailing zeros as the divisor to divide evenly. If
1198	// it has less trailing zeros, then the result must be poison.
1199	if (DivC->countr_zero()) {
1200	KnownBits KnownOp0 = computeKnownBits(V: Op0, /* Depth */ 0, Q);
1201	if (KnownOp0.countMaxTrailingZeros() < DivC->countr_zero())
1202	return PoisonValue::get(T: Op0->getType());
1203	}
1204
1205	// udiv exact (mul nsw X, C), C --> X
1206	// sdiv exact (mul nuw X, C), C --> X
1207	// where C is not a power of 2.
1208	Value *X;
1209	if (!DivC->isPowerOf2() &&
1210	(Opcode == Instruction::UDiv
1211	? match(V: Op0, P: m_NSWMul(L: m_Value(V&: X), R: m_Specific(V: Op1)))
1212	: match(V: Op0, P: m_NUWMul(L: m_Value(V&: X), R: m_Specific(V: Op1)))))
1213	return X;
1214	}
1215
1216	return nullptr;
1217	}
1218
1219	/// These are simplifications common to SRem and URem.
1220	static Value *simplifyRem(Instruction::BinaryOps Opcode, Value *Op0, Value *Op1,
1221	const SimplifyQuery &Q, unsigned MaxRecurse) {
1222	if (Constant *C = foldOrCommuteConstant(Opcode, Op0, Op1, Q))
1223	return C;
1224
1225	if (Value *V = simplifyDivRem(Opcode, Op0, Op1, Q, MaxRecurse))
1226	return V;
1227
1228	// (X << Y) % X -> 0
1229	if (Q.IIQ.UseInstrInfo &&
1230	((Opcode == Instruction::SRem &&
1231	match(V: Op0, P: m_NSWShl(L: m_Specific(V: Op1), R: m_Value()))) ||
1232	(Opcode == Instruction::URem &&
1233	match(V: Op0, P: m_NUWShl(L: m_Specific(V: Op1), R: m_Value())))))
1234	return Constant::getNullValue(Ty: Op0->getType());
1235
1236	return nullptr;
1237	}
1238
1239	/// Given operands for an SDiv, see if we can fold the result.
1240	/// If not, this returns null.
1241	static Value *simplifySDivInst(Value *Op0, Value *Op1, bool IsExact,
1242	const SimplifyQuery &Q, unsigned MaxRecurse) {
1243	// If two operands are negated and no signed overflow, return -1.
1244	if (isKnownNegation(X: Op0, Y: Op1, /*NeedNSW=*/true))
1245	return Constant::getAllOnesValue(Ty: Op0->getType());
1246
1247	return simplifyDiv(Opcode: Instruction::SDiv, Op0, Op1, IsExact, Q, MaxRecurse);
1248	}
1249
1250	Value *llvm::simplifySDivInst(Value *Op0, Value *Op1, bool IsExact,
1251	const SimplifyQuery &Q) {
1252	return ::simplifySDivInst(Op0, Op1, IsExact, Q, MaxRecurse: RecursionLimit);
1253	}
1254
1255	/// Given operands for a UDiv, see if we can fold the result.
1256	/// If not, this returns null.
1257	static Value *simplifyUDivInst(Value *Op0, Value *Op1, bool IsExact,
1258	const SimplifyQuery &Q, unsigned MaxRecurse) {
1259	return simplifyDiv(Opcode: Instruction::UDiv, Op0, Op1, IsExact, Q, MaxRecurse);
1260	}
1261
1262	Value *llvm::simplifyUDivInst(Value *Op0, Value *Op1, bool IsExact,
1263	const SimplifyQuery &Q) {
1264	return ::simplifyUDivInst(Op0, Op1, IsExact, Q, MaxRecurse: RecursionLimit);
1265	}
1266
1267	/// Given operands for an SRem, see if we can fold the result.
1268	/// If not, this returns null.
1269	static Value *simplifySRemInst(Value *Op0, Value *Op1, const SimplifyQuery &Q,
1270	unsigned MaxRecurse) {
1271	// If the divisor is 0, the result is undefined, so assume the divisor is -1.
1272	// srem Op0, (sext i1 X) --> srem Op0, -1 --> 0
1273	Value *X;
1274	if (match(V: Op1, P: m_SExt(Op: m_Value(V&: X))) && X->getType()->isIntOrIntVectorTy(BitWidth: 1))
1275	return ConstantInt::getNullValue(Ty: Op0->getType());
1276
1277	// If the two operands are negated, return 0.
1278	if (isKnownNegation(X: Op0, Y: Op1))
1279	return ConstantInt::getNullValue(Ty: Op0->getType());
1280
1281	return simplifyRem(Opcode: Instruction::SRem, Op0, Op1, Q, MaxRecurse);
1282	}
1283
1284	Value *llvm::simplifySRemInst(Value *Op0, Value *Op1, const SimplifyQuery &Q) {
1285	return ::simplifySRemInst(Op0, Op1, Q, MaxRecurse: RecursionLimit);
1286	}
1287
1288	/// Given operands for a URem, see if we can fold the result.
1289	/// If not, this returns null.
1290	static Value *simplifyURemInst(Value *Op0, Value *Op1, const SimplifyQuery &Q,
1291	unsigned MaxRecurse) {
1292	return simplifyRem(Opcode: Instruction::URem, Op0, Op1, Q, MaxRecurse);
1293	}
1294
1295	Value *llvm::simplifyURemInst(Value *Op0, Value *Op1, const SimplifyQuery &Q) {
1296	return ::simplifyURemInst(Op0, Op1, Q, MaxRecurse: RecursionLimit);
1297	}
1298
1299	/// Returns true if a shift by \c Amount always yields poison.
1300	static bool isPoisonShift(Value *Amount, const SimplifyQuery &Q) {
1301	Constant *C = dyn_cast<Constant>(Val: Amount);
1302	if (!C)
1303	return false;
1304
1305	// X shift by undef -> poison because it may shift by the bitwidth.
1306	if (Q.isUndefValue(V: C))
1307	return true;
1308
1309	// Shifting by the bitwidth or more is poison. This covers scalars and
1310	// fixed/scalable vectors with splat constants.
1311	const APInt *AmountC;
1312	if (match(V: C, P: m_APInt(Res&: AmountC)) && AmountC->uge(RHS: AmountC->getBitWidth()))
1313	return true;
1314
1315	// Try harder for fixed-length vectors:
1316	// If all lanes of a vector shift are poison, the whole shift is poison.
1317	if (isa<ConstantVector>(Val: C) || isa<ConstantDataVector>(Val: C)) {
1318	for (unsigned I = 0,
1319	E = cast<FixedVectorType>(Val: C->getType())->getNumElements();
1320	I != E; ++I)
1321	if (!isPoisonShift(Amount: C->getAggregateElement(Elt: I), Q))
1322	return false;
1323	return true;
1324	}
1325
1326	return false;
1327	}
1328
1329	/// Given operands for an Shl, LShr or AShr, see if we can fold the result.
1330	/// If not, this returns null.
1331	static Value *simplifyShift(Instruction::BinaryOps Opcode, Value *Op0,
1332	Value *Op1, bool IsNSW, const SimplifyQuery &Q,
1333	unsigned MaxRecurse) {
1334	if (Constant *C = foldOrCommuteConstant(Opcode, Op0, Op1, Q))
1335	return C;
1336
1337	// poison shift by X -> poison
1338	if (isa<PoisonValue>(Val: Op0))
1339	return Op0;
1340
1341	// 0 shift by X -> 0
1342	if (match(V: Op0, P: m_Zero()))
1343	return Constant::getNullValue(Ty: Op0->getType());
1344
1345	// X shift by 0 -> X
1346	// Shift-by-sign-extended bool must be shift-by-0 because shift-by-all-ones
1347	// would be poison.
1348	Value *X;
1349	if (match(V: Op1, P: m_Zero()) ||
1350	(match(V: Op1, P: m_SExt(Op: m_Value(V&: X))) && X->getType()->isIntOrIntVectorTy(BitWidth: 1)))
1351	return Op0;
1352
1353	// Fold undefined shifts.
1354	if (isPoisonShift(Amount: Op1, Q))
1355	return PoisonValue::get(T: Op0->getType());
1356
1357	// If the operation is with the result of a select instruction, check whether
1358	// operating on either branch of the select always yields the same value.
1359	if (isa<SelectInst>(Val: Op0) || isa<SelectInst>(Val: Op1))
1360	if (Value *V = threadBinOpOverSelect(Opcode, LHS: Op0, RHS: Op1, Q, MaxRecurse))
1361	return V;
1362
1363	// If the operation is with the result of a phi instruction, check whether
1364	// operating on all incoming values of the phi always yields the same value.
1365	if (isa<PHINode>(Val: Op0) || isa<PHINode>(Val: Op1))
1366	if (Value *V = threadBinOpOverPHI(Opcode, LHS: Op0, RHS: Op1, Q, MaxRecurse))
1367	return V;
1368
1369	// If any bits in the shift amount make that value greater than or equal to
1370	// the number of bits in the type, the shift is undefined.
1371	KnownBits KnownAmt = computeKnownBits(V: Op1, /* Depth */ 0, Q);
1372	if (KnownAmt.getMinValue().uge(RHS: KnownAmt.getBitWidth()))
1373	return PoisonValue::get(T: Op0->getType());
1374
1375	// If all valid bits in the shift amount are known zero, the first operand is
1376	// unchanged.
1377	unsigned NumValidShiftBits = Log2_32_Ceil(Value: KnownAmt.getBitWidth());
1378	if (KnownAmt.countMinTrailingZeros() >= NumValidShiftBits)
1379	return Op0;
1380
1381	// Check for nsw shl leading to a poison value.
1382	if (IsNSW) {
1383	assert(Opcode == Instruction::Shl && "Expected shl for nsw instruction");
1384	KnownBits KnownVal = computeKnownBits(V: Op0, /* Depth */ 0, Q);
1385	KnownBits KnownShl = KnownBits::shl(LHS: KnownVal, RHS: KnownAmt);
1386
1387	if (KnownVal.Zero.isSignBitSet())
1388	KnownShl.Zero.setSignBit();
1389	if (KnownVal.One.isSignBitSet())
1390	KnownShl.One.setSignBit();
1391
1392	if (KnownShl.hasConflict())
1393	return PoisonValue::get(T: Op0->getType());
1394	}
1395
1396	return nullptr;
1397	}
1398
1399	/// Given operands for an LShr or AShr, see if we can fold the result. If not,
1400	/// this returns null.
1401	static Value *simplifyRightShift(Instruction::BinaryOps Opcode, Value *Op0,
1402	Value *Op1, bool IsExact,
1403	const SimplifyQuery &Q, unsigned MaxRecurse) {
1404	if (Value *V =
1405	simplifyShift(Opcode, Op0, Op1, /*IsNSW*/ false, Q, MaxRecurse))
1406	return V;
1407
1408	// X >> X -> 0
1409	if (Op0 == Op1)
1410	return Constant::getNullValue(Ty: Op0->getType());
1411
1412	// undef >> X -> 0
1413	// undef >> X -> undef (if it's exact)
1414	if (Q.isUndefValue(V: Op0))
1415	return IsExact ? Op0 : Constant::getNullValue(Ty: Op0->getType());
1416
1417	// The low bit cannot be shifted out of an exact shift if it is set.
1418	// TODO: Generalize by counting trailing zeros (see fold for exact division).
1419	if (IsExact) {
1420	KnownBits Op0Known = computeKnownBits(V: Op0, /* Depth */ 0, Q);
1421	if (Op0Known.One[0])
1422	return Op0;
1423	}
1424
1425	return nullptr;
1426	}
1427
1428	/// Given operands for an Shl, see if we can fold the result.
1429	/// If not, this returns null.
1430	static Value *simplifyShlInst(Value *Op0, Value *Op1, bool IsNSW, bool IsNUW,
1431	const SimplifyQuery &Q, unsigned MaxRecurse) {
1432	if (Value *V =
1433	simplifyShift(Opcode: Instruction::Shl, Op0, Op1, IsNSW, Q, MaxRecurse))
1434	return V;
1435
1436	Type *Ty = Op0->getType();
1437	// undef << X -> 0
1438	// undef << X -> undef if (if it's NSW/NUW)
1439	if (Q.isUndefValue(V: Op0))
1440	return IsNSW || IsNUW ? Op0 : Constant::getNullValue(Ty);
1441
1442	// (X >> A) << A -> X
1443	Value *X;
1444	if (Q.IIQ.UseInstrInfo &&
1445	match(V: Op0, P: m_Exact(SubPattern: m_Shr(L: m_Value(V&: X), R: m_Specific(V: Op1)))))
1446	return X;
1447
1448	// shl nuw i8 C, %x -> C iff C has sign bit set.
1449	if (IsNUW && match(V: Op0, P: m_Negative()))
1450	return Op0;
1451	// NOTE: could use computeKnownBits() / LazyValueInfo,
1452	// but the cost-benefit analysis suggests it isn't worth it.
1453
1454	// "nuw" guarantees that only zeros are shifted out, and "nsw" guarantees
1455	// that the sign-bit does not change, so the only input that does not
1456	// produce poison is 0, and "0 << (bitwidth-1) --> 0".
1457	if (IsNSW && IsNUW &&
1458	match(V: Op1, P: m_SpecificInt(V: Ty->getScalarSizeInBits() - 1)))
1459	return Constant::getNullValue(Ty);
1460
1461	return nullptr;
1462	}
1463
1464	Value *llvm::simplifyShlInst(Value *Op0, Value *Op1, bool IsNSW, bool IsNUW,
1465	const SimplifyQuery &Q) {
1466	return ::simplifyShlInst(Op0, Op1, IsNSW, IsNUW, Q, MaxRecurse: RecursionLimit);
1467	}
1468
1469	/// Given operands for an LShr, see if we can fold the result.
1470	/// If not, this returns null.
1471	static Value *simplifyLShrInst(Value *Op0, Value *Op1, bool IsExact,
1472	const SimplifyQuery &Q, unsigned MaxRecurse) {
1473	if (Value *V = simplifyRightShift(Opcode: Instruction::LShr, Op0, Op1, IsExact, Q,
1474	MaxRecurse))
1475	return V;
1476
1477	// (X << A) >> A -> X
1478	Value *X;
1479	if (Q.IIQ.UseInstrInfo && match(V: Op0, P: m_NUWShl(L: m_Value(V&: X), R: m_Specific(V: Op1))))
1480	return X;
1481
1482	// ((X << A) | Y) >> A -> X if effective width of Y is not larger than A.
1483	// We can return X as we do in the above case since OR alters no bits in X.
1484	// SimplifyDemandedBits in InstCombine can do more general optimization for
1485	// bit manipulation. This pattern aims to provide opportunities for other
1486	// optimizers by supporting a simple but common case in InstSimplify.
1487	Value *Y;
1488	const APInt *ShRAmt, *ShLAmt;
1489	if (Q.IIQ.UseInstrInfo && match(V: Op1, P: m_APInt(Res&: ShRAmt)) &&
1490	match(V: Op0, P: m_c_Or(L: m_NUWShl(L: m_Value(V&: X), R: m_APInt(Res&: ShLAmt)), R: m_Value(V&: Y))) &&
1491	*ShRAmt == *ShLAmt) {
1492	const KnownBits YKnown = computeKnownBits(V: Y, /* Depth */ 0, Q);
1493	const unsigned EffWidthY = YKnown.countMaxActiveBits();
1494	if (ShRAmt->uge(RHS: EffWidthY))
1495	return X;
1496	}
1497
1498	return nullptr;
1499	}
1500
1501	Value *llvm::simplifyLShrInst(Value *Op0, Value *Op1, bool IsExact,
1502	const SimplifyQuery &Q) {
1503	return ::simplifyLShrInst(Op0, Op1, IsExact, Q, MaxRecurse: RecursionLimit);
1504	}
1505
1506	/// Given operands for an AShr, see if we can fold the result.
1507	/// If not, this returns null.
1508	static Value *simplifyAShrInst(Value *Op0, Value *Op1, bool IsExact,
1509	const SimplifyQuery &Q, unsigned MaxRecurse) {
1510	if (Value *V = simplifyRightShift(Opcode: Instruction::AShr, Op0, Op1, IsExact, Q,
1511	MaxRecurse))
1512	return V;
1513
1514	// -1 >>a X --> -1
1515	// (-1 << X) a>> X --> -1
1516	// We could return the original -1 constant to preserve poison elements.
1517	if (match(V: Op0, P: m_AllOnes()) ||
1518	match(V: Op0, P: m_Shl(L: m_AllOnes(), R: m_Specific(V: Op1))))
1519	return Constant::getAllOnesValue(Ty: Op0->getType());
1520
1521	// (X << A) >> A -> X
1522	Value *X;
1523	if (Q.IIQ.UseInstrInfo && match(V: Op0, P: m_NSWShl(L: m_Value(V&: X), R: m_Specific(V: Op1))))
1524	return X;
1525
1526	// Arithmetic shifting an all-sign-bit value is a no-op.
1527	unsigned NumSignBits = ComputeNumSignBits(Op: Op0, DL: Q.DL, Depth: 0, AC: Q.AC, CxtI: Q.CxtI, DT: Q.DT);
1528	if (NumSignBits == Op0->getType()->getScalarSizeInBits())
1529	return Op0;
1530
1531	return nullptr;
1532	}
1533
1534	Value *llvm::simplifyAShrInst(Value *Op0, Value *Op1, bool IsExact,
1535	const SimplifyQuery &Q) {
1536	return ::simplifyAShrInst(Op0, Op1, IsExact, Q, MaxRecurse: RecursionLimit);
1537	}
1538
1539	/// Commuted variants are assumed to be handled by calling this function again
1540	/// with the parameters swapped.
1541	static Value *simplifyUnsignedRangeCheck(ICmpInst *ZeroICmp,
1542	ICmpInst *UnsignedICmp, bool IsAnd,
1543	const SimplifyQuery &Q) {
1544	Value *X, *Y;
1545
1546	ICmpInst::Predicate EqPred;
1547	if (!match(V: ZeroICmp, P: m_ICmp(Pred&: EqPred, L: m_Value(V&: Y), R: m_Zero())) ||
1548	!ICmpInst::isEquality(P: EqPred))
1549	return nullptr;
1550
1551	ICmpInst::Predicate UnsignedPred;
1552
1553	Value *A, *B;
1554	// Y = (A - B);
1555	if (match(V: Y, P: m_Sub(L: m_Value(V&: A), R: m_Value(V&: B)))) {
1556	if (match(V: UnsignedICmp,
1557	P: m_c_ICmp(Pred&: UnsignedPred, L: m_Specific(V: A), R: m_Specific(V: B))) &&
1558	ICmpInst::isUnsigned(predicate: UnsignedPred)) {
1559	// A >=/<= B || (A - B) != 0 <--> true
1560	if ((UnsignedPred == ICmpInst::ICMP_UGE ||
1561	UnsignedPred == ICmpInst::ICMP_ULE) &&
1562	EqPred == ICmpInst::ICMP_NE && !IsAnd)
1563	return ConstantInt::getTrue(Ty: UnsignedICmp->getType());
1564	// A </> B && (A - B) == 0 <--> false
1565	if ((UnsignedPred == ICmpInst::ICMP_ULT ||
1566	UnsignedPred == ICmpInst::ICMP_UGT) &&
1567	EqPred == ICmpInst::ICMP_EQ && IsAnd)
1568	return ConstantInt::getFalse(Ty: UnsignedICmp->getType());
1569
1570	// A </> B && (A - B) != 0 <--> A </> B
1571	// A </> B || (A - B) != 0 <--> (A - B) != 0
1572	if (EqPred == ICmpInst::ICMP_NE && (UnsignedPred == ICmpInst::ICMP_ULT ||
1573	UnsignedPred == ICmpInst::ICMP_UGT))
1574	return IsAnd ? UnsignedICmp : ZeroICmp;
1575
1576	// A <=/>= B && (A - B) == 0 <--> (A - B) == 0
1577	// A <=/>= B || (A - B) == 0 <--> A <=/>= B
1578	if (EqPred == ICmpInst::ICMP_EQ && (UnsignedPred == ICmpInst::ICMP_ULE ||
1579	UnsignedPred == ICmpInst::ICMP_UGE))
1580	return IsAnd ? ZeroICmp : UnsignedICmp;
1581	}
1582
1583	// Given Y = (A - B)
1584	// Y >= A && Y != 0 --> Y >= A iff B != 0
1585	// Y < A || Y == 0 --> Y < A iff B != 0
1586	if (match(V: UnsignedICmp,
1587	P: m_c_ICmp(Pred&: UnsignedPred, L: m_Specific(V: Y), R: m_Specific(V: A)))) {
1588	if (UnsignedPred == ICmpInst::ICMP_UGE && IsAnd &&
1589	EqPred == ICmpInst::ICMP_NE && isKnownNonZero(V: B, Q))
1590	return UnsignedICmp;
1591	if (UnsignedPred == ICmpInst::ICMP_ULT && !IsAnd &&
1592	EqPred == ICmpInst::ICMP_EQ && isKnownNonZero(V: B, Q))
1593	return UnsignedICmp;
1594	}
1595	}
1596
1597	if (match(V: UnsignedICmp, P: m_ICmp(Pred&: UnsignedPred, L: m_Value(V&: X), R: m_Specific(V: Y))) &&
1598	ICmpInst::isUnsigned(predicate: UnsignedPred))
1599	;
1600	else if (match(V: UnsignedICmp,
1601	P: m_ICmp(Pred&: UnsignedPred, L: m_Specific(V: Y), R: m_Value(V&: X))) &&
1602	ICmpInst::isUnsigned(predicate: UnsignedPred))
1603	UnsignedPred = ICmpInst::getSwappedPredicate(pred: UnsignedPred);
1604	else
1605	return nullptr;
1606
1607	// X > Y && Y == 0 --> Y == 0 iff X != 0
1608	// X > Y || Y == 0 --> X > Y iff X != 0
1609	if (UnsignedPred == ICmpInst::ICMP_UGT && EqPred == ICmpInst::ICMP_EQ &&
1610	isKnownNonZero(V: X, Q))
1611	return IsAnd ? ZeroICmp : UnsignedICmp;
1612
1613	// X <= Y && Y != 0 --> X <= Y iff X != 0
1614	// X <= Y || Y != 0 --> Y != 0 iff X != 0
1615	if (UnsignedPred == ICmpInst::ICMP_ULE && EqPred == ICmpInst::ICMP_NE &&
1616	isKnownNonZero(V: X, Q))
1617	return IsAnd ? UnsignedICmp : ZeroICmp;
1618
1619	// The transforms below here are expected to be handled more generally with
1620	// simplifyAndOrOfICmpsWithLimitConst() or in InstCombine's
1621	// foldAndOrOfICmpsWithConstEq(). If we are looking to trim optimizer overlap,
1622	// these are candidates for removal.
1623
1624	// X < Y && Y != 0 --> X < Y
1625	// X < Y || Y != 0 --> Y != 0
1626	if (UnsignedPred == ICmpInst::ICMP_ULT && EqPred == ICmpInst::ICMP_NE)
1627	return IsAnd ? UnsignedICmp : ZeroICmp;
1628
1629	// X >= Y && Y == 0 --> Y == 0
1630	// X >= Y || Y == 0 --> X >= Y
1631	if (UnsignedPred == ICmpInst::ICMP_UGE && EqPred == ICmpInst::ICMP_EQ)
1632	return IsAnd ? ZeroICmp : UnsignedICmp;
1633
1634	// X < Y && Y == 0 --> false
1635	if (UnsignedPred == ICmpInst::ICMP_ULT && EqPred == ICmpInst::ICMP_EQ &&
1636	IsAnd)
1637	return getFalse(Ty: UnsignedICmp->getType());
1638
1639	// X >= Y || Y != 0 --> true
1640	if (UnsignedPred == ICmpInst::ICMP_UGE && EqPred == ICmpInst::ICMP_NE &&
1641	!IsAnd)
1642	return getTrue(Ty: UnsignedICmp->getType());
1643
1644	return nullptr;
1645	}
1646
1647	/// Test if a pair of compares with a shared operand and 2 constants has an
1648	/// empty set intersection, full set union, or if one compare is a superset of
1649	/// the other.
1650	static Value *simplifyAndOrOfICmpsWithConstants(ICmpInst *Cmp0, ICmpInst *Cmp1,
1651	bool IsAnd) {
1652	// Look for this pattern: {and/or} (icmp X, C0), (icmp X, C1)).
1653	if (Cmp0->getOperand(i_nocapture: 0) != Cmp1->getOperand(i_nocapture: 0))
1654	return nullptr;
1655
1656	const APInt *C0, *C1;
1657	if (!match(V: Cmp0->getOperand(i_nocapture: 1), P: m_APInt(Res&: C0)) ||
1658	!match(V: Cmp1->getOperand(i_nocapture: 1), P: m_APInt(Res&: C1)))
1659	return nullptr;
1660
1661	auto Range0 = ConstantRange::makeExactICmpRegion(Pred: Cmp0->getPredicate(), Other: *C0);
1662	auto Range1 = ConstantRange::makeExactICmpRegion(Pred: Cmp1->getPredicate(), Other: *C1);
1663
1664	// For and-of-compares, check if the intersection is empty:
1665	// (icmp X, C0) && (icmp X, C1) --> empty set --> false
1666	if (IsAnd && Range0.intersectWith(CR: Range1).isEmptySet())
1667	return getFalse(Ty: Cmp0->getType());
1668
1669	// For or-of-compares, check if the union is full:
1670	// (icmp X, C0) || (icmp X, C1) --> full set --> true
1671	if (!IsAnd && Range0.unionWith(CR: Range1).isFullSet())
1672	return getTrue(Ty: Cmp0->getType());
1673
1674	// Is one range a superset of the other?
1675	// If this is and-of-compares, take the smaller set:
1676	// (icmp sgt X, 4) && (icmp sgt X, 42) --> icmp sgt X, 42
1677	// If this is or-of-compares, take the larger set:
1678	// (icmp sgt X, 4) || (icmp sgt X, 42) --> icmp sgt X, 4
1679	if (Range0.contains(CR: Range1))
1680	return IsAnd ? Cmp1 : Cmp0;
1681	if (Range1.contains(CR: Range0))
1682	return IsAnd ? Cmp0 : Cmp1;
1683
1684	return nullptr;
1685	}
1686
1687	static Value *simplifyAndOfICmpsWithAdd(ICmpInst *Op0, ICmpInst *Op1,
1688	const InstrInfoQuery &IIQ) {
1689	// (icmp (add V, C0), C1) & (icmp V, C0)
1690	ICmpInst::Predicate Pred0, Pred1;
1691	const APInt *C0, *C1;
1692	Value *V;
1693	if (!match(V: Op0, P: m_ICmp(Pred&: Pred0, L: m_Add(L: m_Value(V), R: m_APInt(Res&: C0)), R: m_APInt(Res&: C1))))
1694	return nullptr;
1695
1696	if (!match(V: Op1, P: m_ICmp(Pred&: Pred1, L: m_Specific(V), R: m_Value())))
1697	return nullptr;
1698
1699	auto *AddInst = cast<OverflowingBinaryOperator>(Val: Op0->getOperand(i_nocapture: 0));
1700	if (AddInst->getOperand(i_nocapture: 1) != Op1->getOperand(i_nocapture: 1))
1701	return nullptr;
1702
1703	Type *ITy = Op0->getType();
1704	bool IsNSW = IIQ.hasNoSignedWrap(Op: AddInst);
1705	bool IsNUW = IIQ.hasNoUnsignedWrap(Op: AddInst);
1706
1707	const APInt Delta = *C1 - *C0;
1708	if (C0->isStrictlyPositive()) {
1709	if (Delta == 2) {
1710	if (Pred0 == ICmpInst::ICMP_ULT && Pred1 == ICmpInst::ICMP_SGT)
1711	return getFalse(Ty: ITy);
1712	if (Pred0 == ICmpInst::ICMP_SLT && Pred1 == ICmpInst::ICMP_SGT && IsNSW)
1713	return getFalse(Ty: ITy);
1714	}
1715	if (Delta == 1) {
1716	if (Pred0 == ICmpInst::ICMP_ULE && Pred1 == ICmpInst::ICMP_SGT)
1717	return getFalse(Ty: ITy);
1718	if (Pred0 == ICmpInst::ICMP_SLE && Pred1 == ICmpInst::ICMP_SGT && IsNSW)
1719	return getFalse(Ty: ITy);
1720	}
1721	}
1722	if (C0->getBoolValue() && IsNUW) {
1723	if (Delta == 2)
1724	if (Pred0 == ICmpInst::ICMP_ULT && Pred1 == ICmpInst::ICMP_UGT)
1725	return getFalse(Ty: ITy);
1726	if (Delta == 1)
1727	if (Pred0 == ICmpInst::ICMP_ULE && Pred1 == ICmpInst::ICMP_UGT)
1728	return getFalse(Ty: ITy);
1729	}
1730
1731	return nullptr;
1732	}
1733
1734	/// Try to simplify and/or of icmp with ctpop intrinsic.
1735	static Value *simplifyAndOrOfICmpsWithCtpop(ICmpInst *Cmp0, ICmpInst *Cmp1,
1736	bool IsAnd) {
1737	ICmpInst::Predicate Pred0, Pred1;
1738	Value *X;
1739	const APInt *C;
1740	if (!match(Cmp0, m_ICmp(Pred0, m_Intrinsic<Intrinsic::ctpop>(m_Value(X)),
1741	m_APInt(C))) ||
1742	!match(Cmp1, m_ICmp(Pred1, m_Specific(X), m_ZeroInt())) || C->isZero())
1743	return nullptr;
1744
1745	// (ctpop(X) == C) || (X != 0) --> X != 0 where C > 0
1746	if (!IsAnd && Pred0 == ICmpInst::ICMP_EQ && Pred1 == ICmpInst::ICMP_NE)
1747	return Cmp1;
1748	// (ctpop(X) != C) && (X == 0) --> X == 0 where C > 0
1749	if (IsAnd && Pred0 == ICmpInst::ICMP_NE && Pred1 == ICmpInst::ICMP_EQ)
1750	return Cmp1;
1751
1752	return nullptr;
1753	}
1754
1755	static Value *simplifyAndOfICmps(ICmpInst *Op0, ICmpInst *Op1,
1756	const SimplifyQuery &Q) {
1757	if (Value *X = simplifyUnsignedRangeCheck(ZeroICmp: Op0, UnsignedICmp: Op1, /*IsAnd=*/true, Q))
1758	return X;
1759	if (Value *X = simplifyUnsignedRangeCheck(ZeroICmp: Op1, UnsignedICmp: Op0, /*IsAnd=*/true, Q))
1760	return X;
1761
1762	if (Value *X = simplifyAndOrOfICmpsWithConstants(Cmp0: Op0, Cmp1: Op1, IsAnd: true))
1763	return X;
1764
1765	if (Value *X = simplifyAndOrOfICmpsWithCtpop(Cmp0: Op0, Cmp1: Op1, IsAnd: true))
1766	return X;
1767	if (Value *X = simplifyAndOrOfICmpsWithCtpop(Cmp0: Op1, Cmp1: Op0, IsAnd: true))
1768	return X;
1769
1770	if (Value *X = simplifyAndOfICmpsWithAdd(Op0, Op1, IIQ: Q.IIQ))
1771	return X;
1772	if (Value *X = simplifyAndOfICmpsWithAdd(Op0: Op1, Op1: Op0, IIQ: Q.IIQ))
1773	return X;
1774
1775	return nullptr;
1776	}
1777
1778	static Value *simplifyOrOfICmpsWithAdd(ICmpInst *Op0, ICmpInst *Op1,
1779	const InstrInfoQuery &IIQ) {
1780	// (icmp (add V, C0), C1) | (icmp V, C0)
1781	ICmpInst::Predicate Pred0, Pred1;
1782	const APInt *C0, *C1;
1783	Value *V;
1784	if (!match(V: Op0, P: m_ICmp(Pred&: Pred0, L: m_Add(L: m_Value(V), R: m_APInt(Res&: C0)), R: m_APInt(Res&: C1))))
1785	return nullptr;
1786
1787	if (!match(V: Op1, P: m_ICmp(Pred&: Pred1, L: m_Specific(V), R: m_Value())))
1788	return nullptr;
1789
1790	auto *AddInst = cast<BinaryOperator>(Val: Op0->getOperand(i_nocapture: 0));
1791	if (AddInst->getOperand(i_nocapture: 1) != Op1->getOperand(i_nocapture: 1))
1792	return nullptr;
1793
1794	Type *ITy = Op0->getType();
1795	bool IsNSW = IIQ.hasNoSignedWrap(Op: AddInst);
1796	bool IsNUW = IIQ.hasNoUnsignedWrap(Op: AddInst);
1797
1798	const APInt Delta = *C1 - *C0;
1799	if (C0->isStrictlyPositive()) {
1800	if (Delta == 2) {
1801	if (Pred0 == ICmpInst::ICMP_UGE && Pred1 == ICmpInst::ICMP_SLE)
1802	return getTrue(Ty: ITy);
1803	if (Pred0 == ICmpInst::ICMP_SGE && Pred1 == ICmpInst::ICMP_SLE && IsNSW)
1804	return getTrue(Ty: ITy);
1805	}
1806	if (Delta == 1) {
1807	if (Pred0 == ICmpInst::ICMP_UGT && Pred1 == ICmpInst::ICMP_SLE)
1808	return getTrue(Ty: ITy);
1809	if (Pred0 == ICmpInst::ICMP_SGT && Pred1 == ICmpInst::ICMP_SLE && IsNSW)
1810	return getTrue(Ty: ITy);
1811	}
1812	}
1813	if (C0->getBoolValue() && IsNUW) {
1814	if (Delta == 2)
1815	if (Pred0 == ICmpInst::ICMP_UGE && Pred1 == ICmpInst::ICMP_ULE)
1816	return getTrue(Ty: ITy);
1817	if (Delta == 1)
1818	if (Pred0 == ICmpInst::ICMP_UGT && Pred1 == ICmpInst::ICMP_ULE)
1819	return getTrue(Ty: ITy);
1820	}
1821
1822	return nullptr;
1823	}
1824
1825	static Value *simplifyOrOfICmps(ICmpInst *Op0, ICmpInst *Op1,
1826	const SimplifyQuery &Q) {
1827	if (Value *X = simplifyUnsignedRangeCheck(ZeroICmp: Op0, UnsignedICmp: Op1, /*IsAnd=*/false, Q))
1828	return X;
1829	if (Value *X = simplifyUnsignedRangeCheck(ZeroICmp: Op1, UnsignedICmp: Op0, /*IsAnd=*/false, Q))
1830	return X;
1831
1832	if (Value *X = simplifyAndOrOfICmpsWithConstants(Cmp0: Op0, Cmp1: Op1, IsAnd: false))
1833	return X;
1834
1835	if (Value *X = simplifyAndOrOfICmpsWithCtpop(Cmp0: Op0, Cmp1: Op1, IsAnd: false))
1836	return X;
1837	if (Value *X = simplifyAndOrOfICmpsWithCtpop(Cmp0: Op1, Cmp1: Op0, IsAnd: false))
1838	return X;
1839
1840	if (Value *X = simplifyOrOfICmpsWithAdd(Op0, Op1, IIQ: Q.IIQ))
1841	return X;
1842	if (Value *X = simplifyOrOfICmpsWithAdd(Op0: Op1, Op1: Op0, IIQ: Q.IIQ))
1843	return X;
1844
1845	return nullptr;
1846	}
1847
1848	static Value *simplifyAndOrOfFCmps(const SimplifyQuery &Q, FCmpInst *LHS,
1849	FCmpInst *RHS, bool IsAnd) {
1850	Value *LHS0 = LHS->getOperand(i_nocapture: 0), *LHS1 = LHS->getOperand(i_nocapture: 1);
1851	Value *RHS0 = RHS->getOperand(i_nocapture: 0), *RHS1 = RHS->getOperand(i_nocapture: 1);
1852	if (LHS0->getType() != RHS0->getType())
1853	return nullptr;
1854
1855	FCmpInst::Predicate PredL = LHS->getPredicate(), PredR = RHS->getPredicate();
1856	if ((PredL == FCmpInst::FCMP_ORD || PredL == FCmpInst::FCMP_UNO) &&
1857	((FCmpInst::isOrdered(predicate: PredR) && IsAnd) ||
1858	(FCmpInst::isUnordered(predicate: PredR) && !IsAnd))) {
1859	// (fcmp ord X, NNAN) & (fcmp o** X, Y) --> fcmp o** X, Y
1860	// (fcmp uno X, NNAN) & (fcmp o** X, Y) --> false
1861	// (fcmp uno X, NNAN) | (fcmp u** X, Y) --> fcmp u** X, Y
1862	// (fcmp ord X, NNAN) | (fcmp u** X, Y) --> true
1863	if (((LHS1 == RHS0 || LHS1 == RHS1) &&
1864	isKnownNeverNaN(V: LHS0, /*Depth=*/0, SQ: Q)) ||
1865	((LHS0 == RHS0 || LHS0 == RHS1) &&
1866	isKnownNeverNaN(V: LHS1, /*Depth=*/0, SQ: Q)))
1867	return FCmpInst::isOrdered(predicate: PredL) == FCmpInst::isOrdered(predicate: PredR)
1868	? static_cast<Value *>(RHS)
1869	: ConstantInt::getBool(Ty: LHS->getType(), V: !IsAnd);
1870	}
1871
1872	if ((PredR == FCmpInst::FCMP_ORD || PredR == FCmpInst::FCMP_UNO) &&
1873	((FCmpInst::isOrdered(predicate: PredL) && IsAnd) ||
1874	(FCmpInst::isUnordered(predicate: PredL) && !IsAnd))) {
1875	// (fcmp o** X, Y) & (fcmp ord X, NNAN) --> fcmp o** X, Y
1876	// (fcmp o** X, Y) & (fcmp uno X, NNAN) --> false
1877	// (fcmp u** X, Y) | (fcmp uno X, NNAN) --> fcmp u** X, Y
1878	// (fcmp u** X, Y) | (fcmp ord X, NNAN) --> true
1879	if (((RHS1 == LHS0 || RHS1 == LHS1) &&
1880	isKnownNeverNaN(V: RHS0, /*Depth=*/0, SQ: Q)) ||
1881	((RHS0 == LHS0 || RHS0 == LHS1) &&
1882	isKnownNeverNaN(V: RHS1, /*Depth=*/0, SQ: Q)))
1883	return FCmpInst::isOrdered(predicate: PredL) == FCmpInst::isOrdered(predicate: PredR)
1884	? static_cast<Value *>(LHS)
1885	: ConstantInt::getBool(Ty: LHS->getType(), V: !IsAnd);
1886	}
1887
1888	return nullptr;
1889	}
1890
1891	static Value *simplifyAndOrOfCmps(const SimplifyQuery &Q, Value *Op0,
1892	Value *Op1, bool IsAnd) {
1893	// Look through casts of the 'and' operands to find compares.
1894	auto *Cast0 = dyn_cast<CastInst>(Val: Op0);
1895	auto *Cast1 = dyn_cast<CastInst>(Val: Op1);
1896	if (Cast0 && Cast1 && Cast0->getOpcode() == Cast1->getOpcode() &&
1897	Cast0->getSrcTy() == Cast1->getSrcTy()) {
1898	Op0 = Cast0->getOperand(i_nocapture: 0);
1899	Op1 = Cast1->getOperand(i_nocapture: 0);
1900	}
1901
1902	Value *V = nullptr;
1903	auto *ICmp0 = dyn_cast<ICmpInst>(Val: Op0);
1904	auto *ICmp1 = dyn_cast<ICmpInst>(Val: Op1);
1905	if (ICmp0 && ICmp1)
1906	V = IsAnd ? simplifyAndOfICmps(Op0: ICmp0, Op1: ICmp1, Q)
1907	: simplifyOrOfICmps(Op0: ICmp0, Op1: ICmp1, Q);
1908
1909	auto *FCmp0 = dyn_cast<FCmpInst>(Val: Op0);
1910	auto *FCmp1 = dyn_cast<FCmpInst>(Val: Op1);
1911	if (FCmp0 && FCmp1)
1912	V = simplifyAndOrOfFCmps(Q, LHS: FCmp0, RHS: FCmp1, IsAnd);
1913
1914	if (!V)
1915	return nullptr;
1916	if (!Cast0)
1917	return V;
1918
1919	// If we looked through casts, we can only handle a constant simplification
1920	// because we are not allowed to create a cast instruction here.
1921	if (auto *C = dyn_cast<Constant>(Val: V))
1922	return ConstantFoldCastOperand(Opcode: Cast0->getOpcode(), C, DestTy: Cast0->getType(),
1923	DL: Q.DL);
1924
1925	return nullptr;
1926	}
1927
1928	static Value *simplifyWithOpReplaced(Value *V, Value *Op, Value *RepOp,
1929	const SimplifyQuery &Q,
1930	bool AllowRefinement,
1931	SmallVectorImpl<Instruction *> *DropFlags,
1932	unsigned MaxRecurse);
1933
1934	static Value *simplifyAndOrWithICmpEq(unsigned Opcode, Value *Op0, Value *Op1,
1935	const SimplifyQuery &Q,
1936	unsigned MaxRecurse) {
1937	assert((Opcode == Instruction::And || Opcode == Instruction::Or) &&
1938	"Must be and/or");
1939	ICmpInst::Predicate Pred;
1940	Value *A, *B;
1941	if (!match(V: Op0, P: m_ICmp(Pred, L: m_Value(V&: A), R: m_Value(V&: B))) ||
1942	!ICmpInst::isEquality(P: Pred))
1943	return nullptr;
1944
1945	auto Simplify = [&](Value *Res) -> Value * {
1946	Constant *Absorber = ConstantExpr::getBinOpAbsorber(Opcode, Ty: Res->getType());
1947
1948	// and (icmp eq a, b), x implies (a==b) inside x.
1949	// or (icmp ne a, b), x implies (a==b) inside x.
1950	// If x simplifies to true/false, we can simplify the and/or.
1951	if (Pred ==
1952	(Opcode == Instruction::And ? ICmpInst::ICMP_EQ : ICmpInst::ICMP_NE)) {
1953	if (Res == Absorber)
1954	return Absorber;
1955	if (Res == ConstantExpr::getBinOpIdentity(Opcode, Ty: Res->getType()))
1956	return Op0;
1957	return nullptr;
1958	}
1959
1960	// If we have and (icmp ne a, b), x and for a==b we can simplify x to false,
1961	// then we can drop the icmp, as x will already be false in the case where
1962	// the icmp is false. Similar for or and true.
1963	if (Res == Absorber)
1964	return Op1;
1965	return nullptr;
1966	};
1967
1968	if (Value *Res =
1969	simplifyWithOpReplaced(V: Op1, Op: A, RepOp: B, Q, /* AllowRefinement */ true,
1970	/* DropFlags */ nullptr, MaxRecurse))
1971	return Simplify(Res);
1972	if (Value *Res =
1973	simplifyWithOpReplaced(V: Op1, Op: B, RepOp: A, Q, /* AllowRefinement */ true,
1974	/* DropFlags */ nullptr, MaxRecurse))
1975	return Simplify(Res);
1976
1977	return nullptr;
1978	}
1979
1980	/// Given a bitwise logic op, check if the operands are add/sub with a common
1981	/// source value and inverted constant (identity: C - X -> ~(X + ~C)).
1982	static Value *simplifyLogicOfAddSub(Value *Op0, Value *Op1,
1983	Instruction::BinaryOps Opcode) {
1984	assert(Op0->getType() == Op1->getType() && "Mismatched binop types");
1985	assert(BinaryOperator::isBitwiseLogicOp(Opcode) && "Expected logic op");
1986	Value *X;
1987	Constant *C1, *C2;
1988	if ((match(V: Op0, P: m_Add(L: m_Value(V&: X), R: m_Constant(C&: C1))) &&
1989	match(V: Op1, P: m_Sub(L: m_Constant(C&: C2), R: m_Specific(V: X)))) ||
1990	(match(V: Op1, P: m_Add(L: m_Value(V&: X), R: m_Constant(C&: C1))) &&
1991	match(V: Op0, P: m_Sub(L: m_Constant(C&: C2), R: m_Specific(V: X))))) {
1992	if (ConstantExpr::getNot(C: C1) == C2) {
1993	// (X + C) & (~C - X) --> (X + C) & ~(X + C) --> 0
1994	// (X + C) | (~C - X) --> (X + C) | ~(X + C) --> -1
1995	// (X + C) ^ (~C - X) --> (X + C) ^ ~(X + C) --> -1
1996	Type *Ty = Op0->getType();
1997	return Opcode == Instruction::And ? ConstantInt::getNullValue(Ty)
1998	: ConstantInt::getAllOnesValue(Ty);
1999	}
2000	}
2001	return nullptr;
2002	}
2003
2004	// Commutative patterns for and that will be tried with both operand orders.
2005	static Value *simplifyAndCommutative(Value *Op0, Value *Op1,
2006	const SimplifyQuery &Q,
2007	unsigned MaxRecurse) {
2008	// ~A & A = 0
2009	if (match(V: Op0, P: m_Not(V: m_Specific(V: Op1))))
2010	return Constant::getNullValue(Ty: Op0->getType());
2011
2012	// (A | ?) & A = A
2013	if (match(V: Op0, P: m_c_Or(L: m_Specific(V: Op1), R: m_Value())))
2014	return Op1;
2015
2016	// (X | ~Y) & (X | Y) --> X
2017	Value *X, *Y;
2018	if (match(V: Op0, P: m_c_Or(L: m_Value(V&: X), R: m_Not(V: m_Value(V&: Y)))) &&
2019	match(V: Op1, P: m_c_Or(L: m_Deferred(V: X), R: m_Deferred(V: Y))))
2020	return X;
2021
2022	// If we have a multiplication overflow check that is being 'and'ed with a
2023	// check that one of the multipliers is not zero, we can omit the 'and', and
2024	// only keep the overflow check.
2025	if (isCheckForZeroAndMulWithOverflow(Op0, Op1, IsAnd: true))
2026	return Op1;
2027
2028	// -A & A = A if A is a power of two or zero.
2029	if (match(V: Op0, P: m_Neg(V: m_Specific(V: Op1))) &&
2030	isKnownToBeAPowerOfTwo(V: Op1, DL: Q.DL, /*OrZero*/ true, Depth: 0, AC: Q.AC, CxtI: Q.CxtI, DT: Q.DT))
2031	return Op1;
2032
2033	// This is a similar pattern used for checking if a value is a power-of-2:
2034	// (A - 1) & A --> 0 (if A is a power-of-2 or 0)
2035	if (match(V: Op0, P: m_Add(L: m_Specific(V: Op1), R: m_AllOnes())) &&
2036	isKnownToBeAPowerOfTwo(V: Op1, DL: Q.DL, /*OrZero*/ true, Depth: 0, AC: Q.AC, CxtI: Q.CxtI, DT: Q.DT))
2037	return Constant::getNullValue(Ty: Op1->getType());
2038
2039	// (x << N) & ((x << M) - 1) --> 0, where x is known to be a power of 2 and
2040	// M <= N.
2041	const APInt *Shift1, *Shift2;
2042	if (match(V: Op0, P: m_Shl(L: m_Value(V&: X), R: m_APInt(Res&: Shift1))) &&
2043	match(V: Op1, P: m_Add(L: m_Shl(L: m_Specific(V: X), R: m_APInt(Res&: Shift2)), R: m_AllOnes())) &&
2044	isKnownToBeAPowerOfTwo(V: X, DL: Q.DL, /*OrZero*/ true, /*Depth*/ 0, AC: Q.AC,
2045	CxtI: Q.CxtI) &&
2046	Shift1->uge(RHS: *Shift2))
2047	return Constant::getNullValue(Ty: Op0->getType());
2048
2049	if (Value *V =
2050	simplifyAndOrWithICmpEq(Opcode: Instruction::And, Op0, Op1, Q, MaxRecurse))
2051	return V;
2052
2053	return nullptr;
2054	}
2055
2056	/// Given operands for an And, see if we can fold the result.
2057	/// If not, this returns null.
2058	static Value *simplifyAndInst(Value *Op0, Value *Op1, const SimplifyQuery &Q,
2059	unsigned MaxRecurse) {
2060	if (Constant *C = foldOrCommuteConstant(Opcode: Instruction::And, Op0, Op1, Q))
2061	return C;
2062
2063	// X & poison -> poison
2064	if (isa<PoisonValue>(Val: Op1))
2065	return Op1;
2066
2067	// X & undef -> 0
2068	if (Q.isUndefValue(V: Op1))
2069	return Constant::getNullValue(Ty: Op0->getType());
2070
2071	// X & X = X
2072	if (Op0 == Op1)
2073	return Op0;
2074
2075	// X & 0 = 0
2076	if (match(V: Op1, P: m_Zero()))
2077	return Constant::getNullValue(Ty: Op0->getType());
2078
2079	// X & -1 = X
2080	if (match(V: Op1, P: m_AllOnes()))
2081	return Op0;
2082
2083	if (Value *Res = simplifyAndCommutative(Op0, Op1, Q, MaxRecurse))
2084	return Res;
2085	if (Value *Res = simplifyAndCommutative(Op0: Op1, Op1: Op0, Q, MaxRecurse))
2086	return Res;
2087
2088	if (Value *V = simplifyLogicOfAddSub(Op0, Op1, Opcode: Instruction::And))
2089	return V;
2090
2091	// A mask that only clears known zeros of a shifted value is a no-op.
2092	const APInt *Mask;
2093	const APInt *ShAmt;
2094	Value *X, *Y;
2095	if (match(V: Op1, P: m_APInt(Res&: Mask))) {
2096	// If all bits in the inverted and shifted mask are clear:
2097	// and (shl X, ShAmt), Mask --> shl X, ShAmt
2098	if (match(V: Op0, P: m_Shl(L: m_Value(V&: X), R: m_APInt(Res&: ShAmt))) &&
2099	(~(*Mask)).lshr(ShiftAmt: *ShAmt).isZero())
2100	return Op0;
2101
2102	// If all bits in the inverted and shifted mask are clear:
2103	// and (lshr X, ShAmt), Mask --> lshr X, ShAmt
2104	if (match(V: Op0, P: m_LShr(L: m_Value(V&: X), R: m_APInt(Res&: ShAmt))) &&
2105	(~(*Mask)).shl(ShiftAmt: *ShAmt).isZero())
2106	return Op0;
2107	}
2108
2109	// and 2^x-1, 2^C --> 0 where x <= C.
2110	const APInt *PowerC;
2111	Value *Shift;
2112	if (match(V: Op1, P: m_Power2(V&: PowerC)) &&
2113	match(V: Op0, P: m_Add(L: m_Value(V&: Shift), R: m_AllOnes())) &&
2114	isKnownToBeAPowerOfTwo(V: Shift, DL: Q.DL, /*OrZero*/ false, Depth: 0, AC: Q.AC, CxtI: Q.CxtI,
2115	DT: Q.DT)) {
2116	KnownBits Known = computeKnownBits(V: Shift, /* Depth */ 0, Q);
2117	// Use getActiveBits() to make use of the additional power of two knowledge
2118	if (PowerC->getActiveBits() >= Known.getMaxValue().getActiveBits())
2119	return ConstantInt::getNullValue(Ty: Op1->getType());
2120	}
2121
2122	if (Value *V = simplifyAndOrOfCmps(Q, Op0, Op1, IsAnd: true))
2123	return V;
2124
2125	// Try some generic simplifications for associative operations.
2126	if (Value *V =
2127	simplifyAssociativeBinOp(Opcode: Instruction::And, LHS: Op0, RHS: Op1, Q, MaxRecurse))
2128	return V;
2129
2130	// And distributes over Or. Try some generic simplifications based on this.
2131	if (Value *V = expandCommutativeBinOp(Opcode: Instruction::And, L: Op0, R: Op1,
2132	OpcodeToExpand: Instruction::Or, Q, MaxRecurse))
2133	return V;
2134
2135	// And distributes over Xor. Try some generic simplifications based on this.
2136	if (Value *V = expandCommutativeBinOp(Opcode: Instruction::And, L: Op0, R: Op1,
2137	OpcodeToExpand: Instruction::Xor, Q, MaxRecurse))
2138	return V;
2139
2140	if (isa<SelectInst>(Val: Op0) || isa<SelectInst>(Val: Op1)) {
2141	if (Op0->getType()->isIntOrIntVectorTy(BitWidth: 1)) {
2142	// A & (A && B) -> A && B
2143	if (match(V: Op1, P: m_Select(C: m_Specific(V: Op0), L: m_Value(), R: m_Zero())))
2144	return Op1;
2145	else if (match(V: Op0, P: m_Select(C: m_Specific(V: Op1), L: m_Value(), R: m_Zero())))
2146	return Op0;
2147	}
2148	// If the operation is with the result of a select instruction, check
2149	// whether operating on either branch of the select always yields the same
2150	// value.
2151	if (Value *V =
2152	threadBinOpOverSelect(Opcode: Instruction::And, LHS: Op0, RHS: Op1, Q, MaxRecurse))
2153	return V;
2154	}
2155
2156	// If the operation is with the result of a phi instruction, check whether
2157	// operating on all incoming values of the phi always yields the same value.
2158	if (isa<PHINode>(Val: Op0) || isa<PHINode>(Val: Op1))
2159	if (Value *V =
2160	threadBinOpOverPHI(Opcode: Instruction::And, LHS: Op0, RHS: Op1, Q, MaxRecurse))
2161	return V;
2162
2163	// Assuming the effective width of Y is not larger than A, i.e. all bits
2164	// from X and Y are disjoint in (X << A) | Y,
2165	// if the mask of this AND op covers all bits of X or Y, while it covers
2166	// no bits from the other, we can bypass this AND op. E.g.,
2167	// ((X << A) | Y) & Mask -> Y,
2168	// if Mask = ((1 << effective_width_of(Y)) - 1)
2169	// ((X << A) | Y) & Mask -> X << A,
2170	// if Mask = ((1 << effective_width_of(X)) - 1) << A
2171	// SimplifyDemandedBits in InstCombine can optimize the general case.
2172	// This pattern aims to help other passes for a common case.
2173	Value *XShifted;
2174	if (Q.IIQ.UseInstrInfo && match(V: Op1, P: m_APInt(Res&: Mask)) &&
2175	match(V: Op0, P: m_c_Or(L: m_CombineAnd(L: m_NUWShl(L: m_Value(V&: X), R: m_APInt(Res&: ShAmt)),
2176	R: m_Value(V&: XShifted)),
2177	R: m_Value(V&: Y)))) {
2178	const unsigned Width = Op0->getType()->getScalarSizeInBits();
2179	const unsigned ShftCnt = ShAmt->getLimitedValue(Limit: Width);
2180	const KnownBits YKnown = computeKnownBits(V: Y, /* Depth */ 0, Q);
2181	const unsigned EffWidthY = YKnown.countMaxActiveBits();
2182	if (EffWidthY <= ShftCnt) {
2183	const KnownBits XKnown = computeKnownBits(V: X, /* Depth */ 0, Q);
2184	const unsigned EffWidthX = XKnown.countMaxActiveBits();
2185	const APInt EffBitsY = APInt::getLowBitsSet(numBits: Width, loBitsSet: EffWidthY);
2186	const APInt EffBitsX = APInt::getLowBitsSet(numBits: Width, loBitsSet: EffWidthX) << ShftCnt;
2187	// If the mask is extracting all bits from X or Y as is, we can skip
2188	// this AND op.
2189	if (EffBitsY.isSubsetOf(RHS: *Mask) && !EffBitsX.intersects(RHS: *Mask))
2190	return Y;
2191	if (EffBitsX.isSubsetOf(RHS: *Mask) && !EffBitsY.intersects(RHS: *Mask))
2192	return XShifted;
2193	}
2194	}
2195
2196	// ((X | Y) ^ X ) & ((X | Y) ^ Y) --> 0
2197	// ((X | Y) ^ Y ) & ((X | Y) ^ X) --> 0
2198	BinaryOperator *Or;
2199	if (match(V: Op0, P: m_c_Xor(L: m_Value(V&: X),
2200	R: m_CombineAnd(L: m_BinOp(I&: Or),
2201	R: m_c_Or(L: m_Deferred(V: X), R: m_Value(V&: Y))))) &&
2202	match(V: Op1, P: m_c_Xor(L: m_Specific(V: Or), R: m_Specific(V: Y))))
2203	return Constant::getNullValue(Ty: Op0->getType());
2204
2205	const APInt *C1;
2206	Value *A;
2207	// (A ^ C) & (A ^ ~C) -> 0
2208	if (match(V: Op0, P: m_Xor(L: m_Value(V&: A), R: m_APInt(Res&: C1))) &&
2209	match(V: Op1, P: m_Xor(L: m_Specific(V: A), R: m_SpecificInt(V: ~*C1))))
2210	return Constant::getNullValue(Ty: Op0->getType());
2211
2212	if (Op0->getType()->isIntOrIntVectorTy(BitWidth: 1)) {
2213	if (std::optional<bool> Implied = isImpliedCondition(LHS: Op0, RHS: Op1, DL: Q.DL)) {
2214	// If Op0 is true implies Op1 is true, then Op0 is a subset of Op1.
2215	if (*Implied == true)
2216	return Op0;
2217	// If Op0 is true implies Op1 is false, then they are not true together.
2218	if (*Implied == false)
2219	return ConstantInt::getFalse(Ty: Op0->getType());
2220	}
2221	if (std::optional<bool> Implied = isImpliedCondition(LHS: Op1, RHS: Op0, DL: Q.DL)) {
2222	// If Op1 is true implies Op0 is true, then Op1 is a subset of Op0.
2223	if (*Implied)
2224	return Op1;
2225	// If Op1 is true implies Op0 is false, then they are not true together.
2226	if (!*Implied)
2227	return ConstantInt::getFalse(Ty: Op1->getType());
2228	}
2229	}
2230
2231	if (Value *V = simplifyByDomEq(Opcode: Instruction::And, Op0, Op1, Q, MaxRecurse))
2232	return V;
2233
2234	return nullptr;
2235	}
2236
2237	Value *llvm::simplifyAndInst(Value *Op0, Value *Op1, const SimplifyQuery &Q) {
2238	return ::simplifyAndInst(Op0, Op1, Q, MaxRecurse: RecursionLimit);
2239	}
2240
2241	// TODO: Many of these folds could use LogicalAnd/LogicalOr.
2242	static Value *simplifyOrLogic(Value *X, Value *Y) {
2243	assert(X->getType() == Y->getType() && "Expected same type for 'or' ops");
2244	Type *Ty = X->getType();
2245
2246	// X | ~X --> -1
2247	if (match(V: Y, P: m_Not(V: m_Specific(V: X))))
2248	return ConstantInt::getAllOnesValue(Ty);
2249
2250	// X | ~(X & ?) = -1
2251	if (match(V: Y, P: m_Not(V: m_c_And(L: m_Specific(V: X), R: m_Value()))))
2252	return ConstantInt::getAllOnesValue(Ty);
2253
2254	// X | (X & ?) --> X
2255	if (match(V: Y, P: m_c_And(L: m_Specific(V: X), R: m_Value())))
2256	return X;
2257
2258	Value *A, *B;
2259
2260	// (A ^ B) | (A | B) --> A | B
2261	// (A ^ B) | (B | A) --> B | A
2262	if (match(V: X, P: m_Xor(L: m_Value(V&: A), R: m_Value(V&: B))) &&
2263	match(V: Y, P: m_c_Or(L: m_Specific(V: A), R: m_Specific(V: B))))
2264	return Y;
2265
2266	// ~(A ^ B) | (A | B) --> -1
2267	// ~(A ^ B) | (B | A) --> -1
2268	if (match(V: X, P: m_Not(V: m_Xor(L: m_Value(V&: A), R: m_Value(V&: B)))) &&
2269	match(V: Y, P: m_c_Or(L: m_Specific(V: A), R: m_Specific(V: B))))
2270	return ConstantInt::getAllOnesValue(Ty);
2271
2272	// (A & ~B) | (A ^ B) --> A ^ B
2273	// (~B & A) | (A ^ B) --> A ^ B
2274	// (A & ~B) | (B ^ A) --> B ^ A
2275	// (~B & A) | (B ^ A) --> B ^ A
2276	if (match(V: X, P: m_c_And(L: m_Value(V&: A), R: m_Not(V: m_Value(V&: B)))) &&
2277	match(V: Y, P: m_c_Xor(L: m_Specific(V: A), R: m_Specific(V: B))))
2278	return Y;
2279
2280	// (~A ^ B) | (A & B) --> ~A ^ B
2281	// (B ^ ~A) | (A & B) --> B ^ ~A
2282	// (~A ^ B) | (B & A) --> ~A ^ B
2283	// (B ^ ~A) | (B & A) --> B ^ ~A
2284	if (match(V: X, P: m_c_Xor(L: m_Not(V: m_Value(V&: A)), R: m_Value(V&: B))) &&
2285	match(V: Y, P: m_c_And(L: m_Specific(V: A), R: m_Specific(V: B))))
2286	return X;
2287
2288	// (~A | B) | (A ^ B) --> -1
2289	// (~A | B) | (B ^ A) --> -1
2290	// (B | ~A) | (A ^ B) --> -1
2291	// (B | ~A) | (B ^ A) --> -1
2292	if (match(V: X, P: m_c_Or(L: m_Not(V: m_Value(V&: A)), R: m_Value(V&: B))) &&
2293	match(V: Y, P: m_c_Xor(L: m_Specific(V: A), R: m_Specific(V: B))))
2294	return ConstantInt::getAllOnesValue(Ty);
2295
2296	// (~A & B) | ~(A | B) --> ~A
2297	// (~A & B) | ~(B | A) --> ~A
2298	// (B & ~A) | ~(A | B) --> ~A
2299	// (B & ~A) | ~(B | A) --> ~A
2300	Value *NotA;
2301	if (match(V: X, P: m_c_And(L: m_CombineAnd(L: m_Value(V&: NotA), R: m_Not(V: m_Value(V&: A))),
2302	R: m_Value(V&: B))) &&
2303	match(V: Y, P: m_Not(V: m_c_Or(L: m_Specific(V: A), R: m_Specific(V: B)))))
2304	return NotA;
2305	// The same is true of Logical And
2306	// TODO: This could share the logic of the version above if there was a
2307	// version of LogicalAnd that allowed more than just i1 types.
2308	if (match(V: X, P: m_c_LogicalAnd(L: m_CombineAnd(L: m_Value(V&: NotA), R: m_Not(V: m_Value(V&: A))),
2309	R: m_Value(V&: B))) &&
2310	match(V: Y, P: m_Not(V: m_c_LogicalOr(L: m_Specific(V: A), R: m_Specific(V: B)))))
2311	return NotA;
2312
2313	// ~(A ^ B) | (A & B) --> ~(A ^ B)
2314	// ~(A ^ B) | (B & A) --> ~(A ^ B)
2315	Value *NotAB;
2316	if (match(V: X, P: m_CombineAnd(L: m_Not(V: m_Xor(L: m_Value(V&: A), R: m_Value(V&: B))),
2317	R: m_Value(V&: NotAB))) &&
2318	match(V: Y, P: m_c_And(L: m_Specific(V: A), R: m_Specific(V: B))))
2319	return NotAB;
2320
2321	// ~(A & B) | (A ^ B) --> ~(A & B)
2322	// ~(A & B) | (B ^ A) --> ~(A & B)
2323	if (match(V: X, P: m_CombineAnd(L: m_Not(V: m_And(L: m_Value(V&: A), R: m_Value(V&: B))),
2324	R: m_Value(V&: NotAB))) &&
2325	match(V: Y, P: m_c_Xor(L: m_Specific(V: A), R: m_Specific(V: B))))
2326	return NotAB;
2327
2328	return nullptr;
2329	}
2330
2331	/// Given operands for an Or, see if we can fold the result.
2332	/// If not, this returns null.
2333	static Value *simplifyOrInst(Value *Op0, Value *Op1, const SimplifyQuery &Q,
2334	unsigned MaxRecurse) {
2335	if (Constant *C = foldOrCommuteConstant(Opcode: Instruction::Or, Op0, Op1, Q))
2336	return C;
2337
2338	// X | poison -> poison
2339	if (isa<PoisonValue>(Val: Op1))
2340	return Op1;
2341
2342	// X | undef -> -1
2343	// X | -1 = -1
2344	// Do not return Op1 because it may contain undef elements if it's a vector.
2345	if (Q.isUndefValue(V: Op1) || match(V: Op1, P: m_AllOnes()))
2346	return Constant::getAllOnesValue(Ty: Op0->getType());
2347
2348	// X | X = X
2349	// X | 0 = X
2350	if (Op0 == Op1 || match(V: Op1, P: m_Zero()))
2351	return Op0;
2352
2353	if (Value *R = simplifyOrLogic(X: Op0, Y: Op1))
2354	return R;
2355	if (Value *R = simplifyOrLogic(X: Op1, Y: Op0))
2356	return R;
2357
2358	if (Value *V = simplifyLogicOfAddSub(Op0, Op1, Opcode: Instruction::Or))
2359	return V;
2360
2361	// Rotated -1 is still -1:
2362	// (-1 << X) | (-1 >> (C - X)) --> -1
2363	// (-1 >> X) | (-1 << (C - X)) --> -1
2364	// ...with C <= bitwidth (and commuted variants).
2365	Value *X, *Y;
2366	if ((match(V: Op0, P: m_Shl(L: m_AllOnes(), R: m_Value(V&: X))) &&
2367	match(V: Op1, P: m_LShr(L: m_AllOnes(), R: m_Value(V&: Y)))) ||
2368	(match(V: Op1, P: m_Shl(L: m_AllOnes(), R: m_Value(V&: X))) &&
2369	match(V: Op0, P: m_LShr(L: m_AllOnes(), R: m_Value(V&: Y))))) {
2370	const APInt *C;
2371	if ((match(V: X, P: m_Sub(L: m_APInt(Res&: C), R: m_Specific(V: Y))) ||
2372	match(V: Y, P: m_Sub(L: m_APInt(Res&: C), R: m_Specific(V: X)))) &&
2373	C->ule(RHS: X->getType()->getScalarSizeInBits())) {
2374	return ConstantInt::getAllOnesValue(Ty: X->getType());
2375	}
2376	}
2377
2378	// A funnel shift (rotate) can be decomposed into simpler shifts. See if we
2379	// are mixing in another shift that is redundant with the funnel shift.
2380
2381	// (fshl X, ?, Y) | (shl X, Y) --> fshl X, ?, Y
2382	// (shl X, Y) | (fshl X, ?, Y) --> fshl X, ?, Y
2383	if (match(Op0,
2384	m_Intrinsic<Intrinsic::fshl>(m_Value(X), m_Value(), m_Value(Y))) &&
2385	match(Op1, m_Shl(m_Specific(X), m_Specific(Y))))
2386	return Op0;
2387	if (match(Op1,
2388	m_Intrinsic<Intrinsic::fshl>(m_Value(X), m_Value(), m_Value(Y))) &&
2389	match(Op0, m_Shl(m_Specific(X), m_Specific(Y))))
2390	return Op1;
2391
2392	// (fshr ?, X, Y) | (lshr X, Y) --> fshr ?, X, Y
2393	// (lshr X, Y) | (fshr ?, X, Y) --> fshr ?, X, Y
2394	if (match(Op0,
2395	m_Intrinsic<Intrinsic::fshr>(m_Value(), m_Value(X), m_Value(Y))) &&
2396	match(Op1, m_LShr(m_Specific(X), m_Specific(Y))))
2397	return Op0;
2398	if (match(Op1,
2399	m_Intrinsic<Intrinsic::fshr>(m_Value(), m_Value(X), m_Value(Y))) &&
2400	match(Op0, m_LShr(m_Specific(X), m_Specific(Y))))
2401	return Op1;
2402
2403	if (Value *V =
2404	simplifyAndOrWithICmpEq(Opcode: Instruction::Or, Op0, Op1, Q, MaxRecurse))
2405	return V;
2406	if (Value *V =
2407	simplifyAndOrWithICmpEq(Opcode: Instruction::Or, Op0: Op1, Op1: Op0, Q, MaxRecurse))
2408	return V;
2409
2410	if (Value *V = simplifyAndOrOfCmps(Q, Op0, Op1, IsAnd: false))
2411	return V;
2412
2413	// If we have a multiplication overflow check that is being 'and'ed with a
2414	// check that one of the multipliers is not zero, we can omit the 'and', and
2415	// only keep the overflow check.
2416	if (isCheckForZeroAndMulWithOverflow(Op0, Op1, IsAnd: false))
2417	return Op1;
2418	if (isCheckForZeroAndMulWithOverflow(Op0: Op1, Op1: Op0, IsAnd: false))
2419	return Op0;
2420
2421	// Try some generic simplifications for associative operations.
2422	if (Value *V =
2423	simplifyAssociativeBinOp(Opcode: Instruction::Or, LHS: Op0, RHS: Op1, Q, MaxRecurse))
2424	return V;
2425
2426	// Or distributes over And. Try some generic simplifications based on this.
2427	if (Value *V = expandCommutativeBinOp(Opcode: Instruction::Or, L: Op0, R: Op1,
2428	OpcodeToExpand: Instruction::And, Q, MaxRecurse))
2429	return V;
2430
2431	if (isa<SelectInst>(Val: Op0) || isa<SelectInst>(Val: Op1)) {
2432	if (Op0->getType()->isIntOrIntVectorTy(BitWidth: 1)) {
2433	// A | (A || B) -> A || B
2434	if (match(V: Op1, P: m_Select(C: m_Specific(V: Op0), L: m_One(), R: m_Value())))
2435	return Op1;
2436	else if (match(V: Op0, P: m_Select(C: m_Specific(V: Op1), L: m_One(), R: m_Value())))
2437	return Op0;
2438	}
2439	// If the operation is with the result of a select instruction, check
2440	// whether operating on either branch of the select always yields the same
2441	// value.
2442	if (Value *V =
2443	threadBinOpOverSelect(Opcode: Instruction::Or, LHS: Op0, RHS: Op1, Q, MaxRecurse))
2444	return V;
2445	}
2446
2447	// (A & C1)|(B & C2)
2448	Value *A, *B;
2449	const APInt *C1, *C2;
2450	if (match(V: Op0, P: m_And(L: m_Value(V&: A), R: m_APInt(Res&: C1))) &&
2451	match(V: Op1, P: m_And(L: m_Value(V&: B), R: m_APInt(Res&: C2)))) {
2452	if (*C1 == ~*C2) {
2453	// (A & C1)|(B & C2)
2454	// If we have: ((V + N) & C1) | (V & C2)
2455	// .. and C2 = ~C1 and C2 is 0+1+ and (N & C2) == 0
2456	// replace with V+N.
2457	Value *N;
2458	if (C2->isMask() && // C2 == 0+1+
2459	match(V: A, P: m_c_Add(L: m_Specific(V: B), R: m_Value(V&: N)))) {
2460	// Add commutes, try both ways.
2461	if (MaskedValueIsZero(V: N, Mask: *C2, DL: Q))
2462	return A;
2463	}
2464	// Or commutes, try both ways.
2465	if (C1->isMask() && match(V: B, P: m_c_Add(L: m_Specific(V: A), R: m_Value(V&: N)))) {
2466	// Add commutes, try both ways.
2467	if (MaskedValueIsZero(V: N, Mask: *C1, DL: Q))
2468	return B;
2469	}
2470	}
2471	}
2472
2473	// If the operation is with the result of a phi instruction, check whether
2474	// operating on all incoming values of the phi always yields the same value.
2475	if (isa<PHINode>(Val: Op0) || isa<PHINode>(Val: Op1))
2476	if (Value *V = threadBinOpOverPHI(Opcode: Instruction::Or, LHS: Op0, RHS: Op1, Q, MaxRecurse))
2477	return V;
2478
2479	// (A ^ C) | (A ^ ~C) -> -1, i.e. all bits set to one.
2480	if (match(V: Op0, P: m_Xor(L: m_Value(V&: A), R: m_APInt(Res&: C1))) &&
2481	match(V: Op1, P: m_Xor(L: m_Specific(V: A), R: m_SpecificInt(V: ~*C1))))
2482	return Constant::getAllOnesValue(Ty: Op0->getType());
2483
2484	if (Op0->getType()->isIntOrIntVectorTy(BitWidth: 1)) {
2485	if (std::optional<bool> Implied =
2486	isImpliedCondition(LHS: Op0, RHS: Op1, DL: Q.DL, LHSIsTrue: false)) {
2487	// If Op0 is false implies Op1 is false, then Op1 is a subset of Op0.
2488	if (*Implied == false)
2489	return Op0;
2490	// If Op0 is false implies Op1 is true, then at least one is always true.
2491	if (*Implied == true)
2492	return ConstantInt::getTrue(Ty: Op0->getType());
2493	}
2494	if (std::optional<bool> Implied =
2495	isImpliedCondition(LHS: Op1, RHS: Op0, DL: Q.DL, LHSIsTrue: false)) {
2496	// If Op1 is false implies Op0 is false, then Op0 is a subset of Op1.
2497	if (*Implied == false)
2498	return Op1;
2499	// If Op1 is false implies Op0 is true, then at least one is always true.
2500	if (*Implied == true)
2501	return ConstantInt::getTrue(Ty: Op1->getType());
2502	}
2503	}
2504
2505	if (Value *V = simplifyByDomEq(Opcode: Instruction::Or, Op0, Op1, Q, MaxRecurse))
2506	return V;
2507
2508	return nullptr;
2509	}
2510
2511	Value *llvm::simplifyOrInst(Value *Op0, Value *Op1, const SimplifyQuery &Q) {
2512	return ::simplifyOrInst(Op0, Op1, Q, MaxRecurse: RecursionLimit);
2513	}
2514
2515	/// Given operands for a Xor, see if we can fold the result.
2516	/// If not, this returns null.
2517	static Value *simplifyXorInst(Value *Op0, Value *Op1, const SimplifyQuery &Q,
2518	unsigned MaxRecurse) {
2519	if (Constant *C = foldOrCommuteConstant(Opcode: Instruction::Xor, Op0, Op1, Q))
2520	return C;
2521
2522	// X ^ poison -> poison
2523	if (isa<PoisonValue>(Val: Op1))
2524	return Op1;
2525
2526	// A ^ undef -> undef
2527	if (Q.isUndefValue(V: Op1))
2528	return Op1;
2529
2530	// A ^ 0 = A
2531	if (match(V: Op1, P: m_Zero()))
2532	return Op0;
2533
2534	// A ^ A = 0
2535	if (Op0 == Op1)
2536	return Constant::getNullValue(Ty: Op0->getType());
2537
2538	// A ^ ~A = ~A ^ A = -1
2539	if (match(V: Op0, P: m_Not(V: m_Specific(V: Op1))) || match(V: Op1, P: m_Not(V: m_Specific(V: Op0))))
2540	return Constant::getAllOnesValue(Ty: Op0->getType());
2541
2542	auto foldAndOrNot = [](Value *X, Value *Y) -> Value * {
2543	Value *A, *B;
2544	// (~A & B) ^ (A | B) --> A -- There are 8 commuted variants.
2545	if (match(V: X, P: m_c_And(L: m_Not(V: m_Value(V&: A)), R: m_Value(V&: B))) &&
2546	match(V: Y, P: m_c_Or(L: m_Specific(V: A), R: m_Specific(V: B))))
2547	return A;
2548
2549	// (~A | B) ^ (A & B) --> ~A -- There are 8 commuted variants.
2550	// The 'not' op must contain a complete -1 operand (no undef elements for
2551	// vector) for the transform to be safe.
2552	Value *NotA;
2553	if (match(V: X, P: m_c_Or(L: m_CombineAnd(L: m_Not(V: m_Value(V&: A)), R: m_Value(V&: NotA)),
2554	R: m_Value(V&: B))) &&
2555	match(V: Y, P: m_c_And(L: m_Specific(V: A), R: m_Specific(V: B))))
2556	return NotA;
2557
2558	return nullptr;
2559	};
2560	if (Value *R = foldAndOrNot(Op0, Op1))
2561	return R;
2562	if (Value *R = foldAndOrNot(Op1, Op0))
2563	return R;
2564
2565	if (Value *V = simplifyLogicOfAddSub(Op0, Op1, Opcode: Instruction::Xor))
2566	return V;
2567
2568	// Try some generic simplifications for associative operations.
2569	if (Value *V =
2570	simplifyAssociativeBinOp(Opcode: Instruction::Xor, LHS: Op0, RHS: Op1, Q, MaxRecurse))
2571	return V;
2572
2573	// Threading Xor over selects and phi nodes is pointless, so don't bother.
2574	// Threading over the select in "A ^ select(cond, B, C)" means evaluating
2575	// "A^B" and "A^C" and seeing if they are equal; but they are equal if and
2576	// only if B and C are equal. If B and C are equal then (since we assume
2577	// that operands have already been simplified) "select(cond, B, C)" should
2578	// have been simplified to the common value of B and C already. Analysing
2579	// "A^B" and "A^C" thus gains nothing, but costs compile time. Similarly
2580	// for threading over phi nodes.
2581
2582	if (Value *V = simplifyByDomEq(Opcode: Instruction::Xor, Op0, Op1, Q, MaxRecurse))
2583	return V;
2584
2585	return nullptr;
2586	}
2587
2588	Value *llvm::simplifyXorInst(Value *Op0, Value *Op1, const SimplifyQuery &Q) {
2589	return ::simplifyXorInst(Op0, Op1, Q, MaxRecurse: RecursionLimit);
2590	}
2591
2592	static Type *getCompareTy(Value *Op) {
2593	return CmpInst::makeCmpResultType(opnd_type: Op->getType());
2594	}
2595
2596	/// Rummage around inside V looking for something equivalent to the comparison
2597	/// "LHS Pred RHS". Return such a value if found, otherwise return null.
2598	/// Helper function for analyzing max/min idioms.
2599	static Value *extractEquivalentCondition(Value *V, CmpInst::Predicate Pred,
2600	Value *LHS, Value *RHS) {
2601	SelectInst *SI = dyn_cast<SelectInst>(Val: V);
2602	if (!SI)
2603	return nullptr;
2604	CmpInst *Cmp = dyn_cast<CmpInst>(Val: SI->getCondition());
2605	if (!Cmp)
2606	return nullptr;
2607	Value *CmpLHS = Cmp->getOperand(i_nocapture: 0), *CmpRHS = Cmp->getOperand(i_nocapture: 1);
2608	if (Pred == Cmp->getPredicate() && LHS == CmpLHS && RHS == CmpRHS)
2609	return Cmp;
2610	if (Pred == CmpInst::getSwappedPredicate(pred: Cmp->getPredicate()) &&
2611	LHS == CmpRHS && RHS == CmpLHS)
2612	return Cmp;
2613	return nullptr;
2614	}
2615
2616	/// Return true if the underlying object (storage) must be disjoint from
2617	/// storage returned by any noalias return call.
2618	static bool isAllocDisjoint(const Value *V) {
2619	// For allocas, we consider only static ones (dynamic
2620	// allocas might be transformed into calls to malloc not simultaneously
2621	// live with the compared-to allocation). For globals, we exclude symbols
2622	// that might be resolve lazily to symbols in another dynamically-loaded
2623	// library (and, thus, could be malloc'ed by the implementation).
2624	if (const AllocaInst *AI = dyn_cast<AllocaInst>(Val: V))
2625	return AI->isStaticAlloca();
2626	if (const GlobalValue *GV = dyn_cast<GlobalValue>(Val: V))
2627	return (GV->hasLocalLinkage() || GV->hasHiddenVisibility() ||
2628	GV->hasProtectedVisibility() || GV->hasGlobalUnnamedAddr()) &&
2629	!GV->isThreadLocal();
2630	if (const Argument *A = dyn_cast<Argument>(Val: V))
2631	return A->hasByValAttr();
2632	return false;
2633	}
2634
2635	/// Return true if V1 and V2 are each the base of some distict storage region
2636	/// [V, object_size(V)] which do not overlap. Note that zero sized regions
2637	/// *are* possible, and that zero sized regions do not overlap with any other.
2638	static bool haveNonOverlappingStorage(const Value *V1, const Value *V2) {
2639	// Global variables always exist, so they always exist during the lifetime
2640	// of each other and all allocas. Global variables themselves usually have
2641	// non-overlapping storage, but since their addresses are constants, the
2642	// case involving two globals does not reach here and is instead handled in
2643	// constant folding.
2644	//
2645	// Two different allocas usually have different addresses...
2646	//
2647	// However, if there's an @llvm.stackrestore dynamically in between two
2648	// allocas, they may have the same address. It's tempting to reduce the
2649	// scope of the problem by only looking at *static* allocas here. That would
2650	// cover the majority of allocas while significantly reducing the likelihood
2651	// of having an @llvm.stackrestore pop up in the middle. However, it's not
2652	// actually impossible for an @llvm.stackrestore to pop up in the middle of
2653	// an entry block. Also, if we have a block that's not attached to a
2654	// function, we can't tell if it's "static" under the current definition.
2655	// Theoretically, this problem could be fixed by creating a new kind of
2656	// instruction kind specifically for static allocas. Such a new instruction
2657	// could be required to be at the top of the entry block, thus preventing it
2658	// from being subject to a @llvm.stackrestore. Instcombine could even
2659	// convert regular allocas into these special allocas. It'd be nifty.
2660	// However, until then, this problem remains open.
2661	//
2662	// So, we'll assume that two non-empty allocas have different addresses
2663	// for now.
2664	auto isByValArg = [](const Value *V) {
2665	const Argument *A = dyn_cast<Argument>(Val: V);
2666	return A && A->hasByValAttr();
2667	};
2668
2669	// Byval args are backed by store which does not overlap with each other,
2670	// allocas, or globals.
2671	if (isByValArg(V1))
2672	return isa<AllocaInst>(Val: V2) || isa<GlobalVariable>(Val: V2) || isByValArg(V2);
2673	if (isByValArg(V2))
2674	return isa<AllocaInst>(Val: V1) || isa<GlobalVariable>(Val: V1) || isByValArg(V1);
2675
2676	return isa<AllocaInst>(Val: V1) &&
2677	(isa<AllocaInst>(Val: V2) || isa<GlobalVariable>(Val: V2));
2678	}
2679
2680	// A significant optimization not implemented here is assuming that alloca
2681	// addresses are not equal to incoming argument values. They don't *alias*,
2682	// as we say, but that doesn't mean they aren't equal, so we take a
2683	// conservative approach.
2684	//
2685	// This is inspired in part by C++11 5.10p1:
2686	// "Two pointers of the same type compare equal if and only if they are both
2687	// null, both point to the same function, or both represent the same
2688	// address."
2689	//
2690	// This is pretty permissive.
2691	//
2692	// It's also partly due to C11 6.5.9p6:
2693	// "Two pointers compare equal if and only if both are null pointers, both are
2694	// pointers to the same object (including a pointer to an object and a
2695	// subobject at its beginning) or function, both are pointers to one past the
2696	// last element of the same array object, or one is a pointer to one past the
2697	// end of one array object and the other is a pointer to the start of a
2698	// different array object that happens to immediately follow the first array
2699	// object in the address space.)
2700	//
2701	// C11's version is more restrictive, however there's no reason why an argument
2702	// couldn't be a one-past-the-end value for a stack object in the caller and be
2703	// equal to the beginning of a stack object in the callee.
2704	//
2705	// If the C and C++ standards are ever made sufficiently restrictive in this
2706	// area, it may be possible to update LLVM's semantics accordingly and reinstate
2707	// this optimization.
2708	static Constant *computePointerICmp(CmpInst::Predicate Pred, Value *LHS,
2709	Value *RHS, const SimplifyQuery &Q) {
2710	assert(LHS->getType() == RHS->getType() && "Must have same types");
2711	const DataLayout &DL = Q.DL;
2712	const TargetLibraryInfo *TLI = Q.TLI;
2713
2714	// We can only fold certain predicates on pointer comparisons.
2715	switch (Pred) {
2716	default:
2717	return nullptr;
2718
2719	// Equality comparisons are easy to fold.
2720	case CmpInst::ICMP_EQ:
2721	case CmpInst::ICMP_NE:
2722	break;
2723
2724	// We can only handle unsigned relational comparisons because 'inbounds' on
2725	// a GEP only protects against unsigned wrapping.
2726	case CmpInst::ICMP_UGT:
2727	case CmpInst::ICMP_UGE:
2728	case CmpInst::ICMP_ULT:
2729	case CmpInst::ICMP_ULE:
2730	// However, we have to switch them to their signed variants to handle
2731	// negative indices from the base pointer.
2732	Pred = ICmpInst::getSignedPredicate(pred: Pred);
2733	break;
2734	}
2735
2736	// Strip off any constant offsets so that we can reason about them.
2737	// It's tempting to use getUnderlyingObject or even just stripInBoundsOffsets
2738	// here and compare base addresses like AliasAnalysis does, however there are
2739	// numerous hazards. AliasAnalysis and its utilities rely on special rules
2740	// governing loads and stores which don't apply to icmps. Also, AliasAnalysis
2741	// doesn't need to guarantee pointer inequality when it says NoAlias.
2742
2743	// Even if an non-inbounds GEP occurs along the path we can still optimize
2744	// equality comparisons concerning the result.
2745	bool AllowNonInbounds = ICmpInst::isEquality(P: Pred);
2746	unsigned IndexSize = DL.getIndexTypeSizeInBits(Ty: LHS->getType());
2747	APInt LHSOffset(IndexSize, 0), RHSOffset(IndexSize, 0);
2748	LHS = LHS->stripAndAccumulateConstantOffsets(DL, Offset&: LHSOffset, AllowNonInbounds);
2749	RHS = RHS->stripAndAccumulateConstantOffsets(DL, Offset&: RHSOffset, AllowNonInbounds);
2750
2751	// If LHS and RHS are related via constant offsets to the same base
2752	// value, we can replace it with an icmp which just compares the offsets.
2753	if (LHS == RHS)
2754	return ConstantInt::get(Ty: getCompareTy(Op: LHS),
2755	V: ICmpInst::compare(LHS: LHSOffset, RHS: RHSOffset, Pred));
2756
2757	// Various optimizations for (in)equality comparisons.
2758	if (Pred == CmpInst::ICMP_EQ || Pred == CmpInst::ICMP_NE) {
2759	// Different non-empty allocations that exist at the same time have
2760	// different addresses (if the program can tell). If the offsets are
2761	// within the bounds of their allocations (and not one-past-the-end!
2762	// so we can't use inbounds!), and their allocations aren't the same,
2763	// the pointers are not equal.
2764	if (haveNonOverlappingStorage(V1: LHS, V2: RHS)) {
2765	uint64_t LHSSize, RHSSize;
2766	ObjectSizeOpts Opts;
2767	Opts.EvalMode = ObjectSizeOpts::Mode::Min;
2768	auto *F = [](Value *V) -> Function * {
2769	if (auto *I = dyn_cast<Instruction>(Val: V))
2770	return I->getFunction();
2771	if (auto *A = dyn_cast<Argument>(Val: V))
2772	return A->getParent();
2773	return nullptr;
2774	}(LHS);
2775	Opts.NullIsUnknownSize = F ? NullPointerIsDefined(F) : true;
2776	if (getObjectSize(Ptr: LHS, Size&: LHSSize, DL, TLI, Opts) &&
2777	getObjectSize(Ptr: RHS, Size&: RHSSize, DL, TLI, Opts)) {
2778	APInt Dist = LHSOffset - RHSOffset;
2779	if (Dist.isNonNegative() ? Dist.ult(RHS: LHSSize) : (-Dist).ult(RHS: RHSSize))
2780	return ConstantInt::get(Ty: getCompareTy(Op: LHS),
2781	V: !CmpInst::isTrueWhenEqual(predicate: Pred));
2782	}
2783	}
2784
2785	// If one side of the equality comparison must come from a noalias call
2786	// (meaning a system memory allocation function), and the other side must
2787	// come from a pointer that cannot overlap with dynamically-allocated
2788	// memory within the lifetime of the current function (allocas, byval
2789	// arguments, globals), then determine the comparison result here.
2790	SmallVector<const Value *, 8> LHSUObjs, RHSUObjs;
2791	getUnderlyingObjects(V: LHS, Objects&: LHSUObjs);
2792	getUnderlyingObjects(V: RHS, Objects&: RHSUObjs);
2793
2794	// Is the set of underlying objects all noalias calls?
2795	auto IsNAC = [](ArrayRef<const Value *> Objects) {
2796	return all_of(Range&: Objects, P: isNoAliasCall);
2797	};
2798
2799	// Is the set of underlying objects all things which must be disjoint from
2800	// noalias calls. We assume that indexing from such disjoint storage
2801	// into the heap is undefined, and thus offsets can be safely ignored.
2802	auto IsAllocDisjoint = [](ArrayRef<const Value *> Objects) {
2803	return all_of(Range&: Objects, P: ::isAllocDisjoint);
2804	};
2805
2806	if ((IsNAC(LHSUObjs) && IsAllocDisjoint(RHSUObjs)) ||
2807	(IsNAC(RHSUObjs) && IsAllocDisjoint(LHSUObjs)))
2808	return ConstantInt::get(Ty: getCompareTy(Op: LHS),
2809	V: !CmpInst::isTrueWhenEqual(predicate: Pred));
2810
2811	// Fold comparisons for non-escaping pointer even if the allocation call
2812	// cannot be elided. We cannot fold malloc comparison to null. Also, the
2813	// dynamic allocation call could be either of the operands. Note that
2814	// the other operand can not be based on the alloc - if it were, then
2815	// the cmp itself would be a capture.
2816	Value *MI = nullptr;
2817	if (isAllocLikeFn(V: LHS, TLI) && llvm::isKnownNonZero(V: RHS, Q))
2818	MI = LHS;
2819	else if (isAllocLikeFn(V: RHS, TLI) && llvm::isKnownNonZero(V: LHS, Q))
2820	MI = RHS;
2821	if (MI) {
2822	// FIXME: This is incorrect, see PR54002. While we can assume that the
2823	// allocation is at an address that makes the comparison false, this
2824	// requires that *all* comparisons to that address be false, which
2825	// InstSimplify cannot guarantee.
2826	struct CustomCaptureTracker : public CaptureTracker {
2827	bool Captured = false;
2828	void tooManyUses() override { Captured = true; }
2829	bool captured(const Use *U) override {
2830	if (auto *ICmp = dyn_cast<ICmpInst>(Val: U->getUser())) {
2831	// Comparison against value stored in global variable. Given the
2832	// pointer does not escape, its value cannot be guessed and stored
2833	// separately in a global variable.
2834	unsigned OtherIdx = 1 - U->getOperandNo();
2835	auto *LI = dyn_cast<LoadInst>(Val: ICmp->getOperand(i_nocapture: OtherIdx));
2836	if (LI && isa<GlobalVariable>(Val: LI->getPointerOperand()))
2837	return false;
2838	}
2839
2840	Captured = true;
2841	return true;
2842	}
2843	};
2844	CustomCaptureTracker Tracker;
2845	PointerMayBeCaptured(V: MI, Tracker: &Tracker);
2846	if (!Tracker.Captured)
2847	return ConstantInt::get(Ty: getCompareTy(Op: LHS),
2848	V: CmpInst::isFalseWhenEqual(predicate: Pred));
2849	}
2850	}
2851
2852	// Otherwise, fail.
2853	return nullptr;
2854	}
2855
2856	/// Fold an icmp when its operands have i1 scalar type.
2857	static Value *simplifyICmpOfBools(CmpInst::Predicate Pred, Value *LHS,
2858	Value *RHS, const SimplifyQuery &Q) {
2859	Type *ITy = getCompareTy(Op: LHS); // The return type.
2860	Type *OpTy = LHS->getType(); // The operand type.
2861	if (!OpTy->isIntOrIntVectorTy(BitWidth: 1))
2862	return nullptr;
2863
2864	// A boolean compared to true/false can be reduced in 14 out of the 20
2865	// (10 predicates * 2 constants) possible combinations. The other
2866	// 6 cases require a 'not' of the LHS.
2867
2868	auto ExtractNotLHS = [](Value *V) -> Value * {
2869	Value *X;
2870	if (match(V, P: m_Not(V: m_Value(V&: X))))
2871	return X;
2872	return nullptr;
2873	};
2874
2875	if (match(V: RHS, P: m_Zero())) {
2876	switch (Pred) {
2877	case CmpInst::ICMP_NE: // X != 0 -> X
2878	case CmpInst::ICMP_UGT: // X >u 0 -> X
2879	case CmpInst::ICMP_SLT: // X <s 0 -> X
2880	return LHS;
2881
2882	case CmpInst::ICMP_EQ: // not(X) == 0 -> X != 0 -> X
2883	case CmpInst::ICMP_ULE: // not(X) <=u 0 -> X >u 0 -> X
2884	case CmpInst::ICMP_SGE: // not(X) >=s 0 -> X <s 0 -> X
2885	if (Value *X = ExtractNotLHS(LHS))
2886	return X;
2887	break;
2888
2889	case CmpInst::ICMP_ULT: // X <u 0 -> false
2890	case CmpInst::ICMP_SGT: // X >s 0 -> false
2891	return getFalse(Ty: ITy);
2892
2893	case CmpInst::ICMP_UGE: // X >=u 0 -> true
2894	case CmpInst::ICMP_SLE: // X <=s 0 -> true
2895	return getTrue(Ty: ITy);
2896
2897	default:
2898	break;
2899	}
2900	} else if (match(V: RHS, P: m_One())) {
2901	switch (Pred) {
2902	case CmpInst::ICMP_EQ: // X == 1 -> X
2903	case CmpInst::ICMP_UGE: // X >=u 1 -> X
2904	case CmpInst::ICMP_SLE: // X <=s -1 -> X
2905	return LHS;
2906
2907	case CmpInst::ICMP_NE: // not(X) != 1 -> X == 1 -> X
2908	case CmpInst::ICMP_ULT: // not(X) <=u 1 -> X >=u 1 -> X
2909	case CmpInst::ICMP_SGT: // not(X) >s 1 -> X <=s -1 -> X
2910	if (Value *X = ExtractNotLHS(LHS))
2911	return X;
2912	break;
2913
2914	case CmpInst::ICMP_UGT: // X >u 1 -> false
2915	case CmpInst::ICMP_SLT: // X <s -1 -> false
2916	return getFalse(Ty: ITy);
2917
2918	case CmpInst::ICMP_ULE: // X <=u 1 -> true
2919	case CmpInst::ICMP_SGE: // X >=s -1 -> true
2920	return getTrue(Ty: ITy);
2921
2922	default:
2923	break;
2924	}
2925	}
2926
2927	switch (Pred) {
2928	default:
2929	break;
2930	case ICmpInst::ICMP_UGE:
2931	if (isImpliedCondition(LHS: RHS, RHS: LHS, DL: Q.DL).value_or(u: false))
2932	return getTrue(Ty: ITy);
2933	break;
2934	case ICmpInst::ICMP_SGE:
2935	/// For signed comparison, the values for an i1 are 0 and -1
2936	/// respectively. This maps into a truth table of:
2937	/// LHS | RHS | LHS >=s RHS | LHS implies RHS
2938	/// 0 | 0 | 1 (0 >= 0) | 1
2939	/// 0 | 1 | 1 (0 >= -1) | 1
2940	/// 1 | 0 | 0 (-1 >= 0) | 0
2941	/// 1 | 1 | 1 (-1 >= -1) | 1
2942	if (isImpliedCondition(LHS, RHS, DL: Q.DL).value_or(u: false))
2943	return getTrue(Ty: ITy);
2944	break;
2945	case ICmpInst::ICMP_ULE:
2946	if (isImpliedCondition(LHS, RHS, DL: Q.DL).value_or(u: false))
2947	return getTrue(Ty: ITy);
2948	break;
2949	case ICmpInst::ICMP_SLE:
2950	/// SLE follows the same logic as SGE with the LHS and RHS swapped.
2951	if (isImpliedCondition(LHS: RHS, RHS: LHS, DL: Q.DL).value_or(u: false))
2952	return getTrue(Ty: ITy);
2953	break;
2954	}
2955
2956	return nullptr;
2957	}
2958
2959	/// Try hard to fold icmp with zero RHS because this is a common case.
2960	static Value *simplifyICmpWithZero(CmpInst::Predicate Pred, Value *LHS,
2961	Value *RHS, const SimplifyQuery &Q) {
2962	if (!match(V: RHS, P: m_Zero()))
2963	return nullptr;
2964
2965	Type *ITy = getCompareTy(Op: LHS); // The return type.
2966	switch (Pred) {
2967	default:
2968	llvm_unreachable("Unknown ICmp predicate!");
2969	case ICmpInst::ICMP_ULT:
2970	return getFalse(Ty: ITy);
2971	case ICmpInst::ICMP_UGE:
2972	return getTrue(Ty: ITy);
2973	case ICmpInst::ICMP_EQ:
2974	case ICmpInst::ICMP_ULE:
2975	if (isKnownNonZero(V: LHS, Q))
2976	return getFalse(Ty: ITy);
2977	break;
2978	case ICmpInst::ICMP_NE:
2979	case ICmpInst::ICMP_UGT:
2980	if (isKnownNonZero(V: LHS, Q))
2981	return getTrue(Ty: ITy);
2982	break;
2983	case ICmpInst::ICMP_SLT: {
2984	KnownBits LHSKnown = computeKnownBits(V: LHS, /* Depth */ 0, Q);
2985	if (LHSKnown.isNegative())
2986	return getTrue(Ty: ITy);
2987	if (LHSKnown.isNonNegative())
2988	return getFalse(Ty: ITy);
2989	break;
2990	}
2991	case ICmpInst::ICMP_SLE: {
2992	KnownBits LHSKnown = computeKnownBits(V: LHS, /* Depth */ 0, Q);
2993	if (LHSKnown.isNegative())
2994	return getTrue(Ty: ITy);
2995	if (LHSKnown.isNonNegative() && isKnownNonZero(V: LHS, Q))
2996	return getFalse(Ty: ITy);
2997	break;
2998	}
2999	case ICmpInst::ICMP_SGE: {
3000	KnownBits LHSKnown = computeKnownBits(V: LHS, /* Depth */ 0, Q);
3001	if (LHSKnown.isNegative())
3002	return getFalse(Ty: ITy);
3003	if (LHSKnown.isNonNegative())
3004	return getTrue(Ty: ITy);
3005	break;
3006	}
3007	case ICmpInst::ICMP_SGT: {
3008	KnownBits LHSKnown = computeKnownBits(V: LHS, /* Depth */ 0, Q);
3009	if (LHSKnown.isNegative())
3010	return getFalse(Ty: ITy);
3011	if (LHSKnown.isNonNegative() && isKnownNonZero(V: LHS, Q))
3012	return getTrue(Ty: ITy);
3013	break;
3014	}
3015	}
3016
3017	return nullptr;
3018	}
3019
3020	static Value *simplifyICmpWithConstant(CmpInst::Predicate Pred, Value *LHS,
3021	Value *RHS, const InstrInfoQuery &IIQ) {
3022	Type *ITy = getCompareTy(Op: RHS); // The return type.
3023
3024	Value *X;
3025	const APInt *C;
3026	if (!match(V: RHS, P: m_APIntAllowPoison(Res&: C)))
3027	return nullptr;
3028
3029	// Sign-bit checks can be optimized to true/false after unsigned
3030	// floating-point casts:
3031	// icmp slt (bitcast (uitofp X)), 0 --> false
3032	// icmp sgt (bitcast (uitofp X)), -1 --> true
3033	if (match(V: LHS, P: m_ElementWiseBitCast(Op: m_UIToFP(Op: m_Value(V&: X))))) {
3034	bool TrueIfSigned;
3035	if (isSignBitCheck(Pred, RHS: *C, TrueIfSigned))
3036	return ConstantInt::getBool(Ty: ITy, V: !TrueIfSigned);
3037	}
3038
3039	// Rule out tautological comparisons (eg., ult 0 or uge 0).
3040	ConstantRange RHS_CR = ConstantRange::makeExactICmpRegion(Pred, Other: *C);
3041	if (RHS_CR.isEmptySet())
3042	return ConstantInt::getFalse(Ty: ITy);
3043	if (RHS_CR.isFullSet())
3044	return ConstantInt::getTrue(Ty: ITy);
3045
3046	ConstantRange LHS_CR =
3047	computeConstantRange(V: LHS, ForSigned: CmpInst::isSigned(predicate: Pred), UseInstrInfo: IIQ.UseInstrInfo);
3048	if (!LHS_CR.isFullSet()) {
3049	if (RHS_CR.contains(CR: LHS_CR))
3050	return ConstantInt::getTrue(Ty: ITy);
3051	if (RHS_CR.inverse().contains(CR: LHS_CR))
3052	return ConstantInt::getFalse(Ty: ITy);
3053	}
3054
3055	// (mul nuw/nsw X, MulC) != C --> true (if C is not a multiple of MulC)
3056	// (mul nuw/nsw X, MulC) == C --> false (if C is not a multiple of MulC)
3057	const APInt *MulC;
3058	if (IIQ.UseInstrInfo && ICmpInst::isEquality(P: Pred) &&
3059	((match(V: LHS, P: m_NUWMul(L: m_Value(), R: m_APIntAllowPoison(Res&: MulC))) &&
3060	*MulC != 0 && C->urem(RHS: *MulC) != 0) ||
3061	(match(V: LHS, P: m_NSWMul(L: m_Value(), R: m_APIntAllowPoison(Res&: MulC))) &&
3062	*MulC != 0 && C->srem(RHS: *MulC) != 0)))
3063	return ConstantInt::get(Ty: ITy, V: Pred == ICmpInst::ICMP_NE);
3064
3065	return nullptr;
3066	}
3067
3068	static Value *simplifyICmpWithBinOpOnLHS(CmpInst::Predicate Pred,
3069	BinaryOperator *LBO, Value *RHS,
3070	const SimplifyQuery &Q,
3071	unsigned MaxRecurse) {
3072	Type *ITy = getCompareTy(Op: RHS); // The return type.
3073
3074	Value *Y = nullptr;
3075	// icmp pred (or X, Y), X
3076	if (match(V: LBO, P: m_c_Or(L: m_Value(V&: Y), R: m_Specific(V: RHS)))) {
3077	if (Pred == ICmpInst::ICMP_ULT)
3078	return getFalse(Ty: ITy);
3079	if (Pred == ICmpInst::ICMP_UGE)
3080	return getTrue(Ty: ITy);
3081
3082	if (Pred == ICmpInst::ICMP_SLT || Pred == ICmpInst::ICMP_SGE) {
3083	KnownBits RHSKnown = computeKnownBits(V: RHS, /* Depth */ 0, Q);
3084	KnownBits YKnown = computeKnownBits(V: Y, /* Depth */ 0, Q);
3085	if (RHSKnown.isNonNegative() && YKnown.isNegative())
3086	return Pred == ICmpInst::ICMP_SLT ? getTrue(Ty: ITy) : getFalse(Ty: ITy);
3087	if (RHSKnown.isNegative() || YKnown.isNonNegative())
3088	return Pred == ICmpInst::ICMP_SLT ? getFalse(Ty: ITy) : getTrue(Ty: ITy);
3089	}
3090	}
3091
3092	// icmp pred (and X, Y), X
3093	if (match(V: LBO, P: m_c_And(L: m_Value(), R: m_Specific(V: RHS)))) {
3094	if (Pred == ICmpInst::ICMP_UGT)
3095	return getFalse(Ty: ITy);
3096	if (Pred == ICmpInst::ICMP_ULE)
3097	return getTrue(Ty: ITy);
3098	}
3099
3100	// icmp pred (urem X, Y), Y
3101	if (match(V: LBO, P: m_URem(L: m_Value(), R: m_Specific(V: RHS)))) {
3102	switch (Pred) {
3103	default:
3104	break;
3105	case ICmpInst::ICMP_SGT:
3106	case ICmpInst::ICMP_SGE: {
3107	KnownBits Known = computeKnownBits(V: RHS, /* Depth */ 0, Q);
3108	if (!Known.isNonNegative())
3109	break;
3110	[[fallthrough]];
3111	}
3112	case ICmpInst::ICMP_EQ:
3113	case ICmpInst::ICMP_UGT:
3114	case ICmpInst::ICMP_UGE:
3115	return getFalse(Ty: ITy);
3116	case ICmpInst::ICMP_SLT:
3117	case ICmpInst::ICMP_SLE: {
3118	KnownBits Known = computeKnownBits(V: RHS, /* Depth */ 0, Q);
3119	if (!Known.isNonNegative())
3120	break;
3121	[[fallthrough]];
3122	}
3123	case ICmpInst::ICMP_NE:
3124	case ICmpInst::ICMP_ULT:
3125	case ICmpInst::ICMP_ULE:
3126	return getTrue(Ty: ITy);
3127	}
3128	}
3129
3130	// icmp pred (urem X, Y), X
3131	if (match(V: LBO, P: m_URem(L: m_Specific(V: RHS), R: m_Value()))) {
3132	if (Pred == ICmpInst::ICMP_ULE)
3133	return getTrue(Ty: ITy);
3134	if (Pred == ICmpInst::ICMP_UGT)
3135	return getFalse(Ty: ITy);
3136	}
3137
3138	// x >>u y <=u x --> true.
3139	// x >>u y >u x --> false.
3140	// x udiv y <=u x --> true.
3141	// x udiv y >u x --> false.
3142	if (match(V: LBO, P: m_LShr(L: m_Specific(V: RHS), R: m_Value())) ||
3143	match(V: LBO, P: m_UDiv(L: m_Specific(V: RHS), R: m_Value()))) {
3144	// icmp pred (X op Y), X
3145	if (Pred == ICmpInst::ICMP_UGT)
3146	return getFalse(Ty: ITy);
3147	if (Pred == ICmpInst::ICMP_ULE)
3148	return getTrue(Ty: ITy);
3149	}
3150
3151	// If x is nonzero:
3152	// x >>u C <u x --> true for C != 0.
3153	// x >>u C != x --> true for C != 0.
3154	// x >>u C >=u x --> false for C != 0.
3155	// x >>u C == x --> false for C != 0.
3156	// x udiv C <u x --> true for C != 1.
3157	// x udiv C != x --> true for C != 1.
3158	// x udiv C >=u x --> false for C != 1.
3159	// x udiv C == x --> false for C != 1.
3160	// TODO: allow non-constant shift amount/divisor
3161	const APInt *C;
3162	if ((match(V: LBO, P: m_LShr(L: m_Specific(V: RHS), R: m_APInt(Res&: C))) && *C != 0) ||
3163	(match(V: LBO, P: m_UDiv(L: m_Specific(V: RHS), R: m_APInt(Res&: C))) && *C != 1)) {
3164	if (isKnownNonZero(V: RHS, Q)) {
3165	switch (Pred) {
3166	default:
3167	break;
3168	case ICmpInst::ICMP_EQ:
3169	case ICmpInst::ICMP_UGE:
3170	return getFalse(Ty: ITy);
3171	case ICmpInst::ICMP_NE:
3172	case ICmpInst::ICMP_ULT:
3173	return getTrue(Ty: ITy);
3174	case ICmpInst::ICMP_UGT:
3175	case ICmpInst::ICMP_ULE:
3176	// UGT/ULE are handled by the more general case just above
3177	llvm_unreachable("Unexpected UGT/ULE, should have been handled");
3178	}
3179	}
3180	}
3181
3182	// (x*C1)/C2 <= x for C1 <= C2.
3183	// This holds even if the multiplication overflows: Assume that x != 0 and
3184	// arithmetic is modulo M. For overflow to occur we must have C1 >= M/x and
3185	// thus C2 >= M/x. It follows that (x*C1)/C2 <= (M-1)/C2 <= ((M-1)*x)/M < x.
3186	//
3187	// Additionally, either the multiplication and division might be represented
3188	// as shifts:
3189	// (x*C1)>>C2 <= x for C1 < 2**C2.
3190	// (x<<C1)/C2 <= x for 2**C1 < C2.
3191	const APInt *C1, *C2;
3192	if ((match(V: LBO, P: m_UDiv(L: m_Mul(L: m_Specific(V: RHS), R: m_APInt(Res&: C1)), R: m_APInt(Res&: C2))) &&
3193	C1->ule(RHS: *C2)) ||
3194	(match(V: LBO, P: m_LShr(L: m_Mul(L: m_Specific(V: RHS), R: m_APInt(Res&: C1)), R: m_APInt(Res&: C2))) &&
3195	C1->ule(RHS: APInt(C2->getBitWidth(), 1) << *C2)) ||
3196	(match(V: LBO, P: m_UDiv(L: m_Shl(L: m_Specific(V: RHS), R: m_APInt(Res&: C1)), R: m_APInt(Res&: C2))) &&
3197	(APInt(C1->getBitWidth(), 1) << *C1).ule(RHS: *C2))) {
3198	if (Pred == ICmpInst::ICMP_UGT)
3199	return getFalse(Ty: ITy);
3200	if (Pred == ICmpInst::ICMP_ULE)
3201	return getTrue(Ty: ITy);
3202	}
3203
3204	// (sub C, X) == X, C is odd --> false
3205	// (sub C, X) != X, C is odd --> true
3206	if (match(V: LBO, P: m_Sub(L: m_APIntAllowPoison(Res&: C), R: m_Specific(V: RHS))) &&
3207	(*C & 1) == 1 && ICmpInst::isEquality(P: Pred))
3208	return (Pred == ICmpInst::ICMP_EQ) ? getFalse(Ty: ITy) : getTrue(Ty: ITy);
3209
3210	return nullptr;
3211	}
3212
3213	// If only one of the icmp's operands has NSW flags, try to prove that:
3214	//
3215	// icmp slt (x + C1), (x +nsw C2)
3216	//
3217	// is equivalent to:
3218	//
3219	// icmp slt C1, C2
3220	//
3221	// which is true if x + C2 has the NSW flags set and:
3222	// *) C1 < C2 && C1 >= 0, or
3223	// *) C2 < C1 && C1 <= 0.
3224	//
3225	static bool trySimplifyICmpWithAdds(CmpInst::Predicate Pred, Value *LHS,
3226	Value *RHS, const InstrInfoQuery &IIQ) {
3227	// TODO: only support icmp slt for now.
3228	if (Pred != CmpInst::ICMP_SLT || !IIQ.UseInstrInfo)
3229	return false;
3230
3231	// Canonicalize nsw add as RHS.
3232	if (!match(V: RHS, P: m_NSWAdd(L: m_Value(), R: m_Value())))
3233	std::swap(a&: LHS, b&: RHS);
3234	if (!match(V: RHS, P: m_NSWAdd(L: m_Value(), R: m_Value())))
3235	return false;
3236
3237	Value *X;
3238	const APInt *C1, *C2;
3239	if (!match(V: LHS, P: m_Add(L: m_Value(V&: X), R: m_APInt(Res&: C1))) ||
3240	!match(V: RHS, P: m_Add(L: m_Specific(V: X), R: m_APInt(Res&: C2))))
3241	return false;
3242
3243	return (C1->slt(RHS: *C2) && C1->isNonNegative()) ||
3244	(C2->slt(RHS: *C1) && C1->isNonPositive());
3245	}
3246
3247	/// TODO: A large part of this logic is duplicated in InstCombine's
3248	/// foldICmpBinOp(). We should be able to share that and avoid the code
3249	/// duplication.
3250	static Value *simplifyICmpWithBinOp(CmpInst::Predicate Pred, Value *LHS,
3251	Value *RHS, const SimplifyQuery &Q,
3252	unsigned MaxRecurse) {
3253	BinaryOperator *LBO = dyn_cast<BinaryOperator>(Val: LHS);
3254	BinaryOperator *RBO = dyn_cast<BinaryOperator>(Val: RHS);
3255	if (MaxRecurse && (LBO || RBO)) {
3256	// Analyze the case when either LHS or RHS is an add instruction.
3257	Value *A = nullptr, *B = nullptr, *C = nullptr, *D = nullptr;
3258	// LHS = A + B (or A and B are null); RHS = C + D (or C and D are null).
3259	bool NoLHSWrapProblem = false, NoRHSWrapProblem = false;
3260	if (LBO && LBO->getOpcode() == Instruction::Add) {
3261	A = LBO->getOperand(i_nocapture: 0);
3262	B = LBO->getOperand(i_nocapture: 1);
3263	NoLHSWrapProblem =
3264	ICmpInst::isEquality(P: Pred) ||
3265	(CmpInst::isUnsigned(predicate: Pred) &&
3266	Q.IIQ.hasNoUnsignedWrap(Op: cast<OverflowingBinaryOperator>(Val: LBO))) ||
3267	(CmpInst::isSigned(predicate: Pred) &&
3268	Q.IIQ.hasNoSignedWrap(Op: cast<OverflowingBinaryOperator>(Val: LBO)));
3269	}
3270	if (RBO && RBO->getOpcode() == Instruction::Add) {
3271	C = RBO->getOperand(i_nocapture: 0);
3272	D = RBO->getOperand(i_nocapture: 1);
3273	NoRHSWrapProblem =
3274	ICmpInst::isEquality(P: Pred) ||
3275	(CmpInst::isUnsigned(predicate: Pred) &&
3276	Q.IIQ.hasNoUnsignedWrap(Op: cast<OverflowingBinaryOperator>(Val: RBO))) ||
3277	(CmpInst::isSigned(predicate: Pred) &&
3278	Q.IIQ.hasNoSignedWrap(Op: cast<OverflowingBinaryOperator>(Val: RBO)));
3279	}
3280
3281	// icmp (X+Y), X -> icmp Y, 0 for equalities or if there is no overflow.
3282	if ((A == RHS || B == RHS) && NoLHSWrapProblem)
3283	if (Value *V = simplifyICmpInst(Predicate: Pred, LHS: A == RHS ? B : A,
3284	RHS: Constant::getNullValue(Ty: RHS->getType()), Q,
3285	MaxRecurse: MaxRecurse - 1))
3286	return V;
3287
3288	// icmp X, (X+Y) -> icmp 0, Y for equalities or if there is no overflow.
3289	if ((C == LHS || D == LHS) && NoRHSWrapProblem)
3290	if (Value *V =
3291	simplifyICmpInst(Predicate: Pred, LHS: Constant::getNullValue(Ty: LHS->getType()),
3292	RHS: C == LHS ? D : C, Q, MaxRecurse: MaxRecurse - 1))
3293	return V;
3294
3295	// icmp (X+Y), (X+Z) -> icmp Y,Z for equalities or if there is no overflow.
3296	bool CanSimplify = (NoLHSWrapProblem && NoRHSWrapProblem) ||
3297	trySimplifyICmpWithAdds(Pred, LHS, RHS, IIQ: Q.IIQ);
3298	if (A && C && (A == C || A == D || B == C || B == D) && CanSimplify) {
3299	// Determine Y and Z in the form icmp (X+Y), (X+Z).
3300	Value *Y, *Z;
3301	if (A == C) {
3302	// C + B == C + D -> B == D
3303	Y = B;
3304	Z = D;
3305	} else if (A == D) {
3306	// D + B == C + D -> B == C
3307	Y = B;
3308	Z = C;
3309	} else if (B == C) {
3310	// A + C == C + D -> A == D
3311	Y = A;
3312	Z = D;
3313	} else {
3314	assert(B == D);
3315	// A + D == C + D -> A == C
3316	Y = A;
3317	Z = C;
3318	}
3319	if (Value *V = simplifyICmpInst(Predicate: Pred, LHS: Y, RHS: Z, Q, MaxRecurse: MaxRecurse - 1))
3320	return V;
3321	}
3322	}
3323
3324	if (LBO)
3325	if (Value *V = simplifyICmpWithBinOpOnLHS(Pred, LBO, RHS, Q, MaxRecurse))
3326	return V;
3327
3328	if (RBO)
3329	if (Value *V = simplifyICmpWithBinOpOnLHS(
3330	Pred: ICmpInst::getSwappedPredicate(pred: Pred), LBO: RBO, RHS: LHS, Q, MaxRecurse))
3331	return V;
3332
3333	// 0 - (zext X) pred C
3334	if (!CmpInst::isUnsigned(predicate: Pred) && match(V: LHS, P: m_Neg(V: m_ZExt(Op: m_Value())))) {
3335	const APInt *C;
3336	if (match(V: RHS, P: m_APInt(Res&: C))) {
3337	if (C->isStrictlyPositive()) {
3338	if (Pred == ICmpInst::ICMP_SLT || Pred == ICmpInst::ICMP_NE)
3339	return ConstantInt::getTrue(Ty: getCompareTy(Op: RHS));
3340	if (Pred == ICmpInst::ICMP_SGE || Pred == ICmpInst::ICMP_EQ)
3341	return ConstantInt::getFalse(Ty: getCompareTy(Op: RHS));
3342	}
3343	if (C->isNonNegative()) {
3344	if (Pred == ICmpInst::ICMP_SLE)
3345	return ConstantInt::getTrue(Ty: getCompareTy(Op: RHS));
3346	if (Pred == ICmpInst::ICMP_SGT)
3347	return ConstantInt::getFalse(Ty: getCompareTy(Op: RHS));
3348	}
3349	}
3350	}
3351
3352	// If C2 is a power-of-2 and C is not:
3353	// (C2 << X) == C --> false
3354	// (C2 << X) != C --> true
3355	const APInt *C;
3356	if (match(V: LHS, P: m_Shl(L: m_Power2(), R: m_Value())) &&
3357	match(V: RHS, P: m_APIntAllowPoison(Res&: C)) && !C->isPowerOf2()) {
3358	// C2 << X can equal zero in some circumstances.
3359	// This simplification might be unsafe if C is zero.
3360	//
3361	// We know it is safe if:
3362	// - The shift is nsw. We can't shift out the one bit.
3363	// - The shift is nuw. We can't shift out the one bit.
3364	// - C2 is one.
3365	// - C isn't zero.
3366	if (Q.IIQ.hasNoSignedWrap(Op: cast<OverflowingBinaryOperator>(Val: LBO)) ||
3367	Q.IIQ.hasNoUnsignedWrap(Op: cast<OverflowingBinaryOperator>(Val: LBO)) ||
3368	match(V: LHS, P: m_Shl(L: m_One(), R: m_Value())) || !C->isZero()) {
3369	if (Pred == ICmpInst::ICMP_EQ)
3370	return ConstantInt::getFalse(Ty: getCompareTy(Op: RHS));
3371	if (Pred == ICmpInst::ICMP_NE)
3372	return ConstantInt::getTrue(Ty: getCompareTy(Op: RHS));
3373	}
3374	}
3375
3376	// If C is a power-of-2:
3377	// (C << X) >u 0x8000 --> false
3378	// (C << X) <=u 0x8000 --> true
3379	if (match(V: LHS, P: m_Shl(L: m_Power2(), R: m_Value())) && match(V: RHS, P: m_SignMask())) {
3380	if (Pred == ICmpInst::ICMP_UGT)
3381	return ConstantInt::getFalse(Ty: getCompareTy(Op: RHS));
3382	if (Pred == ICmpInst::ICMP_ULE)
3383	return ConstantInt::getTrue(Ty: getCompareTy(Op: RHS));
3384	}
3385
3386	if (!MaxRecurse || !LBO || !RBO || LBO->getOpcode() != RBO->getOpcode())
3387	return nullptr;
3388
3389	if (LBO->getOperand(i_nocapture: 0) == RBO->getOperand(i_nocapture: 0)) {
3390	switch (LBO->getOpcode()) {
3391	default:
3392	break;
3393	case Instruction::Shl: {
3394	bool NUW = Q.IIQ.hasNoUnsignedWrap(Op: LBO) && Q.IIQ.hasNoUnsignedWrap(Op: RBO);
3395	bool NSW = Q.IIQ.hasNoSignedWrap(Op: LBO) && Q.IIQ.hasNoSignedWrap(Op: RBO);
3396	if (!NUW || (ICmpInst::isSigned(predicate: Pred) && !NSW) ||
3397	!isKnownNonZero(V: LBO->getOperand(i_nocapture: 0), Q))
3398	break;
3399	if (Value *V = simplifyICmpInst(Predicate: Pred, LHS: LBO->getOperand(i_nocapture: 1),
3400	RHS: RBO->getOperand(i_nocapture: 1), Q, MaxRecurse: MaxRecurse - 1))
3401	return V;
3402	break;
3403	}
3404	// If C1 & C2 == C1, A = X and/or C1, B = X and/or C2:
3405	// icmp ule A, B -> true
3406	// icmp ugt A, B -> false
3407	// icmp sle A, B -> true (C1 and C2 are the same sign)
3408	// icmp sgt A, B -> false (C1 and C2 are the same sign)
3409	case Instruction::And:
3410	case Instruction::Or: {
3411	const APInt *C1, *C2;
3412	if (ICmpInst::isRelational(P: Pred) &&
3413	match(V: LBO->getOperand(i_nocapture: 1), P: m_APInt(Res&: C1)) &&
3414	match(V: RBO->getOperand(i_nocapture: 1), P: m_APInt(Res&: C2))) {
3415	if (!C1->isSubsetOf(RHS: *C2)) {
3416	std::swap(a&: C1, b&: C2);
3417	Pred = ICmpInst::getSwappedPredicate(pred: Pred);
3418	}
3419	if (C1->isSubsetOf(RHS: *C2)) {
3420	if (Pred == ICmpInst::ICMP_ULE)
3421	return ConstantInt::getTrue(Ty: getCompareTy(Op: LHS));
3422	if (Pred == ICmpInst::ICMP_UGT)
3423	return ConstantInt::getFalse(Ty: getCompareTy(Op: LHS));
3424	if (C1->isNonNegative() == C2->isNonNegative()) {
3425	if (Pred == ICmpInst::ICMP_SLE)
3426	return ConstantInt::getTrue(Ty: getCompareTy(Op: LHS));
3427	if (Pred == ICmpInst::ICMP_SGT)
3428	return ConstantInt::getFalse(Ty: getCompareTy(Op: LHS));
3429	}
3430	}
3431	}
3432	break;
3433	}
3434	}
3435	}
3436
3437	if (LBO->getOperand(i_nocapture: 1) == RBO->getOperand(i_nocapture: 1)) {
3438	switch (LBO->getOpcode()) {
3439	default:
3440	break;
3441	case Instruction::UDiv:
3442	case Instruction::LShr:
3443	if (ICmpInst::isSigned(predicate: Pred) || !Q.IIQ.isExact(Op: LBO) ||
3444	!Q.IIQ.isExact(Op: RBO))
3445	break;
3446	if (Value *V = simplifyICmpInst(Predicate: Pred, LHS: LBO->getOperand(i_nocapture: 0),
3447	RHS: RBO->getOperand(i_nocapture: 0), Q, MaxRecurse: MaxRecurse - 1))
3448	return V;
3449	break;
3450	case Instruction::SDiv:
3451	if (!ICmpInst::isEquality(P: Pred) || !Q.IIQ.isExact(Op: LBO) ||
3452	!Q.IIQ.isExact(Op: RBO))
3453	break;
3454	if (Value *V = simplifyICmpInst(Predicate: Pred, LHS: LBO->getOperand(i_nocapture: 0),
3455	RHS: RBO->getOperand(i_nocapture: 0), Q, MaxRecurse: MaxRecurse - 1))
3456	return V;
3457	break;
3458	case Instruction::AShr:
3459	if (!Q.IIQ.isExact(Op: LBO) || !Q.IIQ.isExact(Op: RBO))
3460	break;
3461	if (Value *V = simplifyICmpInst(Predicate: Pred, LHS: LBO->getOperand(i_nocapture: 0),
3462	RHS: RBO->getOperand(i_nocapture: 0), Q, MaxRecurse: MaxRecurse - 1))
3463	return V;
3464	break;
3465	case Instruction::Shl: {
3466	bool NUW = Q.IIQ.hasNoUnsignedWrap(Op: LBO) && Q.IIQ.hasNoUnsignedWrap(Op: RBO);
3467	bool NSW = Q.IIQ.hasNoSignedWrap(Op: LBO) && Q.IIQ.hasNoSignedWrap(Op: RBO);
3468	if (!NUW && !NSW)
3469	break;
3470	if (!NSW && ICmpInst::isSigned(predicate: Pred))
3471	break;
3472	if (Value *V = simplifyICmpInst(Predicate: Pred, LHS: LBO->getOperand(i_nocapture: 0),
3473	RHS: RBO->getOperand(i_nocapture: 0), Q, MaxRecurse: MaxRecurse - 1))
3474	return V;
3475	break;
3476	}
3477	}
3478	}
3479	return nullptr;
3480	}
3481
3482	/// simplify integer comparisons where at least one operand of the compare
3483	/// matches an integer min/max idiom.
3484	static Value *simplifyICmpWithMinMax(CmpInst::Predicate Pred, Value *LHS,
3485	Value *RHS, const SimplifyQuery &Q,
3486	unsigned MaxRecurse) {
3487	Type *ITy = getCompareTy(Op: LHS); // The return type.
3488	Value *A, *B;
3489	CmpInst::Predicate P = CmpInst::BAD_ICMP_PREDICATE;
3490	CmpInst::Predicate EqP; // Chosen so that "A == max/min(A,B)" iff "A EqP B".
3491
3492	// Signed variants on "max(a,b)>=a -> true".
3493	if (match(V: LHS, P: m_SMax(L: m_Value(V&: A), R: m_Value(V&: B))) && (A == RHS || B == RHS)) {
3494	if (A != RHS)
3495	std::swap(a&: A, b&: B); // smax(A, B) pred A.
3496	EqP = CmpInst::ICMP_SGE; // "A == smax(A, B)" iff "A sge B".
3497	// We analyze this as smax(A, B) pred A.
3498	P = Pred;
3499	} else if (match(V: RHS, P: m_SMax(L: m_Value(V&: A), R: m_Value(V&: B))) &&
3500	(A == LHS || B == LHS)) {
3501	if (A != LHS)
3502	std::swap(a&: A, b&: B); // A pred smax(A, B).
3503	EqP = CmpInst::ICMP_SGE; // "A == smax(A, B)" iff "A sge B".
3504	// We analyze this as smax(A, B) swapped-pred A.
3505	P = CmpInst::getSwappedPredicate(pred: Pred);
3506	} else if (match(V: LHS, P: m_SMin(L: m_Value(V&: A), R: m_Value(V&: B))) &&
3507	(A == RHS || B == RHS)) {
3508	if (A != RHS)
3509	std::swap(a&: A, b&: B); // smin(A, B) pred A.
3510	EqP = CmpInst::ICMP_SLE; // "A == smin(A, B)" iff "A sle B".
3511	// We analyze this as smax(-A, -B) swapped-pred -A.
3512	// Note that we do not need to actually form -A or -B thanks to EqP.
3513	P = CmpInst::getSwappedPredicate(pred: Pred);
3514	} else if (match(V: RHS, P: m_SMin(L: m_Value(V&: A), R: m_Value(V&: B))) &&
3515	(A == LHS || B == LHS)) {
3516	if (A != LHS)
3517	std::swap(a&: A, b&: B); // A pred smin(A, B).
3518	EqP = CmpInst::ICMP_SLE; // "A == smin(A, B)" iff "A sle B".
3519	// We analyze this as smax(-A, -B) pred -A.
3520	// Note that we do not need to actually form -A or -B thanks to EqP.
3521	P = Pred;
3522	}
3523	if (P != CmpInst::BAD_ICMP_PREDICATE) {
3524	// Cases correspond to "max(A, B) p A".
3525	switch (P) {
3526	default:
3527	break;
3528	case CmpInst::ICMP_EQ:
3529	case CmpInst::ICMP_SLE:
3530	// Equivalent to "A EqP B". This may be the same as the condition tested
3531	// in the max/min; if so, we can just return that.
3532	if (Value *V = extractEquivalentCondition(V: LHS, Pred: EqP, LHS: A, RHS: B))
3533	return V;
3534	if (Value *V = extractEquivalentCondition(V: RHS, Pred: EqP, LHS: A, RHS: B))
3535	return V;
3536	// Otherwise, see if "A EqP B" simplifies.
3537	if (MaxRecurse)
3538	if (Value *V = simplifyICmpInst(Predicate: EqP, LHS: A, RHS: B, Q, MaxRecurse: MaxRecurse - 1))
3539	return V;
3540	break;
3541	case CmpInst::ICMP_NE:
3542	case CmpInst::ICMP_SGT: {
3543	CmpInst::Predicate InvEqP = CmpInst::getInversePredicate(pred: EqP);
3544	// Equivalent to "A InvEqP B". This may be the same as the condition
3545	// tested in the max/min; if so, we can just return that.
3546	if (Value *V = extractEquivalentCondition(V: LHS, Pred: InvEqP, LHS: A, RHS: B))
3547	return V;
3548	if (Value *V = extractEquivalentCondition(V: RHS, Pred: InvEqP, LHS: A, RHS: B))
3549	return V;
3550	// Otherwise, see if "A InvEqP B" simplifies.
3551	if (MaxRecurse)
3552	if (Value *V = simplifyICmpInst(Predicate: InvEqP, LHS: A, RHS: B, Q, MaxRecurse: MaxRecurse - 1))
3553	return V;
3554	break;
3555	}
3556	case CmpInst::ICMP_SGE:
3557	// Always true.
3558	return getTrue(Ty: ITy);
3559	case CmpInst::ICMP_SLT:
3560	// Always false.
3561	return getFalse(Ty: ITy);
3562	}
3563	}
3564
3565	// Unsigned variants on "max(a,b)>=a -> true".
3566	P = CmpInst::BAD_ICMP_PREDICATE;
3567	if (match(V: LHS, P: m_UMax(L: m_Value(V&: A), R: m_Value(V&: B))) && (A == RHS || B == RHS)) {
3568	if (A != RHS)
3569	std::swap(a&: A, b&: B); // umax(A, B) pred A.
3570	EqP = CmpInst::ICMP_UGE; // "A == umax(A, B)" iff "A uge B".
3571	// We analyze this as umax(A, B) pred A.
3572	P = Pred;
3573	} else if (match(V: RHS, P: m_UMax(L: m_Value(V&: A), R: m_Value(V&: B))) &&
3574	(A == LHS || B == LHS)) {
3575	if (A != LHS)
3576	std::swap(a&: A, b&: B); // A pred umax(A, B).
3577	EqP = CmpInst::ICMP_UGE; // "A == umax(A, B)" iff "A uge B".
3578	// We analyze this as umax(A, B) swapped-pred A.
3579	P = CmpInst::getSwappedPredicate(pred: Pred);
3580	} else if (match(V: LHS, P: m_UMin(L: m_Value(V&: A), R: m_Value(V&: B))) &&
3581	(A == RHS || B == RHS)) {
3582	if (A != RHS)
3583	std::swap(a&: A, b&: B); // umin(A, B) pred A.
3584	EqP = CmpInst::ICMP_ULE; // "A == umin(A, B)" iff "A ule B".
3585	// We analyze this as umax(-A, -B) swapped-pred -A.
3586	// Note that we do not need to actually form -A or -B thanks to EqP.
3587	P = CmpInst::getSwappedPredicate(pred: Pred);
3588	} else if (match(V: RHS, P: m_UMin(L: m_Value(V&: A), R: m_Value(V&: B))) &&
3589	(A == LHS || B == LHS)) {
3590	if (A != LHS)
3591	std::swap(a&: A, b&: B); // A pred umin(A, B).
3592	EqP = CmpInst::ICMP_ULE; // "A == umin(A, B)" iff "A ule B".
3593	// We analyze this as umax(-A, -B) pred -A.
3594	// Note that we do not need to actually form -A or -B thanks to EqP.
3595	P = Pred;
3596	}
3597	if (P != CmpInst::BAD_ICMP_PREDICATE) {
3598	// Cases correspond to "max(A, B) p A".
3599	switch (P) {
3600	default:
3601	break;
3602	case CmpInst::ICMP_EQ:
3603	case CmpInst::ICMP_ULE:
3604	// Equivalent to "A EqP B". This may be the same as the condition tested
3605	// in the max/min; if so, we can just return that.
3606	if (Value *V = extractEquivalentCondition(V: LHS, Pred: EqP, LHS: A, RHS: B))
3607	return V;
3608	if (Value *V = extractEquivalentCondition(V: RHS, Pred: EqP, LHS: A, RHS: B))
3609	return V;
3610	// Otherwise, see if "A EqP B" simplifies.
3611	if (MaxRecurse)
3612	if (Value *V = simplifyICmpInst(Predicate: EqP, LHS: A, RHS: B, Q, MaxRecurse: MaxRecurse - 1))
3613	return V;
3614	break;
3615	case CmpInst::ICMP_NE:
3616	case CmpInst::ICMP_UGT: {
3617	CmpInst::Predicate InvEqP = CmpInst::getInversePredicate(pred: EqP);
3618	// Equivalent to "A InvEqP B". This may be the same as the condition
3619	// tested in the max/min; if so, we can just return that.
3620	if (Value *V = extractEquivalentCondition(V: LHS, Pred: InvEqP, LHS: A, RHS: B))
3621	return V;
3622	if (Value *V = extractEquivalentCondition(V: RHS, Pred: InvEqP, LHS: A, RHS: B))
3623	return V;
3624	// Otherwise, see if "A InvEqP B" simplifies.
3625	if (MaxRecurse)
3626	if (Value *V = simplifyICmpInst(Predicate: InvEqP, LHS: A, RHS: B, Q, MaxRecurse: MaxRecurse - 1))
3627	return V;
3628	break;
3629	}
3630	case CmpInst::ICMP_UGE:
3631	return getTrue(Ty: ITy);
3632	case CmpInst::ICMP_ULT:
3633	return getFalse(Ty: ITy);
3634	}
3635	}
3636
3637	// Comparing 1 each of min/max with a common operand?
3638	// Canonicalize min operand to RHS.
3639	if (match(V: LHS, P: m_UMin(L: m_Value(), R: m_Value())) ||
3640	match(V: LHS, P: m_SMin(L: m_Value(), R: m_Value()))) {
3641	std::swap(a&: LHS, b&: RHS);
3642	Pred = ICmpInst::getSwappedPredicate(pred: Pred);
3643	}
3644
3645	Value *C, *D;
3646	if (match(V: LHS, P: m_SMax(L: m_Value(V&: A), R: m_Value(V&: B))) &&
3647	match(V: RHS, P: m_SMin(L: m_Value(V&: C), R: m_Value(V&: D))) &&
3648	(A == C || A == D || B == C || B == D)) {
3649	// smax(A, B) >=s smin(A, D) --> true
3650	if (Pred == CmpInst::ICMP_SGE)
3651	return getTrue(Ty: ITy);
3652	// smax(A, B) <s smin(A, D) --> false
3653	if (Pred == CmpInst::ICMP_SLT)
3654	return getFalse(Ty: ITy);
3655	} else if (match(V: LHS, P: m_UMax(L: m_Value(V&: A), R: m_Value(V&: B))) &&
3656	match(V: RHS, P: m_UMin(L: m_Value(V&: C), R: m_Value(V&: D))) &&
3657	(A == C || A == D || B == C || B == D)) {
3658	// umax(A, B) >=u umin(A, D) --> true
3659	if (Pred == CmpInst::ICMP_UGE)
3660	return getTrue(Ty: ITy);
3661	// umax(A, B) <u umin(A, D) --> false
3662	if (Pred == CmpInst::ICMP_ULT)
3663	return getFalse(Ty: ITy);
3664	}
3665
3666	return nullptr;
3667	}
3668
3669	static Value *simplifyICmpWithDominatingAssume(CmpInst::Predicate Predicate,
3670	Value *LHS, Value *RHS,
3671	const SimplifyQuery &Q) {
3672	// Gracefully handle instructions that have not been inserted yet.
3673	if (!Q.AC || !Q.CxtI)
3674	return nullptr;
3675
3676	for (Value *AssumeBaseOp : {LHS, RHS}) {
3677	for (auto &AssumeVH : Q.AC->assumptionsFor(V: AssumeBaseOp)) {
3678	if (!AssumeVH)
3679	continue;
3680
3681	CallInst *Assume = cast<CallInst>(Val&: AssumeVH);
3682	if (std::optional<bool> Imp = isImpliedCondition(
3683	LHS: Assume->getArgOperand(i: 0), RHSPred: Predicate, RHSOp0: LHS, RHSOp1: RHS, DL: Q.DL))
3684	if (isValidAssumeForContext(I: Assume, CxtI: Q.CxtI, DT: Q.DT))
3685	return ConstantInt::get(Ty: getCompareTy(Op: LHS), V: *Imp);
3686	}
3687	}
3688
3689	return nullptr;
3690	}
3691
3692	static Value *simplifyICmpWithIntrinsicOnLHS(CmpInst::Predicate Pred,
3693	Value *LHS, Value *RHS) {
3694	auto *II = dyn_cast<IntrinsicInst>(Val: LHS);
3695	if (!II)
3696	return nullptr;
3697
3698	switch (II->getIntrinsicID()) {
3699	case Intrinsic::uadd_sat:
3700	// uadd.sat(X, Y) uge X, uadd.sat(X, Y) uge Y
3701	if (II->getArgOperand(i: 0) == RHS || II->getArgOperand(i: 1) == RHS) {
3702	if (Pred == ICmpInst::ICMP_UGE)
3703	return ConstantInt::getTrue(Ty: getCompareTy(Op: II));
3704	if (Pred == ICmpInst::ICMP_ULT)
3705	return ConstantInt::getFalse(Ty: getCompareTy(Op: II));
3706	}
3707	return nullptr;
3708	case Intrinsic::usub_sat:
3709	// usub.sat(X, Y) ule X
3710	if (II->getArgOperand(i: 0) == RHS) {
3711	if (Pred == ICmpInst::ICMP_ULE)
3712	return ConstantInt::getTrue(Ty: getCompareTy(Op: II));
3713	if (Pred == ICmpInst::ICMP_UGT)
3714	return ConstantInt::getFalse(Ty: getCompareTy(Op: II));
3715	}
3716	return nullptr;
3717	default:
3718	return nullptr;
3719	}
3720	}
3721
3722	/// Helper method to get range from metadata or attribute.
3723	static std::optional<ConstantRange> getRange(Value *V,
3724	const InstrInfoQuery &IIQ) {
3725	if (Instruction *I = dyn_cast<Instruction>(Val: V))
3726	if (MDNode *MD = IIQ.getMetadata(I, KindID: LLVMContext::MD_range))
3727	return getConstantRangeFromMetadata(RangeMD: *MD);
3728
3729	if (const Argument *A = dyn_cast<Argument>(Val: V))
3730	return A->getRange();
3731	else if (const CallBase *CB = dyn_cast<CallBase>(Val: V))
3732	return CB->getRange();
3733
3734	return std::nullopt;
3735	}
3736
3737	/// Given operands for an ICmpInst, see if we can fold the result.
3738	/// If not, this returns null.
3739	static Value *simplifyICmpInst(unsigned Predicate, Value *LHS, Value *RHS,
3740	const SimplifyQuery &Q, unsigned MaxRecurse) {
3741	CmpInst::Predicate Pred = (CmpInst::Predicate)Predicate;
3742	assert(CmpInst::isIntPredicate(Pred) && "Not an integer compare!");
3743
3744	if (Constant *CLHS = dyn_cast<Constant>(Val: LHS)) {
3745	if (Constant *CRHS = dyn_cast<Constant>(Val: RHS))
3746	return ConstantFoldCompareInstOperands(Predicate: Pred, LHS: CLHS, RHS: CRHS, DL: Q.DL, TLI: Q.TLI);
3747
3748	// If we have a constant, make sure it is on the RHS.
3749	std::swap(a&: LHS, b&: RHS);
3750	Pred = CmpInst::getSwappedPredicate(pred: Pred);
3751	}
3752	assert(!isa<UndefValue>(LHS) && "Unexpected icmp undef,%X");
3753
3754	Type *ITy = getCompareTy(Op: LHS); // The return type.
3755
3756	// icmp poison, X -> poison
3757	if (isa<PoisonValue>(Val: RHS))
3758	return PoisonValue::get(T: ITy);
3759
3760	// For EQ and NE, we can always pick a value for the undef to make the
3761	// predicate pass or fail, so we can return undef.
3762	// Matches behavior in llvm::ConstantFoldCompareInstruction.
3763	if (Q.isUndefValue(V: RHS) && ICmpInst::isEquality(P: Pred))
3764	return UndefValue::get(T: ITy);
3765
3766	// icmp X, X -> true/false
3767	// icmp X, undef -> true/false because undef could be X.
3768	if (LHS == RHS || Q.isUndefValue(V: RHS))
3769	return ConstantInt::get(Ty: ITy, V: CmpInst::isTrueWhenEqual(predicate: Pred));
3770
3771	if (Value *V = simplifyICmpOfBools(Pred, LHS, RHS, Q))
3772	return V;
3773
3774	// TODO: Sink/common this with other potentially expensive calls that use
3775	// ValueTracking? See comment below for isKnownNonEqual().
3776	if (Value *V = simplifyICmpWithZero(Pred, LHS, RHS, Q))
3777	return V;
3778
3779	if (Value *V = simplifyICmpWithConstant(Pred, LHS, RHS, IIQ: Q.IIQ))
3780	return V;
3781
3782	// If both operands have range metadata, use the metadata
3783	// to simplify the comparison.
3784	if (std::optional<ConstantRange> RhsCr = getRange(V: RHS, IIQ: Q.IIQ))
3785	if (std::optional<ConstantRange> LhsCr = getRange(V: LHS, IIQ: Q.IIQ)) {
3786	if (LhsCr->icmp(Pred, Other: *RhsCr))
3787	return ConstantInt::getTrue(Ty: ITy);
3788
3789	if (LhsCr->icmp(Pred: CmpInst::getInversePredicate(pred: Pred), Other: *RhsCr))
3790	return ConstantInt::getFalse(Ty: ITy);
3791	}
3792
3793	// Compare of cast, for example (zext X) != 0 -> X != 0
3794	if (isa<CastInst>(Val: LHS) && (isa<Constant>(Val: RHS) || isa<CastInst>(Val: RHS))) {
3795	Instruction *LI = cast<CastInst>(Val: LHS);
3796	Value *SrcOp = LI->getOperand(i: 0);
3797	Type *SrcTy = SrcOp->getType();
3798	Type *DstTy = LI->getType();
3799
3800	// Turn icmp (ptrtoint x), (ptrtoint/constant) into a compare of the input
3801	// if the integer type is the same size as the pointer type.
3802	if (MaxRecurse && isa<PtrToIntInst>(Val: LI) &&
3803	Q.DL.getTypeSizeInBits(Ty: SrcTy) == DstTy->getPrimitiveSizeInBits()) {
3804	if (Constant *RHSC = dyn_cast<Constant>(Val: RHS)) {
3805	// Transfer the cast to the constant.
3806	if (Value *V = simplifyICmpInst(Predicate: Pred, LHS: SrcOp,
3807	RHS: ConstantExpr::getIntToPtr(C: RHSC, Ty: SrcTy),
3808	Q, MaxRecurse: MaxRecurse - 1))
3809	return V;
3810	} else if (PtrToIntInst *RI = dyn_cast<PtrToIntInst>(Val: RHS)) {
3811	if (RI->getOperand(i_nocapture: 0)->getType() == SrcTy)
3812	// Compare without the cast.
3813	if (Value *V = simplifyICmpInst(Predicate: Pred, LHS: SrcOp, RHS: RI->getOperand(i_nocapture: 0), Q,
3814	MaxRecurse: MaxRecurse - 1))
3815	return V;
3816	}
3817	}
3818
3819	if (isa<ZExtInst>(Val: LHS)) {
3820	// Turn icmp (zext X), (zext Y) into a compare of X and Y if they have the
3821	// same type.
3822	if (ZExtInst *RI = dyn_cast<ZExtInst>(Val: RHS)) {
3823	if (MaxRecurse && SrcTy == RI->getOperand(i_nocapture: 0)->getType())
3824	// Compare X and Y. Note that signed predicates become unsigned.
3825	if (Value *V =
3826	simplifyICmpInst(Predicate: ICmpInst::getUnsignedPredicate(pred: Pred), LHS: SrcOp,
3827	RHS: RI->getOperand(i_nocapture: 0), Q, MaxRecurse: MaxRecurse - 1))
3828	return V;
3829	}
3830	// Fold (zext X) ule (sext X), (zext X) sge (sext X) to true.
3831	else if (SExtInst *RI = dyn_cast<SExtInst>(Val: RHS)) {
3832	if (SrcOp == RI->getOperand(i_nocapture: 0)) {
3833	if (Pred == ICmpInst::ICMP_ULE || Pred == ICmpInst::ICMP_SGE)
3834	return ConstantInt::getTrue(Ty: ITy);
3835	if (Pred == ICmpInst::ICMP_UGT || Pred == ICmpInst::ICMP_SLT)
3836	return ConstantInt::getFalse(Ty: ITy);
3837	}
3838	}
3839	// Turn icmp (zext X), Cst into a compare of X and Cst if Cst is extended
3840	// too. If not, then try to deduce the result of the comparison.
3841	else if (match(V: RHS, P: m_ImmConstant())) {
3842	Constant *C = dyn_cast<Constant>(Val: RHS);
3843	assert(C != nullptr);
3844
3845	// Compute the constant that would happen if we truncated to SrcTy then
3846	// reextended to DstTy.
3847	Constant *Trunc =
3848	ConstantFoldCastOperand(Opcode: Instruction::Trunc, C, DestTy: SrcTy, DL: Q.DL);
3849	assert(Trunc && "Constant-fold of ImmConstant should not fail");
3850	Constant *RExt =
3851	ConstantFoldCastOperand(Opcode: CastInst::ZExt, C: Trunc, DestTy: DstTy, DL: Q.DL);
3852	assert(RExt && "Constant-fold of ImmConstant should not fail");
3853	Constant *AnyEq =
3854	ConstantFoldCompareInstOperands(Predicate: ICmpInst::ICMP_EQ, LHS: RExt, RHS: C, DL: Q.DL);
3855	assert(AnyEq && "Constant-fold of ImmConstant should not fail");
3856
3857	// If the re-extended constant didn't change any of the elements then
3858	// this is effectively also a case of comparing two zero-extended
3859	// values.
3860	if (AnyEq->isAllOnesValue() && MaxRecurse)
3861	if (Value *V = simplifyICmpInst(Predicate: ICmpInst::getUnsignedPredicate(pred: Pred),
3862	LHS: SrcOp, RHS: Trunc, Q, MaxRecurse: MaxRecurse - 1))
3863	return V;
3864
3865	// Otherwise the upper bits of LHS are zero while RHS has a non-zero bit
3866	// there. Use this to work out the result of the comparison.
3867	if (AnyEq->isNullValue()) {
3868	switch (Pred) {
3869	default:
3870	llvm_unreachable("Unknown ICmp predicate!");
3871	// LHS <u RHS.
3872	case ICmpInst::ICMP_EQ:
3873	case ICmpInst::ICMP_UGT:
3874	case ICmpInst::ICMP_UGE:
3875	return Constant::getNullValue(Ty: ITy);
3876
3877	case ICmpInst::ICMP_NE:
3878	case ICmpInst::ICMP_ULT:
3879	case ICmpInst::ICMP_ULE:
3880	return Constant::getAllOnesValue(Ty: ITy);
3881
3882	// LHS is non-negative. If RHS is negative then LHS >s LHS. If RHS
3883	// is non-negative then LHS <s RHS.
3884	case ICmpInst::ICMP_SGT:
3885	case ICmpInst::ICMP_SGE:
3886	return ConstantFoldCompareInstOperands(
3887	Predicate: ICmpInst::ICMP_SLT, LHS: C, RHS: Constant::getNullValue(Ty: C->getType()),
3888	DL: Q.DL);
3889	case ICmpInst::ICMP_SLT:
3890	case ICmpInst::ICMP_SLE:
3891	return ConstantFoldCompareInstOperands(
3892	Predicate: ICmpInst::ICMP_SGE, LHS: C, RHS: Constant::getNullValue(Ty: C->getType()),
3893	DL: Q.DL);
3894	}
3895	}
3896	}
3897	}
3898
3899	if (isa<SExtInst>(Val: LHS)) {
3900	// Turn icmp (sext X), (sext Y) into a compare of X and Y if they have the
3901	// same type.
3902	if (SExtInst *RI = dyn_cast<SExtInst>(Val: RHS)) {
3903	if (MaxRecurse && SrcTy == RI->getOperand(i_nocapture: 0)->getType())
3904	// Compare X and Y. Note that the predicate does not change.
3905	if (Value *V = simplifyICmpInst(Predicate: Pred, LHS: SrcOp, RHS: RI->getOperand(i_nocapture: 0), Q,
3906	MaxRecurse: MaxRecurse - 1))
3907	return V;
3908	}
3909	// Fold (sext X) uge (zext X), (sext X) sle (zext X) to true.
3910	else if (ZExtInst *RI = dyn_cast<ZExtInst>(Val: RHS)) {
3911	if (SrcOp == RI->getOperand(i_nocapture: 0)) {
3912	if (Pred == ICmpInst::ICMP_UGE || Pred == ICmpInst::ICMP_SLE)
3913	return ConstantInt::getTrue(Ty: ITy);
3914	if (Pred == ICmpInst::ICMP_ULT || Pred == ICmpInst::ICMP_SGT)
3915	return ConstantInt::getFalse(Ty: ITy);
3916	}
3917	}
3918	// Turn icmp (sext X), Cst into a compare of X and Cst if Cst is extended
3919	// too. If not, then try to deduce the result of the comparison.
3920	else if (match(V: RHS, P: m_ImmConstant())) {
3921	Constant *C = cast<Constant>(Val: RHS);
3922
3923	// Compute the constant that would happen if we truncated to SrcTy then
3924	// reextended to DstTy.
3925	Constant *Trunc =
3926	ConstantFoldCastOperand(Opcode: Instruction::Trunc, C, DestTy: SrcTy, DL: Q.DL);
3927	assert(Trunc && "Constant-fold of ImmConstant should not fail");
3928	Constant *RExt =
3929	ConstantFoldCastOperand(Opcode: CastInst::SExt, C: Trunc, DestTy: DstTy, DL: Q.DL);
3930	assert(RExt && "Constant-fold of ImmConstant should not fail");
3931	Constant *AnyEq =
3932	ConstantFoldCompareInstOperands(Predicate: ICmpInst::ICMP_EQ, LHS: RExt, RHS: C, DL: Q.DL);
3933	assert(AnyEq && "Constant-fold of ImmConstant should not fail");
3934
3935	// If the re-extended constant didn't change then this is effectively
3936	// also a case of comparing two sign-extended values.
3937	if (AnyEq->isAllOnesValue() && MaxRecurse)
3938	if (Value *V =
3939	simplifyICmpInst(Predicate: Pred, LHS: SrcOp, RHS: Trunc, Q, MaxRecurse: MaxRecurse - 1))
3940	return V;
3941
3942	// Otherwise the upper bits of LHS are all equal, while RHS has varying
3943	// bits there. Use this to work out the result of the comparison.
3944	if (AnyEq->isNullValue()) {
3945	switch (Pred) {
3946	default:
3947	llvm_unreachable("Unknown ICmp predicate!");
3948	case ICmpInst::ICMP_EQ:
3949	return Constant::getNullValue(Ty: ITy);
3950	case ICmpInst::ICMP_NE:
3951	return Constant::getAllOnesValue(Ty: ITy);
3952
3953	// If RHS is non-negative then LHS <s RHS. If RHS is negative then
3954	// LHS >s RHS.
3955	case ICmpInst::ICMP_SGT:
3956	case ICmpInst::ICMP_SGE:
3957	return ConstantExpr::getICmp(pred: ICmpInst::ICMP_SLT, LHS: C,
3958	RHS: Constant::getNullValue(Ty: C->getType()));
3959	case ICmpInst::ICMP_SLT:
3960	case ICmpInst::ICMP_SLE:
3961	return ConstantExpr::getICmp(pred: ICmpInst::ICMP_SGE, LHS: C,
3962	RHS: Constant::getNullValue(Ty: C->getType()));
3963
3964	// If LHS is non-negative then LHS <u RHS. If LHS is negative then
3965	// LHS >u RHS.
3966	case ICmpInst::ICMP_UGT:
3967	case ICmpInst::ICMP_UGE:
3968	// Comparison is true iff the LHS <s 0.
3969	if (MaxRecurse)
3970	if (Value *V = simplifyICmpInst(Predicate: ICmpInst::ICMP_SLT, LHS: SrcOp,
3971	RHS: Constant::getNullValue(Ty: SrcTy), Q,
3972	MaxRecurse: MaxRecurse - 1))
3973	return V;
3974	break;
3975	case ICmpInst::ICMP_ULT:
3976	case ICmpInst::ICMP_ULE:
3977	// Comparison is true iff the LHS >=s 0.
3978	if (MaxRecurse)
3979	if (Value *V = simplifyICmpInst(Predicate: ICmpInst::ICMP_SGE, LHS: SrcOp,
3980	RHS: Constant::getNullValue(Ty: SrcTy), Q,
3981	MaxRecurse: MaxRecurse - 1))
3982	return V;
3983	break;
3984	}
3985	}
3986	}
3987	}
3988	}
3989
3990	// icmp eq|ne X, Y -> false|true if X != Y
3991	// This is potentially expensive, and we have already computedKnownBits for
3992	// compares with 0 above here, so only try this for a non-zero compare.
3993	if (ICmpInst::isEquality(P: Pred) && !match(V: RHS, P: m_Zero()) &&
3994	isKnownNonEqual(V1: LHS, V2: RHS, DL: Q.DL, AC: Q.AC, CxtI: Q.CxtI, DT: Q.DT, UseInstrInfo: Q.IIQ.UseInstrInfo)) {
3995	return Pred == ICmpInst::ICMP_NE ? getTrue(Ty: ITy) : getFalse(Ty: ITy);
3996	}
3997
3998	if (Value *V = simplifyICmpWithBinOp(Pred, LHS, RHS, Q, MaxRecurse))
3999	return V;
4000
4001	if (Value *V = simplifyICmpWithMinMax(Pred, LHS, RHS, Q, MaxRecurse))
4002	return V;
4003
4004	if (Value *V = simplifyICmpWithIntrinsicOnLHS(Pred, LHS, RHS))
4005	return V;
4006	if (Value *V = simplifyICmpWithIntrinsicOnLHS(
4007	Pred: ICmpInst::getSwappedPredicate(pred: Pred), LHS: RHS, RHS: LHS))
4008	return V;
4009
4010	if (Value *V = simplifyICmpWithDominatingAssume(Predicate: Pred, LHS, RHS, Q))
4011	return V;
4012
4013	if (std::optional<bool> Res =
4014	isImpliedByDomCondition(Pred, LHS, RHS, ContextI: Q.CxtI, DL: Q.DL))
4015	return ConstantInt::getBool(Ty: ITy, V: *Res);
4016
4017	// Simplify comparisons of related pointers using a powerful, recursive
4018	// GEP-walk when we have target data available..
4019	if (LHS->getType()->isPointerTy())
4020	if (auto *C = computePointerICmp(Pred, LHS, RHS, Q))
4021	return C;
4022	if (auto *CLHS = dyn_cast<PtrToIntOperator>(Val: LHS))
4023	if (auto *CRHS = dyn_cast<PtrToIntOperator>(Val: RHS))
4024	if (CLHS->getPointerOperandType() == CRHS->getPointerOperandType() &&
4025	Q.DL.getTypeSizeInBits(Ty: CLHS->getPointerOperandType()) ==
4026	Q.DL.getTypeSizeInBits(Ty: CLHS->getType()))
4027	if (auto *C = computePointerICmp(Pred, LHS: CLHS->getPointerOperand(),
4028	RHS: CRHS->getPointerOperand(), Q))
4029	return C;
4030
4031	// If the comparison is with the result of a select instruction, check whether
4032	// comparing with either branch of the select always yields the same value.
4033	if (isa<SelectInst>(Val: LHS) || isa<SelectInst>(Val: RHS))
4034	if (Value *V = threadCmpOverSelect(Pred, LHS, RHS, Q, MaxRecurse))
4035	return V;
4036
4037	// If the comparison is with the result of a phi instruction, check whether
4038	// doing the compare with each incoming phi value yields a common result.
4039	if (isa<PHINode>(Val: LHS) || isa<PHINode>(Val: RHS))
4040	if (Value *V = threadCmpOverPHI(Pred, LHS, RHS, Q, MaxRecurse))
4041	return V;
4042
4043	return nullptr;
4044	}
4045
4046	Value *llvm::simplifyICmpInst(unsigned Predicate, Value *LHS, Value *RHS,
4047	const SimplifyQuery &Q) {
4048	return ::simplifyICmpInst(Predicate, LHS, RHS, Q, MaxRecurse: RecursionLimit);
4049	}
4050
4051	/// Given operands for an FCmpInst, see if we can fold the result.
4052	/// If not, this returns null.
4053	static Value *simplifyFCmpInst(unsigned Predicate, Value *LHS, Value *RHS,
4054	FastMathFlags FMF, const SimplifyQuery &Q,
4055	unsigned MaxRecurse) {
4056	CmpInst::Predicate Pred = (CmpInst::Predicate)Predicate;
4057	assert(CmpInst::isFPPredicate(Pred) && "Not an FP compare!");
4058
4059	if (Constant *CLHS = dyn_cast<Constant>(Val: LHS)) {
4060	if (Constant *CRHS = dyn_cast<Constant>(Val: RHS))
4061	return ConstantFoldCompareInstOperands(Predicate: Pred, LHS: CLHS, RHS: CRHS, DL: Q.DL, TLI: Q.TLI,
4062	I: Q.CxtI);
4063
4064	// If we have a constant, make sure it is on the RHS.
4065	std::swap(a&: LHS, b&: RHS);
4066	Pred = CmpInst::getSwappedPredicate(pred: Pred);
4067	}
4068
4069	// Fold trivial predicates.
4070	Type *RetTy = getCompareTy(Op: LHS);
4071	if (Pred == FCmpInst::FCMP_FALSE)
4072	return getFalse(Ty: RetTy);
4073	if (Pred == FCmpInst::FCMP_TRUE)
4074	return getTrue(Ty: RetTy);
4075
4076	// fcmp pred x, poison and fcmp pred poison, x
4077	// fold to poison
4078	if (isa<PoisonValue>(Val: LHS) || isa<PoisonValue>(Val: RHS))
4079	return PoisonValue::get(T: RetTy);
4080
4081	// fcmp pred x, undef and fcmp pred undef, x
4082	// fold to true if unordered, false if ordered
4083	if (Q.isUndefValue(V: LHS) || Q.isUndefValue(V: RHS)) {
4084	// Choosing NaN for the undef will always make unordered comparison succeed
4085	// and ordered comparison fail.
4086	return ConstantInt::get(Ty: RetTy, V: CmpInst::isUnordered(predicate: Pred));
4087	}
4088
4089	// fcmp x,x -> true/false. Not all compares are foldable.
4090	if (LHS == RHS) {
4091	if (CmpInst::isTrueWhenEqual(predicate: Pred))
4092	return getTrue(Ty: RetTy);
4093	if (CmpInst::isFalseWhenEqual(predicate: Pred))
4094	return getFalse(Ty: RetTy);
4095	}
4096
4097	// Fold (un)ordered comparison if we can determine there are no NaNs.
4098	//
4099	// This catches the 2 variable input case, constants are handled below as a
4100	// class-like compare.
4101	if (Pred == FCmpInst::FCMP_ORD || Pred == FCmpInst::FCMP_UNO) {
4102	if (FMF.noNaNs() || (isKnownNeverNaN(V: RHS, /*Depth=*/0, SQ: Q) &&
4103	isKnownNeverNaN(V: LHS, /*Depth=*/0, SQ: Q)))
4104	return ConstantInt::get(Ty: RetTy, V: Pred == FCmpInst::FCMP_ORD);
4105	}
4106
4107	const APFloat *C = nullptr;
4108	match(V: RHS, P: m_APFloatAllowPoison(Res&: C));
4109	std::optional<KnownFPClass> FullKnownClassLHS;
4110
4111	// Lazily compute the possible classes for LHS. Avoid computing it twice if
4112	// RHS is a 0.
4113	auto computeLHSClass = [=, &FullKnownClassLHS](FPClassTest InterestedFlags =
4114	fcAllFlags) {
4115	if (FullKnownClassLHS)
4116	return *FullKnownClassLHS;
4117	return computeKnownFPClass(V: LHS, FMF, InterestedClasses: InterestedFlags, Depth: 0, SQ: Q);
4118	};
4119
4120	if (C && Q.CxtI) {
4121	// Fold out compares that express a class test.
4122	//
4123	// FIXME: Should be able to perform folds without context
4124	// instruction. Always pass in the context function?
4125
4126	const Function *ParentF = Q.CxtI->getFunction();
4127	auto [ClassVal, ClassTest] = fcmpToClassTest(Pred, F: *ParentF, LHS, ConstRHS: C);
4128	if (ClassVal) {
4129	FullKnownClassLHS = computeLHSClass();
4130	if ((FullKnownClassLHS->KnownFPClasses & ClassTest) == fcNone)
4131	return getFalse(Ty: RetTy);
4132	if ((FullKnownClassLHS->KnownFPClasses & ~ClassTest) == fcNone)
4133	return getTrue(Ty: RetTy);
4134	}
4135	}
4136
4137	// Handle fcmp with constant RHS.
4138	if (C) {
4139	// TODO: If we always required a context function, we wouldn't need to
4140	// special case nans.
4141	if (C->isNaN())
4142	return ConstantInt::get(Ty: RetTy, V: CmpInst::isUnordered(predicate: Pred));
4143
4144	// TODO: Need version fcmpToClassTest which returns implied class when the
4145	// compare isn't a complete class test. e.g. > 1.0 implies fcPositive, but
4146	// isn't implementable as a class call.
4147	if (C->isNegative() && !C->isNegZero()) {
4148	FPClassTest Interested = KnownFPClass::OrderedLessThanZeroMask;
4149
4150	// TODO: We can catch more cases by using a range check rather than
4151	// relying on CannotBeOrderedLessThanZero.
4152	switch (Pred) {
4153	case FCmpInst::FCMP_UGE:
4154	case FCmpInst::FCMP_UGT:
4155	case FCmpInst::FCMP_UNE: {
4156	KnownFPClass KnownClass = computeLHSClass(Interested);
4157
4158	// (X >= 0) implies (X > C) when (C < 0)
4159	if (KnownClass.cannotBeOrderedLessThanZero())
4160	return getTrue(Ty: RetTy);
4161	break;
4162	}
4163	case FCmpInst::FCMP_OEQ:
4164	case FCmpInst::FCMP_OLE:
4165	case FCmpInst::FCMP_OLT: {
4166	KnownFPClass KnownClass = computeLHSClass(Interested);
4167
4168	// (X >= 0) implies !(X < C) when (C < 0)
4169	if (KnownClass.cannotBeOrderedLessThanZero())
4170	return getFalse(Ty: RetTy);
4171	break;
4172	}
4173	default:
4174	break;
4175	}
4176	}
4177	// Check comparison of [minnum/maxnum with constant] with other constant.
4178	const APFloat *C2;
4179	if ((match(LHS, m_Intrinsic<Intrinsic::minnum>(m_Value(), m_APFloat(Res&: C2))) &&
4180	*C2 < *C) ||
4181	(match(LHS, m_Intrinsic<Intrinsic::maxnum>(m_Value(), m_APFloat(Res&: C2))) &&
4182	*C2 > *C)) {
4183	bool IsMaxNum =
4184	cast<IntrinsicInst>(Val: LHS)->getIntrinsicID() == Intrinsic::maxnum;
4185	// The ordered relationship and minnum/maxnum guarantee that we do not
4186	// have NaN constants, so ordered/unordered preds are handled the same.
4187	switch (Pred) {
4188	case FCmpInst::FCMP_OEQ:
4189	case FCmpInst::FCMP_UEQ:
4190	// minnum(X, LesserC) == C --> false
4191	// maxnum(X, GreaterC) == C --> false
4192	return getFalse(Ty: RetTy);
4193	case FCmpInst::FCMP_ONE:
4194	case FCmpInst::FCMP_UNE:
4195	// minnum(X, LesserC) != C --> true
4196	// maxnum(X, GreaterC) != C --> true
4197	return getTrue(Ty: RetTy);
4198	case FCmpInst::FCMP_OGE:
4199	case FCmpInst::FCMP_UGE:
4200	case FCmpInst::FCMP_OGT:
4201	case FCmpInst::FCMP_UGT:
4202	// minnum(X, LesserC) >= C --> false
4203	// minnum(X, LesserC) > C --> false
4204	// maxnum(X, GreaterC) >= C --> true
4205	// maxnum(X, GreaterC) > C --> true
4206	return ConstantInt::get(Ty: RetTy, V: IsMaxNum);
4207	case FCmpInst::FCMP_OLE:
4208	case FCmpInst::FCMP_ULE:
4209	case FCmpInst::FCMP_OLT:
4210	case FCmpInst::FCMP_ULT:
4211	// minnum(X, LesserC) <= C --> true
4212	// minnum(X, LesserC) < C --> true
4213	// maxnum(X, GreaterC) <= C --> false
4214	// maxnum(X, GreaterC) < C --> false
4215	return ConstantInt::get(Ty: RetTy, V: !IsMaxNum);
4216	default:
4217	// TRUE/FALSE/ORD/UNO should be handled before this.
4218	llvm_unreachable("Unexpected fcmp predicate");
4219	}
4220	}
4221	}
4222
4223	// TODO: Could fold this with above if there were a matcher which returned all
4224	// classes in a non-splat vector.
4225	if (match(V: RHS, P: m_AnyZeroFP())) {
4226	switch (Pred) {
4227	case FCmpInst::FCMP_OGE:
4228	case FCmpInst::FCMP_ULT: {
4229	FPClassTest Interested = KnownFPClass::OrderedLessThanZeroMask;
4230	if (!FMF.noNaNs())
4231	Interested |= fcNan;
4232
4233	KnownFPClass Known = computeLHSClass(Interested);
4234
4235	// Positive or zero X >= 0.0 --> true
4236	// Positive or zero X < 0.0 --> false
4237	if ((FMF.noNaNs() || Known.isKnownNeverNaN()) &&
4238	Known.cannotBeOrderedLessThanZero())
4239	return Pred == FCmpInst::FCMP_OGE ? getTrue(Ty: RetTy) : getFalse(Ty: RetTy);
4240	break;
4241	}
4242	case FCmpInst::FCMP_UGE:
4243	case FCmpInst::FCMP_OLT: {
4244	FPClassTest Interested = KnownFPClass::OrderedLessThanZeroMask;
4245	KnownFPClass Known = computeLHSClass(Interested);
4246
4247	// Positive or zero or nan X >= 0.0 --> true
4248	// Positive or zero or nan X < 0.0 --> false
4249	if (Known.cannotBeOrderedLessThanZero())
4250	return Pred == FCmpInst::FCMP_UGE ? getTrue(Ty: RetTy) : getFalse(Ty: RetTy);
4251	break;
4252	}
4253	default:
4254	break;
4255	}
4256	}
4257
4258	// If the comparison is with the result of a select instruction, check whether
4259	// comparing with either branch of the select always yields the same value.
4260	if (isa<SelectInst>(Val: LHS) || isa<SelectInst>(Val: RHS))
4261	if (Value *V = threadCmpOverSelect(Pred, LHS, RHS, Q, MaxRecurse))
4262	return V;
4263
4264	// If the comparison is with the result of a phi instruction, check whether
4265	// doing the compare with each incoming phi value yields a common result.
4266	if (isa<PHINode>(Val: LHS) || isa<PHINode>(Val: RHS))
4267	if (Value *V = threadCmpOverPHI(Pred, LHS, RHS, Q, MaxRecurse))
4268	return V;
4269
4270	return nullptr;
4271	}
4272
4273	Value *llvm::simplifyFCmpInst(unsigned Predicate, Value *LHS, Value *RHS,
4274	FastMathFlags FMF, const SimplifyQuery &Q) {
4275	return ::simplifyFCmpInst(Predicate, LHS, RHS, FMF, Q, MaxRecurse: RecursionLimit);
4276	}
4277
4278	static Value *simplifyWithOpReplaced(Value *V, Value *Op, Value *RepOp,
4279	const SimplifyQuery &Q,
4280	bool AllowRefinement,
4281	SmallVectorImpl<Instruction *> *DropFlags,
4282	unsigned MaxRecurse) {
4283	// Trivial replacement.
4284	if (V == Op)
4285	return RepOp;
4286
4287	if (!MaxRecurse--)
4288	return nullptr;
4289
4290	// We cannot replace a constant, and shouldn't even try.
4291	if (isa<Constant>(Val: Op))
4292	return nullptr;
4293
4294	auto *I = dyn_cast<Instruction>(Val: V);
4295	if (!I)
4296	return nullptr;
4297
4298	// The arguments of a phi node might refer to a value from a previous
4299	// cycle iteration.
4300	if (isa<PHINode>(Val: I))
4301	return nullptr;
4302
4303	if (Op->getType()->isVectorTy()) {
4304	// For vector types, the simplification must hold per-lane, so forbid
4305	// potentially cross-lane operations like shufflevector.
4306	if (!I->getType()->isVectorTy() || isa<ShuffleVectorInst>(Val: I) ||
4307	isa<CallBase>(Val: I) || isa<BitCastInst>(Val: I))
4308	return nullptr;
4309	}
4310
4311	// Don't fold away llvm.is.constant checks based on assumptions.
4312	if (match(I, m_Intrinsic<Intrinsic::is_constant>()))
4313	return nullptr;
4314
4315	// Replace Op with RepOp in instruction operands.
4316	SmallVector<Value *, 8> NewOps;
4317	bool AnyReplaced = false;
4318	for (Value *InstOp : I->operands()) {
4319	if (Value *NewInstOp = simplifyWithOpReplaced(
4320	V: InstOp, Op, RepOp, Q, AllowRefinement, DropFlags, MaxRecurse)) {
4321	NewOps.push_back(Elt: NewInstOp);
4322	AnyReplaced = InstOp != NewInstOp;
4323	} else {
4324	NewOps.push_back(Elt: InstOp);
4325	}
4326	}
4327
4328	if (!AnyReplaced)
4329	return nullptr;
4330
4331	if (!AllowRefinement) {
4332	// General InstSimplify functions may refine the result, e.g. by returning
4333	// a constant for a potentially poison value. To avoid this, implement only
4334	// a few non-refining but profitable transforms here.
4335
4336	if (auto *BO = dyn_cast<BinaryOperator>(Val: I)) {
4337	unsigned Opcode = BO->getOpcode();
4338	// id op x -> x, x op id -> x
4339	if (NewOps[0] == ConstantExpr::getBinOpIdentity(Opcode, Ty: I->getType()))
4340	return NewOps[1];
4341	if (NewOps[1] == ConstantExpr::getBinOpIdentity(Opcode, Ty: I->getType(),
4342	/* RHS */ AllowRHSConstant: true))
4343	return NewOps[0];
4344
4345	// x & x -> x, x | x -> x
4346	if ((Opcode == Instruction::And || Opcode == Instruction::Or) &&
4347	NewOps[0] == NewOps[1]) {
4348	// or disjoint x, x results in poison.
4349	if (auto *PDI = dyn_cast<PossiblyDisjointInst>(Val: BO)) {
4350	if (PDI->isDisjoint()) {
4351	if (!DropFlags)
4352	return nullptr;
4353	DropFlags->push_back(Elt: BO);
4354	}
4355	}
4356	return NewOps[0];
4357	}
4358
4359	// x - x -> 0, x ^ x -> 0. This is non-refining, because x is non-poison
4360	// by assumption and this case never wraps, so nowrap flags can be
4361	// ignored.
4362	if ((Opcode == Instruction::Sub || Opcode == Instruction::Xor) &&
4363	NewOps[0] == RepOp && NewOps[1] == RepOp)
4364	return Constant::getNullValue(Ty: I->getType());
4365
4366	// If we are substituting an absorber constant into a binop and extra
4367	// poison can't leak if we remove the select -- because both operands of
4368	// the binop are based on the same value -- then it may be safe to replace
4369	// the value with the absorber constant. Examples:
4370	// (Op == 0) ? 0 : (Op & -Op) --> Op & -Op
4371	// (Op == 0) ? 0 : (Op * (binop Op, C)) --> Op * (binop Op, C)
4372	// (Op == -1) ? -1 : (Op | (binop C, Op) --> Op | (binop C, Op)
4373	Constant *Absorber =
4374	ConstantExpr::getBinOpAbsorber(Opcode, Ty: I->getType());
4375	if ((NewOps[0] == Absorber || NewOps[1] == Absorber) &&
4376	impliesPoison(ValAssumedPoison: BO, V: Op))
4377	return Absorber;
4378	}
4379
4380	if (isa<GetElementPtrInst>(Val: I)) {
4381	// getelementptr x, 0 -> x.
4382	// This never returns poison, even if inbounds is set.
4383	if (NewOps.size() == 2 && match(V: NewOps[1], P: m_Zero()))
4384	return NewOps[0];
4385	}
4386	} else {
4387	// The simplification queries below may return the original value. Consider:
4388	// %div = udiv i32 %arg, %arg2
4389	// %mul = mul nsw i32 %div, %arg2
4390	// %cmp = icmp eq i32 %mul, %arg
4391	// %sel = select i1 %cmp, i32 %div, i32 undef
4392	// Replacing %arg by %mul, %div becomes "udiv i32 %mul, %arg2", which
4393	// simplifies back to %arg. This can only happen because %mul does not
4394	// dominate %div. To ensure a consistent return value contract, we make sure
4395	// that this case returns nullptr as well.
4396	auto PreventSelfSimplify = [V](Value *Simplified) {
4397	return Simplified != V ? Simplified : nullptr;
4398	};
4399
4400	return PreventSelfSimplify(
4401	::simplifyInstructionWithOperands(I, NewOps, SQ: Q, MaxRecurse));
4402	}
4403
4404	// If all operands are constant after substituting Op for RepOp then we can
4405	// constant fold the instruction.
4406	SmallVector<Constant *, 8> ConstOps;
4407	for (Value *NewOp : NewOps) {
4408	if (Constant *ConstOp = dyn_cast<Constant>(Val: NewOp))
4409	ConstOps.push_back(Elt: ConstOp);
4410	else
4411	return nullptr;
4412	}
4413
4414	// Consider:
4415	// %cmp = icmp eq i32 %x, 2147483647
4416	// %add = add nsw i32 %x, 1
4417	// %sel = select i1 %cmp, i32 -2147483648, i32 %add
4418	//
4419	// We can't replace %sel with %add unless we strip away the flags (which
4420	// will be done in InstCombine).
4421	// TODO: This may be unsound, because it only catches some forms of
4422	// refinement.
4423	if (!AllowRefinement) {
4424	if (canCreatePoison(Op: cast<Operator>(Val: I), ConsiderFlagsAndMetadata: !DropFlags)) {
4425	// abs cannot create poison if the value is known to never be int_min.
4426	if (auto *II = dyn_cast<IntrinsicInst>(Val: I);
4427	II && II->getIntrinsicID() == Intrinsic::abs) {
4428	if (!ConstOps[0]->isNotMinSignedValue())
4429	return nullptr;
4430	} else
4431	return nullptr;
4432	}
4433	Constant *Res = ConstantFoldInstOperands(I, Ops: ConstOps, DL: Q.DL, TLI: Q.TLI);
4434	if (DropFlags && Res && I->hasPoisonGeneratingAnnotations())
4435	DropFlags->push_back(Elt: I);
4436	return Res;
4437	}
4438
4439	return ConstantFoldInstOperands(I, Ops: ConstOps, DL: Q.DL, TLI: Q.TLI);
4440	}
4441
4442	Value *llvm::simplifyWithOpReplaced(Value *V, Value *Op, Value *RepOp,
4443	const SimplifyQuery &Q,
4444	bool AllowRefinement,
4445	SmallVectorImpl<Instruction *> *DropFlags) {
4446	return ::simplifyWithOpReplaced(V, Op, RepOp, Q, AllowRefinement, DropFlags,
4447	MaxRecurse: RecursionLimit);
4448	}
4449
4450	/// Try to simplify a select instruction when its condition operand is an
4451	/// integer comparison where one operand of the compare is a constant.
4452	static Value *simplifySelectBitTest(Value *TrueVal, Value *FalseVal, Value *X,
4453	const APInt *Y, bool TrueWhenUnset) {
4454	const APInt *C;
4455
4456	// (X & Y) == 0 ? X & ~Y : X --> X
4457	// (X & Y) != 0 ? X & ~Y : X --> X & ~Y
4458	if (FalseVal == X && match(V: TrueVal, P: m_And(L: m_Specific(V: X), R: m_APInt(Res&: C))) &&
4459	*Y == ~*C)
4460	return TrueWhenUnset ? FalseVal : TrueVal;
4461
4462	// (X & Y) == 0 ? X : X & ~Y --> X & ~Y
4463	// (X & Y) != 0 ? X : X & ~Y --> X
4464	if (TrueVal == X && match(V: FalseVal, P: m_And(L: m_Specific(V: X), R: m_APInt(Res&: C))) &&
4465	*Y == ~*C)
4466	return TrueWhenUnset ? FalseVal : TrueVal;
4467
4468	if (Y->isPowerOf2()) {
4469	// (X & Y) == 0 ? X | Y : X --> X | Y
4470	// (X & Y) != 0 ? X | Y : X --> X
4471	if (FalseVal == X && match(V: TrueVal, P: m_Or(L: m_Specific(V: X), R: m_APInt(Res&: C))) &&
4472	*Y == *C) {
4473	// We can't return the or if it has the disjoint flag.
4474	if (TrueWhenUnset && cast<PossiblyDisjointInst>(Val: TrueVal)->isDisjoint())
4475	return nullptr;
4476	return TrueWhenUnset ? TrueVal : FalseVal;
4477	}
4478
4479	// (X & Y) == 0 ? X : X | Y --> X
4480	// (X & Y) != 0 ? X : X | Y --> X | Y
4481	if (TrueVal == X && match(V: FalseVal, P: m_Or(L: m_Specific(V: X), R: m_APInt(Res&: C))) &&
4482	*Y == *C) {
4483	// We can't return the or if it has the disjoint flag.
4484	if (!TrueWhenUnset && cast<PossiblyDisjointInst>(Val: FalseVal)->isDisjoint())
4485	return nullptr;
4486	return TrueWhenUnset ? TrueVal : FalseVal;
4487	}
4488	}
4489
4490	return nullptr;
4491	}
4492
4493	static Value *simplifyCmpSelOfMaxMin(Value *CmpLHS, Value *CmpRHS,
4494	ICmpInst::Predicate Pred, Value *TVal,
4495	Value *FVal) {
4496	// Canonicalize common cmp+sel operand as CmpLHS.
4497	if (CmpRHS == TVal || CmpRHS == FVal) {
4498	std::swap(a&: CmpLHS, b&: CmpRHS);
4499	Pred = ICmpInst::getSwappedPredicate(pred: Pred);
4500	}
4501
4502	// Canonicalize common cmp+sel operand as TVal.
4503	if (CmpLHS == FVal) {
4504	std::swap(a&: TVal, b&: FVal);
4505	Pred = ICmpInst::getInversePredicate(pred: Pred);
4506	}
4507
4508	// A vector select may be shuffling together elements that are equivalent
4509	// based on the max/min/select relationship.
4510	Value *X = CmpLHS, *Y = CmpRHS;
4511	bool PeekedThroughSelectShuffle = false;
4512	auto *Shuf = dyn_cast<ShuffleVectorInst>(Val: FVal);
4513	if (Shuf && Shuf->isSelect()) {
4514	if (Shuf->getOperand(i_nocapture: 0) == Y)
4515	FVal = Shuf->getOperand(i_nocapture: 1);
4516	else if (Shuf->getOperand(i_nocapture: 1) == Y)
4517	FVal = Shuf->getOperand(i_nocapture: 0);
4518	else
4519	return nullptr;
4520	PeekedThroughSelectShuffle = true;
4521	}
4522
4523	// (X pred Y) ? X : max/min(X, Y)
4524	auto *MMI = dyn_cast<MinMaxIntrinsic>(Val: FVal);
4525	if (!MMI || TVal != X ||
4526	!match(V: FVal, P: m_c_MaxOrMin(L: m_Specific(V: X), R: m_Specific(V: Y))))
4527	return nullptr;
4528
4529	// (X > Y) ? X : max(X, Y) --> max(X, Y)
4530	// (X >= Y) ? X : max(X, Y) --> max(X, Y)
4531	// (X < Y) ? X : min(X, Y) --> min(X, Y)
4532	// (X <= Y) ? X : min(X, Y) --> min(X, Y)
4533	//
4534	// The equivalence allows a vector select (shuffle) of max/min and Y. Ex:
4535	// (X > Y) ? X : (Z ? max(X, Y) : Y)
4536	// If Z is true, this reduces as above, and if Z is false:
4537	// (X > Y) ? X : Y --> max(X, Y)
4538	ICmpInst::Predicate MMPred = MMI->getPredicate();
4539	if (MMPred == CmpInst::getStrictPredicate(pred: Pred))
4540	return MMI;
4541
4542	// Other transforms are not valid with a shuffle.
4543	if (PeekedThroughSelectShuffle)
4544	return nullptr;
4545
4546	// (X == Y) ? X : max/min(X, Y) --> max/min(X, Y)
4547	if (Pred == CmpInst::ICMP_EQ)
4548	return MMI;
4549
4550	// (X != Y) ? X : max/min(X, Y) --> X
4551	if (Pred == CmpInst::ICMP_NE)
4552	return X;
4553
4554	// (X < Y) ? X : max(X, Y) --> X
4555	// (X <= Y) ? X : max(X, Y) --> X
4556	// (X > Y) ? X : min(X, Y) --> X
4557	// (X >= Y) ? X : min(X, Y) --> X
4558	ICmpInst::Predicate InvPred = CmpInst::getInversePredicate(pred: Pred);
4559	if (MMPred == CmpInst::getStrictPredicate(pred: InvPred))
4560	return X;
4561
4562	return nullptr;
4563	}
4564
4565	/// An alternative way to test if a bit is set or not uses sgt/slt instead of
4566	/// eq/ne.
4567	static Value *simplifySelectWithFakeICmpEq(Value *CmpLHS, Value *CmpRHS,
4568	ICmpInst::Predicate Pred,
4569	Value *TrueVal, Value *FalseVal) {
4570	Value *X;
4571	APInt Mask;
4572	if (!decomposeBitTestICmp(LHS: CmpLHS, RHS: CmpRHS, Pred, X, Mask))
4573	return nullptr;
4574
4575	return simplifySelectBitTest(TrueVal, FalseVal, X, Y: &Mask,
4576	TrueWhenUnset: Pred == ICmpInst::ICMP_EQ);
4577	}
4578
4579	/// Try to simplify a select instruction when its condition operand is an
4580	/// integer equality comparison.
4581	static Value *simplifySelectWithICmpEq(Value *CmpLHS, Value *CmpRHS,
4582	Value *TrueVal, Value *FalseVal,
4583	const SimplifyQuery &Q,
4584	unsigned MaxRecurse) {
4585	if (simplifyWithOpReplaced(V: FalseVal, Op: CmpLHS, RepOp: CmpRHS, Q,
4586	/* AllowRefinement */ false,
4587	/* DropFlags */ nullptr, MaxRecurse) == TrueVal)
4588	return FalseVal;
4589	if (simplifyWithOpReplaced(V: TrueVal, Op: CmpLHS, RepOp: CmpRHS, Q,
4590	/* AllowRefinement */ true,
4591	/* DropFlags */ nullptr, MaxRecurse) == FalseVal)
4592	return FalseVal;
4593
4594	return nullptr;
4595	}
4596
4597	/// Try to simplify a select instruction when its condition operand is an
4598	/// integer comparison.
4599	static Value *simplifySelectWithICmpCond(Value *CondVal, Value *TrueVal,
4600	Value *FalseVal,
4601	const SimplifyQuery &Q,
4602	unsigned MaxRecurse) {
4603	ICmpInst::Predicate Pred;
4604	Value *CmpLHS, *CmpRHS;
4605	if (!match(V: CondVal, P: m_ICmp(Pred, L: m_Value(V&: CmpLHS), R: m_Value(V&: CmpRHS))))
4606	return nullptr;
4607
4608	if (Value *V = simplifyCmpSelOfMaxMin(CmpLHS, CmpRHS, Pred, TVal: TrueVal, FVal: FalseVal))
4609	return V;
4610
4611	// Canonicalize ne to eq predicate.
4612	if (Pred == ICmpInst::ICMP_NE) {
4613	Pred = ICmpInst::ICMP_EQ;
4614	std::swap(a&: TrueVal, b&: FalseVal);
4615	}
4616
4617	// Check for integer min/max with a limit constant:
4618	// X > MIN_INT ? X : MIN_INT --> X
4619	// X < MAX_INT ? X : MAX_INT --> X
4620	if (TrueVal->getType()->isIntOrIntVectorTy()) {
4621	Value *X, *Y;
4622	SelectPatternFlavor SPF =
4623	matchDecomposedSelectPattern(CmpI: cast<ICmpInst>(Val: CondVal), TrueVal, FalseVal,
4624	LHS&: X, RHS&: Y)
4625	.Flavor;
4626	if (SelectPatternResult::isMinOrMax(SPF) && Pred == getMinMaxPred(SPF)) {
4627	APInt LimitC = getMinMaxLimit(SPF: getInverseMinMaxFlavor(SPF),
4628	BitWidth: X->getType()->getScalarSizeInBits());
4629	if (match(V: Y, P: m_SpecificInt(V: LimitC)))
4630	return X;
4631	}
4632	}
4633
4634	if (Pred == ICmpInst::ICMP_EQ && match(V: CmpRHS, P: m_Zero())) {
4635	Value *X;
4636	const APInt *Y;
4637	if (match(V: CmpLHS, P: m_And(L: m_Value(V&: X), R: m_APInt(Res&: Y))))
4638	if (Value *V = simplifySelectBitTest(TrueVal, FalseVal, X, Y,
4639	/*TrueWhenUnset=*/true))
4640	return V;
4641
4642	// Test for a bogus zero-shift-guard-op around funnel-shift or rotate.
4643	Value *ShAmt;
4644	auto isFsh = m_CombineOr(L: m_FShl(Op0: m_Value(V&: X), Op1: m_Value(), Op2: m_Value(V&: ShAmt)),
4645	R: m_FShr(Op0: m_Value(), Op1: m_Value(V&: X), Op2: m_Value(V&: ShAmt)));
4646	// (ShAmt == 0) ? fshl(X, *, ShAmt) : X --> X
4647	// (ShAmt == 0) ? fshr(*, X, ShAmt) : X --> X
4648	if (match(V: TrueVal, P: isFsh) && FalseVal == X && CmpLHS == ShAmt)
4649	return X;
4650
4651	// Test for a zero-shift-guard-op around rotates. These are used to
4652	// avoid UB from oversized shifts in raw IR rotate patterns, but the
4653	// intrinsics do not have that problem.
4654	// We do not allow this transform for the general funnel shift case because
4655	// that would not preserve the poison safety of the original code.
4656	auto isRotate =
4657	m_CombineOr(L: m_FShl(Op0: m_Value(V&: X), Op1: m_Deferred(V: X), Op2: m_Value(V&: ShAmt)),
4658	R: m_FShr(Op0: m_Value(V&: X), Op1: m_Deferred(V: X), Op2: m_Value(V&: ShAmt)));
4659	// (ShAmt == 0) ? X : fshl(X, X, ShAmt) --> fshl(X, X, ShAmt)
4660	// (ShAmt == 0) ? X : fshr(X, X, ShAmt) --> fshr(X, X, ShAmt)
4661	if (match(V: FalseVal, P: isRotate) && TrueVal == X && CmpLHS == ShAmt &&
4662	Pred == ICmpInst::ICMP_EQ)
4663	return FalseVal;
4664
4665	// X == 0 ? abs(X) : -abs(X) --> -abs(X)
4666	// X == 0 ? -abs(X) : abs(X) --> abs(X)
4667	if (match(TrueVal, m_Intrinsic<Intrinsic::abs>(m_Specific(V: CmpLHS))) &&
4668	match(FalseVal, m_Neg(m_Intrinsic<Intrinsic::abs>(m_Specific(V: CmpLHS)))))
4669	return FalseVal;
4670	if (match(TrueVal,
4671	m_Neg(m_Intrinsic<Intrinsic::abs>(m_Specific(V: CmpLHS)))) &&
4672	match(FalseVal, m_Intrinsic<Intrinsic::abs>(m_Specific(V: CmpLHS))))
4673	return FalseVal;
4674	}
4675
4676	// Check for other compares that behave like bit test.
4677	if (Value *V =
4678	simplifySelectWithFakeICmpEq(CmpLHS, CmpRHS, Pred, TrueVal, FalseVal))
4679	return V;
4680
4681	// If we have a scalar equality comparison, then we know the value in one of
4682	// the arms of the select. See if substituting this value into the arm and
4683	// simplifying the result yields the same value as the other arm.
4684	if (Pred == ICmpInst::ICMP_EQ) {
4685	if (Value *V = simplifySelectWithICmpEq(CmpLHS, CmpRHS, TrueVal, FalseVal,
4686	Q, MaxRecurse))
4687	return V;
4688	if (Value *V = simplifySelectWithICmpEq(CmpLHS: CmpRHS, CmpRHS: CmpLHS, TrueVal, FalseVal,
4689	Q, MaxRecurse))
4690	return V;
4691
4692	Value *X;
4693	Value *Y;
4694	// select((X | Y) == 0 ? X : 0) --> 0 (commuted 2 ways)
4695	if (match(V: CmpLHS, P: m_Or(L: m_Value(V&: X), R: m_Value(V&: Y))) &&
4696	match(V: CmpRHS, P: m_Zero())) {
4697	// (X | Y) == 0 implies X == 0 and Y == 0.
4698	if (Value *V = simplifySelectWithICmpEq(CmpLHS: X, CmpRHS, TrueVal, FalseVal, Q,
4699	MaxRecurse))
4700	return V;
4701	if (Value *V = simplifySelectWithICmpEq(CmpLHS: Y, CmpRHS, TrueVal, FalseVal, Q,
4702	MaxRecurse))
4703	return V;
4704	}
4705
4706	// select((X & Y) == -1 ? X : -1) --> -1 (commuted 2 ways)
4707	if (match(V: CmpLHS, P: m_And(L: m_Value(V&: X), R: m_Value(V&: Y))) &&
4708	match(V: CmpRHS, P: m_AllOnes())) {
4709	// (X & Y) == -1 implies X == -1 and Y == -1.
4710	if (Value *V = simplifySelectWithICmpEq(CmpLHS: X, CmpRHS, TrueVal, FalseVal, Q,
4711	MaxRecurse))
4712	return V;
4713	if (Value *V = simplifySelectWithICmpEq(CmpLHS: Y, CmpRHS, TrueVal, FalseVal, Q,
4714	MaxRecurse))
4715	return V;
4716	}
4717	}
4718
4719	return nullptr;
4720	}
4721
4722	/// Try to simplify a select instruction when its condition operand is a
4723	/// floating-point comparison.
4724	static Value *simplifySelectWithFCmp(Value *Cond, Value *T, Value *F,
4725	const SimplifyQuery &Q) {
4726	FCmpInst::Predicate Pred;
4727	if (!match(V: Cond, P: m_FCmp(Pred, L: m_Specific(V: T), R: m_Specific(V: F))) &&
4728	!match(V: Cond, P: m_FCmp(Pred, L: m_Specific(V: F), R: m_Specific(V: T))))
4729	return nullptr;
4730
4731	// This transform is safe if we do not have (do not care about) -0.0 or if
4732	// at least one operand is known to not be -0.0. Otherwise, the select can
4733	// change the sign of a zero operand.
4734	bool HasNoSignedZeros =
4735	Q.CxtI && isa<FPMathOperator>(Val: Q.CxtI) && Q.CxtI->hasNoSignedZeros();
4736	const APFloat *C;
4737	if (HasNoSignedZeros || (match(V: T, P: m_APFloat(Res&: C)) && C->isNonZero()) ||
4738	(match(V: F, P: m_APFloat(Res&: C)) && C->isNonZero())) {
4739	// (T == F) ? T : F --> F
4740	// (F == T) ? T : F --> F
4741	if (Pred == FCmpInst::FCMP_OEQ)
4742	return F;
4743
4744	// (T != F) ? T : F --> T
4745	// (F != T) ? T : F --> T
4746	if (Pred == FCmpInst::FCMP_UNE)
4747	return T;
4748	}
4749
4750	return nullptr;
4751	}
4752
4753	/// Given operands for a SelectInst, see if we can fold the result.
4754	/// If not, this returns null.
4755	static Value *simplifySelectInst(Value *Cond, Value *TrueVal, Value *FalseVal,
4756	const SimplifyQuery &Q, unsigned MaxRecurse) {
4757	if (auto *CondC = dyn_cast<Constant>(Val: Cond)) {
4758	if (auto *TrueC = dyn_cast<Constant>(Val: TrueVal))
4759	if (auto *FalseC = dyn_cast<Constant>(Val: FalseVal))
4760	if (Constant *C = ConstantFoldSelectInstruction(Cond: CondC, V1: TrueC, V2: FalseC))
4761	return C;
4762
4763	// select poison, X, Y -> poison
4764	if (isa<PoisonValue>(Val: CondC))
4765	return PoisonValue::get(T: TrueVal->getType());
4766
4767	// select undef, X, Y -> X or Y
4768	if (Q.isUndefValue(V: CondC))
4769	return isa<Constant>(Val: FalseVal) ? FalseVal : TrueVal;
4770
4771	// select true, X, Y --> X
4772	// select false, X, Y --> Y
4773	// For vectors, allow undef/poison elements in the condition to match the
4774	// defined elements, so we can eliminate the select.
4775	if (match(V: CondC, P: m_One()))
4776	return TrueVal;
4777	if (match(V: CondC, P: m_Zero()))
4778	return FalseVal;
4779	}
4780
4781	assert(Cond->getType()->isIntOrIntVectorTy(1) &&
4782	"Select must have bool or bool vector condition");
4783	assert(TrueVal->getType() == FalseVal->getType() &&
4784	"Select must have same types for true/false ops");
4785
4786	if (Cond->getType() == TrueVal->getType()) {
4787	// select i1 Cond, i1 true, i1 false --> i1 Cond
4788	if (match(V: TrueVal, P: m_One()) && match(V: FalseVal, P: m_ZeroInt()))
4789	return Cond;
4790
4791	// (X && Y) ? X : Y --> Y (commuted 2 ways)
4792	if (match(V: Cond, P: m_c_LogicalAnd(L: m_Specific(V: TrueVal), R: m_Specific(V: FalseVal))))
4793	return FalseVal;
4794
4795	// (X || Y) ? X : Y --> X (commuted 2 ways)
4796	if (match(V: Cond, P: m_c_LogicalOr(L: m_Specific(V: TrueVal), R: m_Specific(V: FalseVal))))
4797	return TrueVal;
4798
4799	// (X || Y) ? false : X --> false (commuted 2 ways)
4800	if (match(V: Cond, P: m_c_LogicalOr(L: m_Specific(V: FalseVal), R: m_Value())) &&
4801	match(V: TrueVal, P: m_ZeroInt()))
4802	return ConstantInt::getFalse(Ty: Cond->getType());
4803
4804	// Match patterns that end in logical-and.
4805	if (match(V: FalseVal, P: m_ZeroInt())) {
4806	// !(X || Y) && X --> false (commuted 2 ways)
4807	if (match(V: Cond, P: m_Not(V: m_c_LogicalOr(L: m_Specific(V: TrueVal), R: m_Value()))))
4808	return ConstantInt::getFalse(Ty: Cond->getType());
4809	// X && !(X || Y) --> false (commuted 2 ways)
4810	if (match(V: TrueVal, P: m_Not(V: m_c_LogicalOr(L: m_Specific(V: Cond), R: m_Value()))))
4811	return ConstantInt::getFalse(Ty: Cond->getType());
4812
4813	// (X || Y) && Y --> Y (commuted 2 ways)
4814	if (match(V: Cond, P: m_c_LogicalOr(L: m_Specific(V: TrueVal), R: m_Value())))
4815	return TrueVal;
4816	// Y && (X || Y) --> Y (commuted 2 ways)
4817	if (match(V: TrueVal, P: m_c_LogicalOr(L: m_Specific(V: Cond), R: m_Value())))
4818	return Cond;
4819
4820	// (X || Y) && (X || !Y) --> X (commuted 8 ways)
4821	Value *X, *Y;
4822	if (match(V: Cond, P: m_c_LogicalOr(L: m_Value(V&: X), R: m_Not(V: m_Value(V&: Y)))) &&
4823	match(V: TrueVal, P: m_c_LogicalOr(L: m_Specific(V: X), R: m_Specific(V: Y))))
4824	return X;
4825	if (match(V: TrueVal, P: m_c_LogicalOr(L: m_Value(V&: X), R: m_Not(V: m_Value(V&: Y)))) &&
4826	match(V: Cond, P: m_c_LogicalOr(L: m_Specific(V: X), R: m_Specific(V: Y))))
4827	return X;
4828	}
4829
4830	// Match patterns that end in logical-or.
4831	if (match(V: TrueVal, P: m_One())) {
4832	// !(X && Y) || X --> true (commuted 2 ways)
4833	if (match(V: Cond, P: m_Not(V: m_c_LogicalAnd(L: m_Specific(V: FalseVal), R: m_Value()))))
4834	return ConstantInt::getTrue(Ty: Cond->getType());
4835	// X || !(X && Y) --> true (commuted 2 ways)
4836	if (match(V: FalseVal, P: m_Not(V: m_c_LogicalAnd(L: m_Specific(V: Cond), R: m_Value()))))
4837	return ConstantInt::getTrue(Ty: Cond->getType());
4838
4839	// (X && Y) || Y --> Y (commuted 2 ways)
4840	if (match(V: Cond, P: m_c_LogicalAnd(L: m_Specific(V: FalseVal), R: m_Value())))
4841	return FalseVal;
4842	// Y || (X && Y) --> Y (commuted 2 ways)
4843	if (match(V: FalseVal, P: m_c_LogicalAnd(L: m_Specific(V: Cond), R: m_Value())))
4844	return Cond;
4845	}
4846	}
4847
4848	// select ?, X, X -> X
4849	if (TrueVal == FalseVal)
4850	return TrueVal;
4851
4852	if (Cond == TrueVal) {
4853	// select i1 X, i1 X, i1 false --> X (logical-and)
4854	if (match(V: FalseVal, P: m_ZeroInt()))
4855	return Cond;
4856	// select i1 X, i1 X, i1 true --> true
4857	if (match(V: FalseVal, P: m_One()))
4858	return ConstantInt::getTrue(Ty: Cond->getType());
4859	}
4860	if (Cond == FalseVal) {
4861	// select i1 X, i1 true, i1 X --> X (logical-or)
4862	if (match(V: TrueVal, P: m_One()))
4863	return Cond;
4864	// select i1 X, i1 false, i1 X --> false
4865	if (match(V: TrueVal, P: m_ZeroInt()))
4866	return ConstantInt::getFalse(Ty: Cond->getType());
4867	}
4868
4869	// If the true or false value is poison, we can fold to the other value.
4870	// If the true or false value is undef, we can fold to the other value as
4871	// long as the other value isn't poison.
4872	// select ?, poison, X -> X
4873	// select ?, undef, X -> X
4874	if (isa<PoisonValue>(Val: TrueVal) ||
4875	(Q.isUndefValue(V: TrueVal) && impliesPoison(ValAssumedPoison: FalseVal, V: Cond)))
4876	return FalseVal;
4877	// select ?, X, poison -> X
4878	// select ?, X, undef -> X
4879	if (isa<PoisonValue>(Val: FalseVal) ||
4880	(Q.isUndefValue(V: FalseVal) && impliesPoison(ValAssumedPoison: TrueVal, V: Cond)))
4881	return TrueVal;
4882
4883	// Deal with partial undef vector constants: select ?, VecC, VecC' --> VecC''
4884	Constant *TrueC, *FalseC;
4885	if (isa<FixedVectorType>(Val: TrueVal->getType()) &&
4886	match(V: TrueVal, P: m_Constant(C&: TrueC)) &&
4887	match(V: FalseVal, P: m_Constant(C&: FalseC))) {
4888	unsigned NumElts =
4889	cast<FixedVectorType>(Val: TrueC->getType())->getNumElements();
4890	SmallVector<Constant *, 16> NewC;
4891	for (unsigned i = 0; i != NumElts; ++i) {
4892	// Bail out on incomplete vector constants.
4893	Constant *TEltC = TrueC->getAggregateElement(Elt: i);
4894	Constant *FEltC = FalseC->getAggregateElement(Elt: i);
4895	if (!TEltC || !FEltC)
4896	break;
4897
4898	// If the elements match (undef or not), that value is the result. If only
4899	// one element is undef, choose the defined element as the safe result.
4900	if (TEltC == FEltC)
4901	NewC.push_back(Elt: TEltC);
4902	else if (isa<PoisonValue>(Val: TEltC) ||
4903	(Q.isUndefValue(V: TEltC) && isGuaranteedNotToBePoison(V: FEltC)))
4904	NewC.push_back(Elt: FEltC);
4905	else if (isa<PoisonValue>(Val: FEltC) ||
4906	(Q.isUndefValue(V: FEltC) && isGuaranteedNotToBePoison(V: TEltC)))
4907	NewC.push_back(Elt: TEltC);
4908	else
4909	break;
4910	}
4911	if (NewC.size() == NumElts)
4912	return ConstantVector::get(V: NewC);
4913	}
4914
4915	if (Value *V =
4916	simplifySelectWithICmpCond(CondVal: Cond, TrueVal, FalseVal, Q, MaxRecurse))
4917	return V;
4918
4919	if (Value *V = simplifySelectWithFCmp(Cond, T: TrueVal, F: FalseVal, Q))
4920	return V;
4921
4922	if (Value *V = foldSelectWithBinaryOp(Cond, TrueVal, FalseVal))
4923	return V;
4924
4925	std::optional<bool> Imp = isImpliedByDomCondition(Cond, ContextI: Q.CxtI, DL: Q.DL);
4926	if (Imp)
4927	return *Imp ? TrueVal : FalseVal;
4928
4929	return nullptr;
4930	}
4931
4932	Value *llvm::simplifySelectInst(Value *Cond, Value *TrueVal, Value *FalseVal,
4933	const SimplifyQuery &Q) {
4934	return ::simplifySelectInst(Cond, TrueVal, FalseVal, Q, MaxRecurse: RecursionLimit);
4935	}
4936
4937	/// Given operands for an GetElementPtrInst, see if we can fold the result.
4938	/// If not, this returns null.
4939	static Value *simplifyGEPInst(Type *SrcTy, Value *Ptr,
4940	ArrayRef<Value *> Indices, bool InBounds,
4941	const SimplifyQuery &Q, unsigned) {
4942	// The type of the GEP pointer operand.
4943	unsigned AS =
4944	cast<PointerType>(Val: Ptr->getType()->getScalarType())->getAddressSpace();
4945
4946	// getelementptr P -> P.
4947	if (Indices.empty())
4948	return Ptr;
4949
4950	// Compute the (pointer) type returned by the GEP instruction.
4951	Type *LastType = GetElementPtrInst::getIndexedType(Ty: SrcTy, IdxList: Indices);
4952	Type *GEPTy = Ptr->getType();
4953	if (!GEPTy->isVectorTy()) {
4954	for (Value *Op : Indices) {
4955	// If one of the operands is a vector, the result type is a vector of
4956	// pointers. All vector operands must have the same number of elements.
4957	if (VectorType *VT = dyn_cast<VectorType>(Val: Op->getType())) {
4958	GEPTy = VectorType::get(ElementType: GEPTy, EC: VT->getElementCount());
4959	break;
4960	}
4961	}
4962	}
4963
4964	// All-zero GEP is a no-op, unless it performs a vector splat.
4965	if (Ptr->getType() == GEPTy &&
4966	all_of(Range&: Indices, P: [](const auto *V) { return match(V, m_Zero()); }))
4967	return Ptr;
4968
4969	// getelementptr poison, idx -> poison
4970	// getelementptr baseptr, poison -> poison
4971	if (isa<PoisonValue>(Val: Ptr) ||
4972	any_of(Range&: Indices, P: [](const auto *V) { return isa<PoisonValue>(V); }))
4973	return PoisonValue::get(T: GEPTy);
4974
4975	// getelementptr undef, idx -> undef
4976	if (Q.isUndefValue(V: Ptr))
4977	return UndefValue::get(T: GEPTy);
4978
4979	bool IsScalableVec =
4980	SrcTy->isScalableTy() || any_of(Range&: Indices, P: [](const Value *V) {
4981	return isa<ScalableVectorType>(Val: V->getType());
4982	});
4983
4984	if (Indices.size() == 1) {
4985	Type *Ty = SrcTy;
4986	if (!IsScalableVec && Ty->isSized()) {
4987	Value *P;
4988	uint64_t C;
4989	uint64_t TyAllocSize = Q.DL.getTypeAllocSize(Ty);
4990	// getelementptr P, N -> P if P points to a type of zero size.
4991	if (TyAllocSize == 0 && Ptr->getType() == GEPTy)
4992	return Ptr;
4993
4994	// The following transforms are only safe if the ptrtoint cast
4995	// doesn't truncate the pointers.
4996	if (Indices[0]->getType()->getScalarSizeInBits() ==
4997	Q.DL.getPointerSizeInBits(AS)) {
4998	auto CanSimplify = [GEPTy, &P, Ptr]() -> bool {
4999	return P->getType() == GEPTy &&
5000	getUnderlyingObject(V: P) == getUnderlyingObject(V: Ptr);
5001	};
5002	// getelementptr V, (sub P, V) -> P if P points to a type of size 1.
5003	if (TyAllocSize == 1 &&
5004	match(V: Indices[0],
5005	P: m_Sub(L: m_PtrToInt(Op: m_Value(V&: P)), R: m_PtrToInt(Op: m_Specific(V: Ptr)))) &&
5006	CanSimplify())
5007	return P;
5008
5009	// getelementptr V, (ashr (sub P, V), C) -> P if P points to a type of
5010	// size 1 << C.
5011	if (match(V: Indices[0], P: m_AShr(L: m_Sub(L: m_PtrToInt(Op: m_Value(V&: P)),
5012	R: m_PtrToInt(Op: m_Specific(V: Ptr))),
5013	R: m_ConstantInt(V&: C))) &&
5014	TyAllocSize == 1ULL << C && CanSimplify())
5015	return P;
5016
5017	// getelementptr V, (sdiv (sub P, V), C) -> P if P points to a type of
5018	// size C.
5019	if (match(V: Indices[0], P: m_SDiv(L: m_Sub(L: m_PtrToInt(Op: m_Value(V&: P)),
5020	R: m_PtrToInt(Op: m_Specific(V: Ptr))),
5021	R: m_SpecificInt(V: TyAllocSize))) &&
5022	CanSimplify())
5023	return P;
5024	}
5025	}
5026	}
5027
5028	if (!IsScalableVec && Q.DL.getTypeAllocSize(Ty: LastType) == 1 &&
5029	all_of(Range: Indices.drop_back(N: 1),
5030	P: [](Value *Idx) { return match(V: Idx, P: m_Zero()); })) {
5031	unsigned IdxWidth =
5032	Q.DL.getIndexSizeInBits(AS: Ptr->getType()->getPointerAddressSpace());
5033	if (Q.DL.getTypeSizeInBits(Ty: Indices.back()->getType()) == IdxWidth) {
5034	APInt BasePtrOffset(IdxWidth, 0);
5035	Value *StrippedBasePtr =
5036	Ptr->stripAndAccumulateInBoundsConstantOffsets(DL: Q.DL, Offset&: BasePtrOffset);
5037
5038	// Avoid creating inttoptr of zero here: While LLVMs treatment of
5039	// inttoptr is generally conservative, this particular case is folded to
5040	// a null pointer, which will have incorrect provenance.
5041
5042	// gep (gep V, C), (sub 0, V) -> C
5043	if (match(V: Indices.back(),
5044	P: m_Neg(V: m_PtrToInt(Op: m_Specific(V: StrippedBasePtr)))) &&
5045	!BasePtrOffset.isZero()) {
5046	auto *CI = ConstantInt::get(Context&: GEPTy->getContext(), V: BasePtrOffset);
5047	return ConstantExpr::getIntToPtr(C: CI, Ty: GEPTy);
5048	}
5049	// gep (gep V, C), (xor V, -1) -> C-1
5050	if (match(V: Indices.back(),
5051	P: m_Xor(L: m_PtrToInt(Op: m_Specific(V: StrippedBasePtr)), R: m_AllOnes())) &&
5052	!BasePtrOffset.isOne()) {
5053	auto *CI = ConstantInt::get(Context&: GEPTy->getContext(), V: BasePtrOffset - 1);
5054	return ConstantExpr::getIntToPtr(C: CI, Ty: GEPTy);
5055	}
5056	}
5057	}
5058
5059	// Check to see if this is constant foldable.
5060	if (!isa<Constant>(Val: Ptr) ||
5061	!all_of(Range&: Indices, P: [](Value *V) { return isa<Constant>(Val: V); }))
5062	return nullptr;
5063
5064	if (!ConstantExpr::isSupportedGetElementPtr(SrcElemTy: SrcTy))
5065	return ConstantFoldGetElementPtr(Ty: SrcTy, C: cast<Constant>(Val: Ptr), InBounds,
5066	InRange: std::nullopt, Idxs: Indices);
5067
5068	auto *CE = ConstantExpr::getGetElementPtr(Ty: SrcTy, C: cast<Constant>(Val: Ptr), IdxList: Indices,
5069	InBounds);
5070	return ConstantFoldConstant(C: CE, DL: Q.DL);
5071	}
5072
5073	Value *llvm::simplifyGEPInst(Type *SrcTy, Value *Ptr, ArrayRef<Value *> Indices,
5074	bool InBounds, const SimplifyQuery &Q) {
5075	return ::simplifyGEPInst(SrcTy, Ptr, Indices, InBounds, Q, RecursionLimit);
5076	}
5077
5078	/// Given operands for an InsertValueInst, see if we can fold the result.
5079	/// If not, this returns null.
5080	static Value *simplifyInsertValueInst(Value *Agg, Value *Val,
5081	ArrayRef<unsigned> Idxs,
5082	const SimplifyQuery &Q, unsigned) {
5083	if (Constant *CAgg = dyn_cast<Constant>(Val: Agg))
5084	if (Constant *CVal = dyn_cast<Constant>(Val))
5085	return ConstantFoldInsertValueInstruction(Agg: CAgg, Val: CVal, Idxs);
5086
5087	// insertvalue x, poison, n -> x
5088	// insertvalue x, undef, n -> x if x cannot be poison
5089	if (isa<PoisonValue>(Val) ||
5090	(Q.isUndefValue(V: Val) && isGuaranteedNotToBePoison(V: Agg)))
5091	return Agg;
5092
5093	// insertvalue x, (extractvalue y, n), n
5094	if (ExtractValueInst *EV = dyn_cast<ExtractValueInst>(Val))
5095	if (EV->getAggregateOperand()->getType() == Agg->getType() &&
5096	EV->getIndices() == Idxs) {
5097	// insertvalue poison, (extractvalue y, n), n -> y
5098	// insertvalue undef, (extractvalue y, n), n -> y if y cannot be poison
5099	if (isa<PoisonValue>(Val: Agg) ||
5100	(Q.isUndefValue(V: Agg) &&
5101	isGuaranteedNotToBePoison(V: EV->getAggregateOperand())))
5102	return EV->getAggregateOperand();
5103
5104	// insertvalue y, (extractvalue y, n), n -> y
5105	if (Agg == EV->getAggregateOperand())
5106	return Agg;
5107	}
5108
5109	return nullptr;
5110	}
5111
5112	Value *llvm::simplifyInsertValueInst(Value *Agg, Value *Val,
5113	ArrayRef<unsigned> Idxs,
5114	const SimplifyQuery &Q) {
5115	return ::simplifyInsertValueInst(Agg, Val, Idxs, Q, RecursionLimit);
5116	}
5117
5118	Value *llvm::simplifyInsertElementInst(Value *Vec, Value *Val, Value *Idx,
5119	const SimplifyQuery &Q) {
5120	// Try to constant fold.
5121	auto *VecC = dyn_cast<Constant>(Val: Vec);
5122	auto *ValC = dyn_cast<Constant>(Val);
5123	auto *IdxC = dyn_cast<Constant>(Val: Idx);
5124	if (VecC && ValC && IdxC)
5125	return ConstantExpr::getInsertElement(Vec: VecC, Elt: ValC, Idx: IdxC);
5126
5127	// For fixed-length vector, fold into poison if index is out of bounds.
5128	if (auto *CI = dyn_cast<ConstantInt>(Val: Idx)) {
5129	if (isa<FixedVectorType>(Val: Vec->getType()) &&
5130	CI->uge(Num: cast<FixedVectorType>(Val: Vec->getType())->getNumElements()))
5131	return PoisonValue::get(T: Vec->getType());
5132	}
5133
5134	// If index is undef, it might be out of bounds (see above case)
5135	if (Q.isUndefValue(V: Idx))
5136	return PoisonValue::get(T: Vec->getType());
5137
5138	// If the scalar is poison, or it is undef and there is no risk of
5139	// propagating poison from the vector value, simplify to the vector value.
5140	if (isa<PoisonValue>(Val) ||
5141	(Q.isUndefValue(V: Val) && isGuaranteedNotToBePoison(V: Vec)))
5142	return Vec;
5143
5144	// If we are extracting a value from a vector, then inserting it into the same
5145	// place, that's the input vector:
5146	// insertelt Vec, (extractelt Vec, Idx), Idx --> Vec
5147	if (match(V: Val, P: m_ExtractElt(Val: m_Specific(V: Vec), Idx: m_Specific(V: Idx))))
5148	return Vec;
5149
5150	return nullptr;
5151	}
5152
5153	/// Given operands for an ExtractValueInst, see if we can fold the result.
5154	/// If not, this returns null.
5155	static Value *simplifyExtractValueInst(Value *Agg, ArrayRef<unsigned> Idxs,
5156	const SimplifyQuery &, unsigned) {
5157	if (auto *CAgg = dyn_cast<Constant>(Val: Agg))
5158	return ConstantFoldExtractValueInstruction(Agg: CAgg, Idxs);
5159
5160	// extractvalue x, (insertvalue y, elt, n), n -> elt
5161	unsigned NumIdxs = Idxs.size();
5162	for (auto *IVI = dyn_cast<InsertValueInst>(Val: Agg); IVI != nullptr;
5163	IVI = dyn_cast<InsertValueInst>(Val: IVI->getAggregateOperand())) {
5164	ArrayRef<unsigned> InsertValueIdxs = IVI->getIndices();
5165	unsigned NumInsertValueIdxs = InsertValueIdxs.size();
5166	unsigned NumCommonIdxs = std::min(a: NumInsertValueIdxs, b: NumIdxs);
5167	if (InsertValueIdxs.slice(N: 0, M: NumCommonIdxs) ==
5168	Idxs.slice(N: 0, M: NumCommonIdxs)) {
5169	if (NumIdxs == NumInsertValueIdxs)
5170	return IVI->getInsertedValueOperand();
5171	break;
5172	}
5173	}
5174
5175	return nullptr;
5176	}
5177
5178	Value *llvm::simplifyExtractValueInst(Value *Agg, ArrayRef<unsigned> Idxs,
5179	const SimplifyQuery &Q) {
5180	return ::simplifyExtractValueInst(Agg, Idxs, Q, RecursionLimit);
5181	}
5182
5183	/// Given operands for an ExtractElementInst, see if we can fold the result.
5184	/// If not, this returns null.
5185	static Value *simplifyExtractElementInst(Value *Vec, Value *Idx,
5186	const SimplifyQuery &Q, unsigned) {
5187	auto *VecVTy = cast<VectorType>(Val: Vec->getType());
5188	if (auto *CVec = dyn_cast<Constant>(Val: Vec)) {
5189	if (auto *CIdx = dyn_cast<Constant>(Val: Idx))
5190	return ConstantExpr::getExtractElement(Vec: CVec, Idx: CIdx);
5191
5192	if (Q.isUndefValue(V: Vec))
5193	return UndefValue::get(T: VecVTy->getElementType());
5194	}
5195
5196	// An undef extract index can be arbitrarily chosen to be an out-of-range
5197	// index value, which would result in the instruction being poison.
5198	if (Q.isUndefValue(V: Idx))
5199	return PoisonValue::get(T: VecVTy->getElementType());
5200
5201	// If extracting a specified index from the vector, see if we can recursively
5202	// find a previously computed scalar that was inserted into the vector.
5203	if (auto *IdxC = dyn_cast<ConstantInt>(Val: Idx)) {
5204	// For fixed-length vector, fold into undef if index is out of bounds.
5205	unsigned MinNumElts = VecVTy->getElementCount().getKnownMinValue();
5206	if (isa<FixedVectorType>(Val: VecVTy) && IdxC->getValue().uge(RHS: MinNumElts))
5207	return PoisonValue::get(T: VecVTy->getElementType());
5208	// Handle case where an element is extracted from a splat.
5209	if (IdxC->getValue().ult(RHS: MinNumElts))
5210	if (auto *Splat = getSplatValue(V: Vec))
5211	return Splat;
5212	if (Value *Elt = findScalarElement(V: Vec, EltNo: IdxC->getZExtValue()))
5213	return Elt;
5214	} else {
5215	// extractelt x, (insertelt y, elt, n), n -> elt
5216	// If the possibly-variable indices are trivially known to be equal
5217	// (because they are the same operand) then use the value that was
5218	// inserted directly.
5219	auto *IE = dyn_cast<InsertElementInst>(Val: Vec);
5220	if (IE && IE->getOperand(i_nocapture: 2) == Idx)
5221	return IE->getOperand(i_nocapture: 1);
5222
5223	// The index is not relevant if our vector is a splat.
5224	if (Value *Splat = getSplatValue(V: Vec))
5225	return Splat;
5226	}
5227	return nullptr;
5228	}
5229
5230	Value *llvm::simplifyExtractElementInst(Value *Vec, Value *Idx,
5231	const SimplifyQuery &Q) {
5232	return ::simplifyExtractElementInst(Vec, Idx, Q, RecursionLimit);
5233	}
5234
5235	/// See if we can fold the given phi. If not, returns null.
5236	static Value *simplifyPHINode(PHINode *PN, ArrayRef<Value *> IncomingValues,
5237	const SimplifyQuery &Q) {
5238	// WARNING: no matter how worthwhile it may seem, we can not perform PHI CSE
5239	// here, because the PHI we may succeed simplifying to was not
5240	// def-reachable from the original PHI!
5241
5242	// If all of the PHI's incoming values are the same then replace the PHI node
5243	// with the common value.
5244	Value *CommonValue = nullptr;
5245	bool HasUndefInput = false;
5246	for (Value *Incoming : IncomingValues) {
5247	// If the incoming value is the phi node itself, it can safely be skipped.
5248	if (Incoming == PN)
5249	continue;
5250	if (Q.isUndefValue(V: Incoming)) {
5251	// Remember that we saw an undef value, but otherwise ignore them.
5252	HasUndefInput = true;
5253	continue;
5254	}
5255	if (CommonValue && Incoming != CommonValue)
5256	return nullptr; // Not the same, bail out.
5257	CommonValue = Incoming;
5258	}
5259
5260	// If CommonValue is null then all of the incoming values were either undef or
5261	// equal to the phi node itself.
5262	if (!CommonValue)
5263	return UndefValue::get(T: PN->getType());
5264
5265	if (HasUndefInput) {
5266	// If we have a PHI node like phi(X, undef, X), where X is defined by some
5267	// instruction, we cannot return X as the result of the PHI node unless it
5268	// dominates the PHI block.
5269	return valueDominatesPHI(V: CommonValue, P: PN, DT: Q.DT) ? CommonValue : nullptr;
5270	}
5271
5272	return CommonValue;
5273	}
5274
5275	static Value *simplifyCastInst(unsigned CastOpc, Value *Op, Type *Ty,
5276	const SimplifyQuery &Q, unsigned MaxRecurse) {
5277	if (auto *C = dyn_cast<Constant>(Val: Op))
5278	return ConstantFoldCastOperand(Opcode: CastOpc, C, DestTy: Ty, DL: Q.DL);
5279
5280	if (auto *CI = dyn_cast<CastInst>(Val: Op)) {
5281	auto *Src = CI->getOperand(i_nocapture: 0);
5282	Type *SrcTy = Src->getType();
5283	Type *MidTy = CI->getType();
5284	Type *DstTy = Ty;
5285	if (Src->getType() == Ty) {
5286	auto FirstOp = static_cast<Instruction::CastOps>(CI->getOpcode());
5287	auto SecondOp = static_cast<Instruction::CastOps>(CastOpc);
5288	Type *SrcIntPtrTy =
5289	SrcTy->isPtrOrPtrVectorTy() ? Q.DL.getIntPtrType(SrcTy) : nullptr;
5290	Type *MidIntPtrTy =
5291	MidTy->isPtrOrPtrVectorTy() ? Q.DL.getIntPtrType(MidTy) : nullptr;
5292	Type *DstIntPtrTy =
5293	DstTy->isPtrOrPtrVectorTy() ? Q.DL.getIntPtrType(DstTy) : nullptr;
5294	if (CastInst::isEliminableCastPair(firstOpcode: FirstOp, secondOpcode: SecondOp, SrcTy, MidTy, DstTy,
5295	SrcIntPtrTy, MidIntPtrTy,
5296	DstIntPtrTy) == Instruction::BitCast)
5297	return Src;
5298	}
5299	}
5300
5301	// bitcast x -> x
5302	if (CastOpc == Instruction::BitCast)
5303	if (Op->getType() == Ty)
5304	return Op;
5305
5306	return nullptr;
5307	}
5308
5309	Value *llvm::simplifyCastInst(unsigned CastOpc, Value *Op, Type *Ty,
5310	const SimplifyQuery &Q) {
5311	return ::simplifyCastInst(CastOpc, Op, Ty, Q, MaxRecurse: RecursionLimit);
5312	}
5313
5314	/// For the given destination element of a shuffle, peek through shuffles to
5315	/// match a root vector source operand that contains that element in the same
5316	/// vector lane (ie, the same mask index), so we can eliminate the shuffle(s).
5317	static Value *foldIdentityShuffles(int DestElt, Value *Op0, Value *Op1,
5318	int MaskVal, Value *RootVec,
5319	unsigned MaxRecurse) {
5320	if (!MaxRecurse--)
5321	return nullptr;
5322
5323	// Bail out if any mask value is undefined. That kind of shuffle may be
5324	// simplified further based on demanded bits or other folds.
5325	if (MaskVal == -1)
5326	return nullptr;
5327
5328	// The mask value chooses which source operand we need to look at next.
5329	int InVecNumElts = cast<FixedVectorType>(Val: Op0->getType())->getNumElements();
5330	int RootElt = MaskVal;
5331	Value *SourceOp = Op0;
5332	if (MaskVal >= InVecNumElts) {
5333	RootElt = MaskVal - InVecNumElts;
5334	SourceOp = Op1;
5335	}
5336
5337	// If the source operand is a shuffle itself, look through it to find the
5338	// matching root vector.
5339	if (auto *SourceShuf = dyn_cast<ShuffleVectorInst>(Val: SourceOp)) {
5340	return foldIdentityShuffles(
5341	DestElt, Op0: SourceShuf->getOperand(i_nocapture: 0), Op1: SourceShuf->getOperand(i_nocapture: 1),
5342	MaskVal: SourceShuf->getMaskValue(Elt: RootElt), RootVec, MaxRecurse);
5343	}
5344
5345	// TODO: Look through bitcasts? What if the bitcast changes the vector element
5346	// size?
5347
5348	// The source operand is not a shuffle. Initialize the root vector value for
5349	// this shuffle if that has not been done yet.
5350	if (!RootVec)
5351	RootVec = SourceOp;
5352
5353	// Give up as soon as a source operand does not match the existing root value.
5354	if (RootVec != SourceOp)
5355	return nullptr;
5356
5357	// The element must be coming from the same lane in the source vector
5358	// (although it may have crossed lanes in intermediate shuffles).
5359	if (RootElt != DestElt)
5360	return nullptr;
5361
5362	return RootVec;
5363	}
5364
5365	static Value *simplifyShuffleVectorInst(Value *Op0, Value *Op1,
5366	ArrayRef<int> Mask, Type *RetTy,
5367	const SimplifyQuery &Q,
5368	unsigned MaxRecurse) {
5369	if (all_of(Range&: Mask, P: [](int Elem) { return Elem == PoisonMaskElem; }))
5370	return PoisonValue::get(T: RetTy);
5371
5372	auto *InVecTy = cast<VectorType>(Val: Op0->getType());
5373	unsigned MaskNumElts = Mask.size();
5374	ElementCount InVecEltCount = InVecTy->getElementCount();
5375
5376	bool Scalable = InVecEltCount.isScalable();
5377
5378	SmallVector<int, 32> Indices;
5379	Indices.assign(in_start: Mask.begin(), in_end: Mask.end());
5380
5381	// Canonicalization: If mask does not select elements from an input vector,
5382	// replace that input vector with poison.
5383	if (!Scalable) {
5384	bool MaskSelects0 = false, MaskSelects1 = false;
5385	unsigned InVecNumElts = InVecEltCount.getKnownMinValue();
5386	for (unsigned i = 0; i != MaskNumElts; ++i) {
5387	if (Indices[i] == -1)
5388	continue;
5389	if ((unsigned)Indices[i] < InVecNumElts)
5390	MaskSelects0 = true;
5391	else
5392	MaskSelects1 = true;
5393	}
5394	if (!MaskSelects0)
5395	Op0 = PoisonValue::get(T: InVecTy);
5396	if (!MaskSelects1)
5397	Op1 = PoisonValue::get(T: InVecTy);
5398	}
5399
5400	auto *Op0Const = dyn_cast<Constant>(Val: Op0);
5401	auto *Op1Const = dyn_cast<Constant>(Val: Op1);
5402
5403	// If all operands are constant, constant fold the shuffle. This
5404	// transformation depends on the value of the mask which is not known at
5405	// compile time for scalable vectors
5406	if (Op0Const && Op1Const)
5407	return ConstantExpr::getShuffleVector(V1: Op0Const, V2: Op1Const, Mask);
5408
5409	// Canonicalization: if only one input vector is constant, it shall be the
5410	// second one. This transformation depends on the value of the mask which
5411	// is not known at compile time for scalable vectors
5412	if (!Scalable && Op0Const && !Op1Const) {
5413	std::swap(a&: Op0, b&: Op1);
5414	ShuffleVectorInst::commuteShuffleMask(Mask: Indices,
5415	InVecNumElts: InVecEltCount.getKnownMinValue());
5416	}
5417
5418	// A splat of an inserted scalar constant becomes a vector constant:
5419	// shuf (inselt ?, C, IndexC), undef, <IndexC, IndexC...> --> <C, C...>
5420	// NOTE: We may have commuted above, so analyze the updated Indices, not the
5421	// original mask constant.
5422	// NOTE: This transformation depends on the value of the mask which is not
5423	// known at compile time for scalable vectors
5424	Constant *C;
5425	ConstantInt *IndexC;
5426	if (!Scalable && match(V: Op0, P: m_InsertElt(Val: m_Value(), Elt: m_Constant(C),
5427	Idx: m_ConstantInt(CI&: IndexC)))) {
5428	// Match a splat shuffle mask of the insert index allowing undef elements.
5429	int InsertIndex = IndexC->getZExtValue();
5430	if (all_of(Range&: Indices, P: [InsertIndex](int MaskElt) {
5431	return MaskElt == InsertIndex || MaskElt == -1;
5432	})) {
5433	assert(isa<UndefValue>(Op1) && "Expected undef operand 1 for splat");
5434
5435	// Shuffle mask poisons become poison constant result elements.
5436	SmallVector<Constant *, 16> VecC(MaskNumElts, C);
5437	for (unsigned i = 0; i != MaskNumElts; ++i)
5438	if (Indices[i] == -1)
5439	VecC[i] = PoisonValue::get(T: C->getType());
5440	return ConstantVector::get(V: VecC);
5441	}
5442	}
5443
5444	// A shuffle of a splat is always the splat itself. Legal if the shuffle's
5445	// value type is same as the input vectors' type.
5446	if (auto *OpShuf = dyn_cast<ShuffleVectorInst>(Val: Op0))
5447	if (Q.isUndefValue(V: Op1) && RetTy == InVecTy &&
5448	all_equal(Range: OpShuf->getShuffleMask()))
5449	return Op0;
5450
5451	// All remaining transformation depend on the value of the mask, which is
5452	// not known at compile time for scalable vectors.
5453	if (Scalable)
5454	return nullptr;
5455
5456	// Don't fold a shuffle with undef mask elements. This may get folded in a
5457	// better way using demanded bits or other analysis.
5458	// TODO: Should we allow this?
5459	if (is_contained(Range&: Indices, Element: -1))
5460	return nullptr;
5461
5462	// Check if every element of this shuffle can be mapped back to the
5463	// corresponding element of a single root vector. If so, we don't need this
5464	// shuffle. This handles simple identity shuffles as well as chains of
5465	// shuffles that may widen/narrow and/or move elements across lanes and back.
5466	Value *RootVec = nullptr;
5467	for (unsigned i = 0; i != MaskNumElts; ++i) {
5468	// Note that recursion is limited for each vector element, so if any element
5469	// exceeds the limit, this will fail to simplify.
5470	RootVec =
5471	foldIdentityShuffles(DestElt: i, Op0, Op1, MaskVal: Indices[i], RootVec, MaxRecurse);
5472
5473	// We can't replace a widening/narrowing shuffle with one of its operands.
5474	if (!RootVec || RootVec->getType() != RetTy)
5475	return nullptr;
5476	}
5477	return RootVec;
5478	}
5479
5480	/// Given operands for a ShuffleVectorInst, fold the result or return null.
5481	Value *llvm::simplifyShuffleVectorInst(Value *Op0, Value *Op1,
5482	ArrayRef<int> Mask, Type *RetTy,
5483	const SimplifyQuery &Q) {
5484	return ::simplifyShuffleVectorInst(Op0, Op1, Mask, RetTy, Q, MaxRecurse: RecursionLimit);
5485	}
5486
5487	static Constant *foldConstant(Instruction::UnaryOps Opcode, Value *&Op,
5488	const SimplifyQuery &Q) {
5489	if (auto *C = dyn_cast<Constant>(Val: Op))
5490	return ConstantFoldUnaryOpOperand(Opcode, Op: C, DL: Q.DL);
5491	return nullptr;
5492	}
5493
5494	/// Given the operand for an FNeg, see if we can fold the result. If not, this
5495	/// returns null.
5496	static Value *simplifyFNegInst(Value *Op, FastMathFlags FMF,
5497	const SimplifyQuery &Q, unsigned MaxRecurse) {
5498	if (Constant *C = foldConstant(Opcode: Instruction::FNeg, Op, Q))
5499	return C;
5500
5501	Value *X;
5502	// fneg (fneg X) ==> X
5503	if (match(V: Op, P: m_FNeg(X: m_Value(V&: X))))
5504	return X;
5505
5506	return nullptr;
5507	}
5508
5509	Value *llvm::simplifyFNegInst(Value *Op, FastMathFlags FMF,
5510	const SimplifyQuery &Q) {
5511	return ::simplifyFNegInst(Op, FMF, Q, MaxRecurse: RecursionLimit);
5512	}
5513
5514	/// Try to propagate existing NaN values when possible. If not, replace the
5515	/// constant or elements in the constant with a canonical NaN.
5516	static Constant *propagateNaN(Constant *In) {
5517	Type *Ty = In->getType();
5518	if (auto *VecTy = dyn_cast<FixedVectorType>(Val: Ty)) {
5519	unsigned NumElts = VecTy->getNumElements();
5520	SmallVector<Constant *, 32> NewC(NumElts);
5521	for (unsigned i = 0; i != NumElts; ++i) {
5522	Constant *EltC = In->getAggregateElement(Elt: i);
5523	// Poison elements propagate. NaN propagates except signaling is quieted.
5524	// Replace unknown or undef elements with canonical NaN.
5525	if (EltC && isa<PoisonValue>(Val: EltC))
5526	NewC[i] = EltC;
5527	else if (EltC && EltC->isNaN())
5528	NewC[i] = ConstantFP::get(
5529	Ty: EltC->getType(), V: cast<ConstantFP>(Val: EltC)->getValue().makeQuiet());
5530	else
5531	NewC[i] = ConstantFP::getNaN(Ty: VecTy->getElementType());
5532	}
5533	return ConstantVector::get(V: NewC);
5534	}
5535
5536	// If it is not a fixed vector, but not a simple NaN either, return a
5537	// canonical NaN.
5538	if (!In->isNaN())
5539	return ConstantFP::getNaN(Ty);
5540
5541	// If we known this is a NaN, and it's scalable vector, we must have a splat
5542	// on our hands. Grab that before splatting a QNaN constant.
5543	if (isa<ScalableVectorType>(Val: Ty)) {
5544	auto *Splat = In->getSplatValue();
5545	assert(Splat && Splat->isNaN() &&
5546	"Found a scalable-vector NaN but not a splat");
5547	In = Splat;
5548	}
5549
5550	// Propagate an existing QNaN constant. If it is an SNaN, make it quiet, but
5551	// preserve the sign/payload.
5552	return ConstantFP::get(Ty, V: cast<ConstantFP>(Val: In)->getValue().makeQuiet());
5553	}
5554
5555	/// Perform folds that are common to any floating-point operation. This implies
5556	/// transforms based on poison/undef/NaN because the operation itself makes no
5557	/// difference to the result.
5558	static Constant *simplifyFPOp(ArrayRef<Value *> Ops, FastMathFlags FMF,
5559	const SimplifyQuery &Q,
5560	fp::ExceptionBehavior ExBehavior,
5561	RoundingMode Rounding) {
5562	// Poison is independent of anything else. It always propagates from an
5563	// operand to a math result.
5564	if (any_of(Range&: Ops, P: [](Value *V) { return match(V, P: m_Poison()); }))
5565	return PoisonValue::get(T: Ops[0]->getType());
5566
5567	for (Value *V : Ops) {
5568	bool IsNan = match(V, P: m_NaN());
5569	bool IsInf = match(V, P: m_Inf());
5570	bool IsUndef = Q.isUndefValue(V);
5571
5572	// If this operation has 'nnan' or 'ninf' and at least 1 disallowed operand
5573	// (an undef operand can be chosen to be Nan/Inf), then the result of
5574	// this operation is poison.
5575	if (FMF.noNaNs() && (IsNan || IsUndef))
5576	return PoisonValue::get(T: V->getType());
5577	if (FMF.noInfs() && (IsInf || IsUndef))
5578	return PoisonValue::get(T: V->getType());
5579
5580	if (isDefaultFPEnvironment(EB: ExBehavior, RM: Rounding)) {
5581	// Undef does not propagate because undef means that all bits can take on
5582	// any value. If this is undef * NaN for example, then the result values
5583	// (at least the exponent bits) are limited. Assume the undef is a
5584	// canonical NaN and propagate that.
5585	if (IsUndef)
5586	return ConstantFP::getNaN(Ty: V->getType());
5587	if (IsNan)
5588	return propagateNaN(In: cast<Constant>(Val: V));
5589	} else if (ExBehavior != fp::ebStrict) {
5590	if (IsNan)
5591	return propagateNaN(In: cast<Constant>(Val: V));
5592	}
5593	}
5594	return nullptr;
5595	}
5596
5597	/// Given operands for an FAdd, see if we can fold the result. If not, this
5598	/// returns null.
5599	static Value *
5600	simplifyFAddInst(Value *Op0, Value *Op1, FastMathFlags FMF,
5601	const SimplifyQuery &Q, unsigned MaxRecurse,
5602	fp::ExceptionBehavior ExBehavior = fp::ebIgnore,
5603	RoundingMode Rounding = RoundingMode::NearestTiesToEven) {
5604	if (isDefaultFPEnvironment(EB: ExBehavior, RM: Rounding))
5605	if (Constant *C = foldOrCommuteConstant(Opcode: Instruction::FAdd, Op0, Op1, Q))
5606	return C;
5607
5608	if (Constant *C = simplifyFPOp(Ops: {Op0, Op1}, FMF, Q, ExBehavior, Rounding))
5609	return C;
5610
5611	// fadd X, -0 ==> X
5612	// With strict/constrained FP, we have these possible edge cases that do
5613	// not simplify to Op0:
5614	// fadd SNaN, -0.0 --> QNaN
5615	// fadd +0.0, -0.0 --> -0.0 (but only with round toward negative)
5616	if (canIgnoreSNaN(EB: ExBehavior, FMF) &&
5617	(!canRoundingModeBe(RM: Rounding, QRM: RoundingMode::TowardNegative) ||
5618	FMF.noSignedZeros()))
5619	if (match(V: Op1, P: m_NegZeroFP()))
5620	return Op0;
5621
5622	// fadd X, 0 ==> X, when we know X is not -0
5623	if (canIgnoreSNaN(EB: ExBehavior, FMF))
5624	if (match(V: Op1, P: m_PosZeroFP()) &&
5625	(FMF.noSignedZeros() || cannotBeNegativeZero(V: Op0, /*Depth=*/0, SQ: Q)))
5626	return Op0;
5627
5628	if (!isDefaultFPEnvironment(EB: ExBehavior, RM: Rounding))
5629	return nullptr;
5630
5631	if (FMF.noNaNs()) {
5632	// With nnan: X + {+/-}Inf --> {+/-}Inf
5633	if (match(V: Op1, P: m_Inf()))
5634	return Op1;
5635
5636	// With nnan: -X + X --> 0.0 (and commuted variant)
5637	// We don't have to explicitly exclude infinities (ninf): INF + -INF == NaN.
5638	// Negative zeros are allowed because we always end up with positive zero:
5639	// X = -0.0: (-0.0 - (-0.0)) + (-0.0) == ( 0.0) + (-0.0) == 0.0
5640	// X = -0.0: ( 0.0 - (-0.0)) + (-0.0) == ( 0.0) + (-0.0) == 0.0
5641	// X = 0.0: (-0.0 - ( 0.0)) + ( 0.0) == (-0.0) + ( 0.0) == 0.0
5642	// X = 0.0: ( 0.0 - ( 0.0)) + ( 0.0) == ( 0.0) + ( 0.0) == 0.0
5643	if (match(V: Op0, P: m_FSub(L: m_AnyZeroFP(), R: m_Specific(V: Op1))) ||
5644	match(V: Op1, P: m_FSub(L: m_AnyZeroFP(), R: m_Specific(V: Op0))))
5645	return ConstantFP::getZero(Ty: Op0->getType());
5646
5647	if (match(V: Op0, P: m_FNeg(X: m_Specific(V: Op1))) ||
5648	match(V: Op1, P: m_FNeg(X: m_Specific(V: Op0))))
5649	return ConstantFP::getZero(Ty: Op0->getType());
5650	}
5651
5652	// (X - Y) + Y --> X
5653	// Y + (X - Y) --> X
5654	Value *X;
5655	if (FMF.noSignedZeros() && FMF.allowReassoc() &&
5656	(match(V: Op0, P: m_FSub(L: m_Value(V&: X), R: m_Specific(V: Op1))) ||
5657	match(V: Op1, P: m_FSub(L: m_Value(V&: X), R: m_Specific(V: Op0)))))
5658	return X;
5659
5660	return nullptr;
5661	}
5662
5663	/// Given operands for an FSub, see if we can fold the result. If not, this
5664	/// returns null.
5665	static Value *
5666	simplifyFSubInst(Value *Op0, Value *Op1, FastMathFlags FMF,
5667	const SimplifyQuery &Q, unsigned MaxRecurse,
5668	fp::ExceptionBehavior ExBehavior = fp::ebIgnore,
5669	RoundingMode Rounding = RoundingMode::NearestTiesToEven) {
5670	if (isDefaultFPEnvironment(EB: ExBehavior, RM: Rounding))
5671	if (Constant *C = foldOrCommuteConstant(Opcode: Instruction::FSub, Op0, Op1, Q))
5672	return C;
5673
5674	if (Constant *C = simplifyFPOp(Ops: {Op0, Op1}, FMF, Q, ExBehavior, Rounding))
5675	return C;
5676
5677	// fsub X, +0 ==> X
5678	if (canIgnoreSNaN(EB: ExBehavior, FMF) &&
5679	(!canRoundingModeBe(RM: Rounding, QRM: RoundingMode::TowardNegative) ||
5680	FMF.noSignedZeros()))
5681	if (match(V: Op1, P: m_PosZeroFP()))
5682	return Op0;
5683
5684	// fsub X, -0 ==> X, when we know X is not -0
5685	if (canIgnoreSNaN(EB: ExBehavior, FMF))
5686	if (match(V: Op1, P: m_NegZeroFP()) &&
5687	(FMF.noSignedZeros() || cannotBeNegativeZero(V: Op0, /*Depth=*/0, SQ: Q)))
5688	return Op0;
5689
5690	// fsub -0.0, (fsub -0.0, X) ==> X
5691	// fsub -0.0, (fneg X) ==> X
5692	Value *X;
5693	if (canIgnoreSNaN(EB: ExBehavior, FMF))
5694	if (match(V: Op0, P: m_NegZeroFP()) && match(V: Op1, P: m_FNeg(X: m_Value(V&: X))))
5695	return X;
5696
5697	// fsub 0.0, (fsub 0.0, X) ==> X if signed zeros are ignored.
5698	// fsub 0.0, (fneg X) ==> X if signed zeros are ignored.
5699	if (canIgnoreSNaN(EB: ExBehavior, FMF))
5700	if (FMF.noSignedZeros() && match(V: Op0, P: m_AnyZeroFP()) &&
5701	(match(V: Op1, P: m_FSub(L: m_AnyZeroFP(), R: m_Value(V&: X))) ||
5702	match(V: Op1, P: m_FNeg(X: m_Value(V&: X)))))
5703	return X;
5704
5705	if (!isDefaultFPEnvironment(EB: ExBehavior, RM: Rounding))
5706	return nullptr;
5707
5708	if (FMF.noNaNs()) {
5709	// fsub nnan x, x ==> 0.0
5710	if (Op0 == Op1)
5711	return Constant::getNullValue(Ty: Op0->getType());
5712
5713	// With nnan: {+/-}Inf - X --> {+/-}Inf
5714	if (match(V: Op0, P: m_Inf()))
5715	return Op0;
5716
5717	// With nnan: X - {+/-}Inf --> {-/+}Inf
5718	if (match(V: Op1, P: m_Inf()))
5719	return foldConstant(Opcode: Instruction::FNeg, Op&: Op1, Q);
5720	}
5721
5722	// Y - (Y - X) --> X
5723	// (X + Y) - Y --> X
5724	if (FMF.noSignedZeros() && FMF.allowReassoc() &&
5725	(match(V: Op1, P: m_FSub(L: m_Specific(V: Op0), R: m_Value(V&: X))) ||
5726	match(V: Op0, P: m_c_FAdd(L: m_Specific(V: Op1), R: m_Value(V&: X)))))
5727	return X;
5728
5729	return nullptr;
5730	}
5731
5732	static Value *simplifyFMAFMul(Value *Op0, Value *Op1, FastMathFlags FMF,
5733	const SimplifyQuery &Q, unsigned MaxRecurse,
5734	fp::ExceptionBehavior ExBehavior,
5735	RoundingMode Rounding) {
5736	if (Constant *C = simplifyFPOp(Ops: {Op0, Op1}, FMF, Q, ExBehavior, Rounding))
5737	return C;
5738
5739	if (!isDefaultFPEnvironment(EB: ExBehavior, RM: Rounding))
5740	return nullptr;
5741
5742	// Canonicalize special constants as operand 1.
5743	if (match(V: Op0, P: m_FPOne()) || match(V: Op0, P: m_AnyZeroFP()))
5744	std::swap(a&: Op0, b&: Op1);
5745
5746	// X * 1.0 --> X
5747	if (match(V: Op1, P: m_FPOne()))
5748	return Op0;
5749
5750	if (match(V: Op1, P: m_AnyZeroFP())) {
5751	// X * 0.0 --> 0.0 (with nnan and nsz)
5752	if (FMF.noNaNs() && FMF.noSignedZeros())
5753	return ConstantFP::getZero(Ty: Op0->getType());
5754
5755	KnownFPClass Known =
5756	computeKnownFPClass(V: Op0, FMF, InterestedClasses: fcInf | fcNan, /*Depth=*/0, SQ: Q);
5757	if (Known.isKnownNever(Mask: fcInf | fcNan)) {
5758	// +normal number * (-)0.0 --> (-)0.0
5759	if (Known.SignBit == false)
5760	return Op1;
5761	// -normal number * (-)0.0 --> -(-)0.0
5762	if (Known.SignBit == true)
5763	return foldConstant(Opcode: Instruction::FNeg, Op&: Op1, Q);
5764	}
5765	}
5766
5767	// sqrt(X) * sqrt(X) --> X, if we can:
5768	// 1. Remove the intermediate rounding (reassociate).
5769	// 2. Ignore non-zero negative numbers because sqrt would produce NAN.
5770	// 3. Ignore -0.0 because sqrt(-0.0) == -0.0, but -0.0 * -0.0 == 0.0.
5771	Value *X;
5772	if (Op0 == Op1 && match(V: Op0, P: m_Sqrt(Op0: m_Value(V&: X))) && FMF.allowReassoc() &&
5773	FMF.noNaNs() && FMF.noSignedZeros())
5774	return X;
5775
5776	return nullptr;
5777	}
5778
5779	/// Given the operands for an FMul, see if we can fold the result
5780	static Value *
5781	simplifyFMulInst(Value *Op0, Value *Op1, FastMathFlags FMF,
5782	const SimplifyQuery &Q, unsigned MaxRecurse,
5783	fp::ExceptionBehavior ExBehavior = fp::ebIgnore,
5784	RoundingMode Rounding = RoundingMode::NearestTiesToEven) {
5785	if (isDefaultFPEnvironment(EB: ExBehavior, RM: Rounding))
5786	if (Constant *C = foldOrCommuteConstant(Opcode: Instruction::FMul, Op0, Op1, Q))
5787	return C;
5788
5789	// Now apply simplifications that do not require rounding.
5790	return simplifyFMAFMul(Op0, Op1, FMF, Q, MaxRecurse, ExBehavior, Rounding);
5791	}
5792
5793	Value *llvm::simplifyFAddInst(Value *Op0, Value *Op1, FastMathFlags FMF,
5794	const SimplifyQuery &Q,
5795	fp::ExceptionBehavior ExBehavior,
5796	RoundingMode Rounding) {
5797	return ::simplifyFAddInst(Op0, Op1, FMF, Q, MaxRecurse: RecursionLimit, ExBehavior,
5798	Rounding);
5799	}
5800
5801	Value *llvm::simplifyFSubInst(Value *Op0, Value *Op1, FastMathFlags FMF,
5802	const SimplifyQuery &Q,
5803	fp::ExceptionBehavior ExBehavior,
5804	RoundingMode Rounding) {
5805	return ::simplifyFSubInst(Op0, Op1, FMF, Q, MaxRecurse: RecursionLimit, ExBehavior,
5806	Rounding);
5807	}
5808
5809	Value *llvm::simplifyFMulInst(Value *Op0, Value *Op1, FastMathFlags FMF,
5810	const SimplifyQuery &Q,
5811	fp::ExceptionBehavior ExBehavior,
5812	RoundingMode Rounding) {
5813	return ::simplifyFMulInst(Op0, Op1, FMF, Q, MaxRecurse: RecursionLimit, ExBehavior,
5814	Rounding);
5815	}
5816
5817	Value *llvm::simplifyFMAFMul(Value *Op0, Value *Op1, FastMathFlags FMF,
5818	const SimplifyQuery &Q,
5819	fp::ExceptionBehavior ExBehavior,
5820	RoundingMode Rounding) {
5821	return ::simplifyFMAFMul(Op0, Op1, FMF, Q, MaxRecurse: RecursionLimit, ExBehavior,
5822	Rounding);
5823	}
5824
5825	static Value *
5826	simplifyFDivInst(Value *Op0, Value *Op1, FastMathFlags FMF,
5827	const SimplifyQuery &Q, unsigned,
5828	fp::ExceptionBehavior ExBehavior = fp::ebIgnore,
5829	RoundingMode Rounding = RoundingMode::NearestTiesToEven) {
5830	if (isDefaultFPEnvironment(EB: ExBehavior, RM: Rounding))
5831	if (Constant *C = foldOrCommuteConstant(Opcode: Instruction::FDiv, Op0, Op1, Q))
5832	return C;
5833
5834	if (Constant *C = simplifyFPOp(Ops: {Op0, Op1}, FMF, Q, ExBehavior, Rounding))
5835	return C;
5836
5837	if (!isDefaultFPEnvironment(EB: ExBehavior, RM: Rounding))
5838	return nullptr;
5839
5840	// X / 1.0 -> X
5841	if (match(V: Op1, P: m_FPOne()))
5842	return Op0;
5843
5844	// 0 / X -> 0
5845	// Requires that NaNs are off (X could be zero) and signed zeroes are
5846	// ignored (X could be positive or negative, so the output sign is unknown).
5847	if (FMF.noNaNs() && FMF.noSignedZeros() && match(V: Op0, P: m_AnyZeroFP()))
5848	return ConstantFP::getZero(Ty: Op0->getType());
5849
5850	if (FMF.noNaNs()) {
5851	// X / X -> 1.0 is legal when NaNs are ignored.
5852	// We can ignore infinities because INF/INF is NaN.
5853	if (Op0 == Op1)
5854	return ConstantFP::get(Ty: Op0->getType(), V: 1.0);
5855
5856	// (X * Y) / Y --> X if we can reassociate to the above form.
5857	Value *X;
5858	if (FMF.allowReassoc() && match(V: Op0, P: m_c_FMul(L: m_Value(V&: X), R: m_Specific(V: Op1))))
5859	return X;
5860
5861	// -X / X -> -1.0 and
5862	// X / -X -> -1.0 are legal when NaNs are ignored.
5863	// We can ignore signed zeros because +-0.0/+-0.0 is NaN and ignored.
5864	if (match(V: Op0, P: m_FNegNSZ(X: m_Specific(V: Op1))) ||
5865	match(V: Op1, P: m_FNegNSZ(X: m_Specific(V: Op0))))
5866	return ConstantFP::get(Ty: Op0->getType(), V: -1.0);
5867
5868	// nnan ninf X / [-]0.0 -> poison
5869	if (FMF.noInfs() && match(V: Op1, P: m_AnyZeroFP()))
5870	return PoisonValue::get(T: Op1->getType());
5871	}
5872
5873	return nullptr;
5874	}
5875
5876	Value *llvm::simplifyFDivInst(Value *Op0, Value *Op1, FastMathFlags FMF,
5877	const SimplifyQuery &Q,
5878	fp::ExceptionBehavior ExBehavior,
5879	RoundingMode Rounding) {
5880	return ::simplifyFDivInst(Op0, Op1, FMF, Q, RecursionLimit, ExBehavior,
5881	Rounding);
5882	}
5883
5884	static Value *
5885	simplifyFRemInst(Value *Op0, Value *Op1, FastMathFlags FMF,
5886	const SimplifyQuery &Q, unsigned,
5887	fp::ExceptionBehavior ExBehavior = fp::ebIgnore,
5888	RoundingMode Rounding = RoundingMode::NearestTiesToEven) {
5889	if (isDefaultFPEnvironment(EB: ExBehavior, RM: Rounding))
5890	if (Constant *C = foldOrCommuteConstant(Opcode: Instruction::FRem, Op0, Op1, Q))
5891	return C;
5892
5893	if (Constant *C = simplifyFPOp(Ops: {Op0, Op1}, FMF, Q, ExBehavior, Rounding))
5894	return C;
5895
5896	if (!isDefaultFPEnvironment(EB: ExBehavior, RM: Rounding))
5897	return nullptr;
5898
5899	// Unlike fdiv, the result of frem always matches the sign of the dividend.
5900	// The constant match may include undef elements in a vector, so return a full
5901	// zero constant as the result.
5902	if (FMF.noNaNs()) {
5903	// +0 % X -> 0
5904	if (match(V: Op0, P: m_PosZeroFP()))
5905	return ConstantFP::getZero(Ty: Op0->getType());
5906	// -0 % X -> -0
5907	if (match(V: Op0, P: m_NegZeroFP()))
5908	return ConstantFP::getNegativeZero(Ty: Op0->getType());
5909	}
5910
5911	return nullptr;
5912	}
5913
5914	Value *llvm::simplifyFRemInst(Value *Op0, Value *Op1, FastMathFlags FMF,
5915	const SimplifyQuery &Q,
5916	fp::ExceptionBehavior ExBehavior,
5917	RoundingMode Rounding) {
5918	return ::simplifyFRemInst(Op0, Op1, FMF, Q, RecursionLimit, ExBehavior,
5919	Rounding);
5920	}
5921
5922	//=== Helper functions for higher up the class hierarchy.
5923
5924	/// Given the operand for a UnaryOperator, see if we can fold the result.
5925	/// If not, this returns null.
5926	static Value *simplifyUnOp(unsigned Opcode, Value *Op, const SimplifyQuery &Q,
5927	unsigned MaxRecurse) {
5928	switch (Opcode) {
5929	case Instruction::FNeg:
5930	return simplifyFNegInst(Op, FMF: FastMathFlags(), Q, MaxRecurse);
5931	default:
5932	llvm_unreachable("Unexpected opcode");
5933	}
5934	}
5935
5936	/// Given the operand for a UnaryOperator, see if we can fold the result.
5937	/// If not, this returns null.
5938	/// Try to use FastMathFlags when folding the result.
5939	static Value *simplifyFPUnOp(unsigned Opcode, Value *Op,
5940	const FastMathFlags &FMF, const SimplifyQuery &Q,
5941	unsigned MaxRecurse) {
5942	switch (Opcode) {
5943	case Instruction::FNeg:
5944	return simplifyFNegInst(Op, FMF, Q, MaxRecurse);
5945	default:
5946	return simplifyUnOp(Opcode, Op, Q, MaxRecurse);
5947	}
5948	}
5949
5950	Value *llvm::simplifyUnOp(unsigned Opcode, Value *Op, const SimplifyQuery &Q) {
5951	return ::simplifyUnOp(Opcode, Op, Q, MaxRecurse: RecursionLimit);
5952	}
5953
5954	Value *llvm::simplifyUnOp(unsigned Opcode, Value *Op, FastMathFlags FMF,
5955	const SimplifyQuery &Q) {
5956	return ::simplifyFPUnOp(Opcode, Op, FMF, Q, MaxRecurse: RecursionLimit);
5957	}
5958
5959	/// Given operands for a BinaryOperator, see if we can fold the result.
5960	/// If not, this returns null.
5961	static Value *simplifyBinOp(unsigned Opcode, Value *LHS, Value *RHS,
5962	const SimplifyQuery &Q, unsigned MaxRecurse) {
5963	switch (Opcode) {
5964	case Instruction::Add:
5965	return simplifyAddInst(Op0: LHS, Op1: RHS, /* IsNSW */ false, /* IsNUW */ false, Q,
5966	MaxRecurse);
5967	case Instruction::Sub:
5968	return simplifySubInst(Op0: LHS, Op1: RHS, /* IsNSW */ false, /* IsNUW */ false, Q,
5969	MaxRecurse);
5970	case Instruction::Mul:
5971	return simplifyMulInst(Op0: LHS, Op1: RHS, /* IsNSW */ false, /* IsNUW */ false, Q,
5972	MaxRecurse);
5973	case Instruction::SDiv:
5974	return simplifySDivInst(Op0: LHS, Op1: RHS, /* IsExact */ false, Q, MaxRecurse);
5975	case Instruction::UDiv:
5976	return simplifyUDivInst(Op0: LHS, Op1: RHS, /* IsExact */ false, Q, MaxRecurse);
5977	case Instruction::SRem:
5978	return simplifySRemInst(Op0: LHS, Op1: RHS, Q, MaxRecurse);
5979	case Instruction::URem:
5980	return simplifyURemInst(Op0: LHS, Op1: RHS, Q, MaxRecurse);
5981	case Instruction::Shl:
5982	return simplifyShlInst(Op0: LHS, Op1: RHS, /* IsNSW */ false, /* IsNUW */ false, Q,
5983	MaxRecurse);
5984	case Instruction::LShr:
5985	return simplifyLShrInst(Op0: LHS, Op1: RHS, /* IsExact */ false, Q, MaxRecurse);
5986	case Instruction::AShr:
5987	return simplifyAShrInst(Op0: LHS, Op1: RHS, /* IsExact */ false, Q, MaxRecurse);
5988	case Instruction::And:
5989	return simplifyAndInst(Op0: LHS, Op1: RHS, Q, MaxRecurse);
5990	case Instruction::Or:
5991	return simplifyOrInst(Op0: LHS, Op1: RHS, Q, MaxRecurse);
5992	case Instruction::Xor:
5993	return simplifyXorInst(Op0: LHS, Op1: RHS, Q, MaxRecurse);
5994	case Instruction::FAdd:
5995	return simplifyFAddInst(Op0: LHS, Op1: RHS, FMF: FastMathFlags(), Q, MaxRecurse);
5996	case Instruction::FSub:
5997	return simplifyFSubInst(Op0: LHS, Op1: RHS, FMF: FastMathFlags(), Q, MaxRecurse);
5998	case Instruction::FMul:
5999	return simplifyFMulInst(Op0: LHS, Op1: RHS, FMF: FastMathFlags(), Q, MaxRecurse);
6000	case Instruction::FDiv:
6001	return simplifyFDivInst(Op0: LHS, Op1: RHS, FMF: FastMathFlags(), Q, MaxRecurse);
6002	case Instruction::FRem:
6003	return simplifyFRemInst(Op0: LHS, Op1: RHS, FMF: FastMathFlags(), Q, MaxRecurse);
6004	default:
6005	llvm_unreachable("Unexpected opcode");
6006	}
6007	}
6008
6009	/// Given operands for a BinaryOperator, see if we can fold the result.
6010	/// If not, this returns null.
6011	/// Try to use FastMathFlags when folding the result.
6012	static Value *simplifyBinOp(unsigned Opcode, Value *LHS, Value *RHS,
6013	const FastMathFlags &FMF, const SimplifyQuery &Q,
6014	unsigned MaxRecurse) {
6015	switch (Opcode) {
6016	case Instruction::FAdd:
6017	return simplifyFAddInst(Op0: LHS, Op1: RHS, FMF, Q, MaxRecurse);
6018	case Instruction::FSub:
6019	return simplifyFSubInst(Op0: LHS, Op1: RHS, FMF, Q, MaxRecurse);
6020	case Instruction::FMul:
6021	return simplifyFMulInst(Op0: LHS, Op1: RHS, FMF, Q, MaxRecurse);
6022	case Instruction::FDiv:
6023	return simplifyFDivInst(Op0: LHS, Op1: RHS, FMF, Q, MaxRecurse);
6024	default:
6025	return simplifyBinOp(Opcode, LHS, RHS, Q, MaxRecurse);
6026	}
6027	}
6028
6029	Value *llvm::simplifyBinOp(unsigned Opcode, Value *LHS, Value *RHS,
6030	const SimplifyQuery &Q) {
6031	return ::simplifyBinOp(Opcode, LHS, RHS, Q, MaxRecurse: RecursionLimit);
6032	}
6033
6034	Value *llvm::simplifyBinOp(unsigned Opcode, Value *LHS, Value *RHS,
6035	FastMathFlags FMF, const SimplifyQuery &Q) {
6036	return ::simplifyBinOp(Opcode, LHS, RHS, FMF, Q, MaxRecurse: RecursionLimit);
6037	}
6038
6039	/// Given operands for a CmpInst, see if we can fold the result.
6040	static Value *simplifyCmpInst(unsigned Predicate, Value *LHS, Value *RHS,
6041	const SimplifyQuery &Q, unsigned MaxRecurse) {
6042	if (CmpInst::isIntPredicate(P: (CmpInst::Predicate)Predicate))
6043	return simplifyICmpInst(Predicate, LHS, RHS, Q, MaxRecurse);
6044	return simplifyFCmpInst(Predicate, LHS, RHS, FMF: FastMathFlags(), Q, MaxRecurse);
6045	}
6046
6047	Value *llvm::simplifyCmpInst(unsigned Predicate, Value *LHS, Value *RHS,
6048	const SimplifyQuery &Q) {
6049	return ::simplifyCmpInst(Predicate, LHS, RHS, Q, MaxRecurse: RecursionLimit);
6050	}
6051
6052	static bool isIdempotent(Intrinsic::ID ID) {
6053	switch (ID) {
6054	default:
6055	return false;
6056
6057	// Unary idempotent: f(f(x)) = f(x)
6058	case Intrinsic::fabs:
6059	case Intrinsic::floor:
6060	case Intrinsic::ceil:
6061	case Intrinsic::trunc:
6062	case Intrinsic::rint:
6063	case Intrinsic::nearbyint:
6064	case Intrinsic::round:
6065	case Intrinsic::roundeven:
6066	case Intrinsic::canonicalize:
6067	case Intrinsic::arithmetic_fence:
6068	return true;
6069	}
6070	}
6071
6072	/// Return true if the intrinsic rounds a floating-point value to an integral
6073	/// floating-point value (not an integer type).
6074	static bool removesFPFraction(Intrinsic::ID ID) {
6075	switch (ID) {
6076	default:
6077	return false;
6078
6079	case Intrinsic::floor:
6080	case Intrinsic::ceil:
6081	case Intrinsic::trunc:
6082	case Intrinsic::rint:
6083	case Intrinsic::nearbyint:
6084	case Intrinsic::round:
6085	case Intrinsic::roundeven:
6086	return true;
6087	}
6088	}
6089
6090	static Value *simplifyRelativeLoad(Constant *Ptr, Constant *Offset,
6091	const DataLayout &DL) {
6092	GlobalValue *PtrSym;
6093	APInt PtrOffset;
6094	if (!IsConstantOffsetFromGlobal(C: Ptr, GV&: PtrSym, Offset&: PtrOffset, DL))
6095	return nullptr;
6096
6097	Type *Int32Ty = Type::getInt32Ty(C&: Ptr->getContext());
6098
6099	auto *OffsetConstInt = dyn_cast<ConstantInt>(Val: Offset);
6100	if (!OffsetConstInt || OffsetConstInt->getBitWidth() > 64)
6101	return nullptr;
6102
6103	APInt OffsetInt = OffsetConstInt->getValue().sextOrTrunc(
6104	width: DL.getIndexTypeSizeInBits(Ty: Ptr->getType()));
6105	if (OffsetInt.srem(RHS: 4) != 0)
6106	return nullptr;
6107
6108	Constant *Loaded =
6109	ConstantFoldLoadFromConstPtr(C: Ptr, Ty: Int32Ty, Offset: std::move(OffsetInt), DL);
6110	if (!Loaded)
6111	return nullptr;
6112
6113	auto *LoadedCE = dyn_cast<ConstantExpr>(Val: Loaded);
6114	if (!LoadedCE)
6115	return nullptr;
6116
6117	if (LoadedCE->getOpcode() == Instruction::Trunc) {
6118	LoadedCE = dyn_cast<ConstantExpr>(Val: LoadedCE->getOperand(i_nocapture: 0));
6119	if (!LoadedCE)
6120	return nullptr;
6121	}
6122
6123	if (LoadedCE->getOpcode() != Instruction::Sub)
6124	return nullptr;
6125
6126	auto *LoadedLHS = dyn_cast<ConstantExpr>(Val: LoadedCE->getOperand(i_nocapture: 0));
6127	if (!LoadedLHS || LoadedLHS->getOpcode() != Instruction::PtrToInt)
6128	return nullptr;
6129	auto *LoadedLHSPtr = LoadedLHS->getOperand(i_nocapture: 0);
6130
6131	Constant *LoadedRHS = LoadedCE->getOperand(i_nocapture: 1);
6132	GlobalValue *LoadedRHSSym;
6133	APInt LoadedRHSOffset;
6134	if (!IsConstantOffsetFromGlobal(C: LoadedRHS, GV&: LoadedRHSSym, Offset&: LoadedRHSOffset,
6135	DL) ||
6136	PtrSym != LoadedRHSSym || PtrOffset != LoadedRHSOffset)
6137	return nullptr;
6138
6139	return LoadedLHSPtr;
6140	}
6141
6142	// TODO: Need to pass in FastMathFlags
6143	static Value *simplifyLdexp(Value *Op0, Value *Op1, const SimplifyQuery &Q,
6144	bool IsStrict) {
6145	// ldexp(poison, x) -> poison
6146	// ldexp(x, poison) -> poison
6147	if (isa<PoisonValue>(Val: Op0) || isa<PoisonValue>(Val: Op1))
6148	return Op0;
6149
6150	// ldexp(undef, x) -> nan
6151	if (Q.isUndefValue(V: Op0))
6152	return ConstantFP::getNaN(Ty: Op0->getType());
6153
6154	if (!IsStrict) {
6155	// TODO: Could insert a canonicalize for strict
6156
6157	// ldexp(x, undef) -> x
6158	if (Q.isUndefValue(V: Op1))
6159	return Op0;
6160	}
6161
6162	const APFloat *C = nullptr;
6163	match(V: Op0, P: PatternMatch::m_APFloat(Res&: C));
6164
6165	// These cases should be safe, even with strictfp.
6166	// ldexp(0.0, x) -> 0.0
6167	// ldexp(-0.0, x) -> -0.0
6168	// ldexp(inf, x) -> inf
6169	// ldexp(-inf, x) -> -inf
6170	if (C && (C->isZero() || C->isInfinity()))
6171	return Op0;
6172
6173	// These are canonicalization dropping, could do it if we knew how we could
6174	// ignore denormal flushes and target handling of nan payload bits.
6175	if (IsStrict)
6176	return nullptr;
6177
6178	// TODO: Could quiet this with strictfp if the exception mode isn't strict.
6179	if (C && C->isNaN())
6180	return ConstantFP::get(Ty: Op0->getType(), V: C->makeQuiet());
6181
6182	// ldexp(x, 0) -> x
6183
6184	// TODO: Could fold this if we know the exception mode isn't
6185	// strict, we know the denormal mode and other target modes.
6186	if (match(V: Op1, P: PatternMatch::m_ZeroInt()))
6187	return Op0;
6188
6189	return nullptr;
6190	}
6191
6192	static Value *simplifyUnaryIntrinsic(Function *F, Value *Op0,
6193	const SimplifyQuery &Q,
6194	const CallBase *Call) {
6195	// Idempotent functions return the same result when called repeatedly.
6196	Intrinsic::ID IID = F->getIntrinsicID();
6197	if (isIdempotent(ID: IID))
6198	if (auto *II = dyn_cast<IntrinsicInst>(Val: Op0))
6199	if (II->getIntrinsicID() == IID)
6200	return II;
6201
6202	if (removesFPFraction(ID: IID)) {
6203	// Converting from int or calling a rounding function always results in a
6204	// finite integral number or infinity. For those inputs, rounding functions
6205	// always return the same value, so the (2nd) rounding is eliminated. Ex:
6206	// floor (sitofp x) -> sitofp x
6207	// round (ceil x) -> ceil x
6208	auto *II = dyn_cast<IntrinsicInst>(Val: Op0);
6209	if ((II && removesFPFraction(ID: II->getIntrinsicID())) ||
6210	match(V: Op0, P: m_SIToFP(Op: m_Value())) || match(V: Op0, P: m_UIToFP(Op: m_Value())))
6211	return Op0;
6212	}
6213
6214	Value *X;
6215	switch (IID) {
6216	case Intrinsic::fabs:
6217	if (computeKnownFPSignBit(V: Op0, /*Depth=*/0, SQ: Q) == false)
6218	return Op0;
6219	break;
6220	case Intrinsic::bswap:
6221	// bswap(bswap(x)) -> x
6222	if (match(V: Op0, P: m_BSwap(Op0: m_Value(V&: X))))
6223	return X;
6224	break;
6225	case Intrinsic::bitreverse:
6226	// bitreverse(bitreverse(x)) -> x
6227	if (match(V: Op0, P: m_BitReverse(Op0: m_Value(V&: X))))
6228	return X;
6229	break;
6230	case Intrinsic::ctpop: {
6231	// ctpop(X) -> 1 iff X is non-zero power of 2.
6232	if (isKnownToBeAPowerOfTwo(V: Op0, DL: Q.DL, /*OrZero*/ false, Depth: 0, AC: Q.AC, CxtI: Q.CxtI,
6233	DT: Q.DT))
6234	return ConstantInt::get(Ty: Op0->getType(), V: 1);
6235	// If everything but the lowest bit is zero, that bit is the pop-count. Ex:
6236	// ctpop(and X, 1) --> and X, 1
6237	unsigned BitWidth = Op0->getType()->getScalarSizeInBits();
6238	if (MaskedValueIsZero(V: Op0, Mask: APInt::getHighBitsSet(numBits: BitWidth, hiBitsSet: BitWidth - 1),
6239	DL: Q))
6240	return Op0;
6241	break;
6242	}
6243	case Intrinsic::exp:
6244	// exp(log(x)) -> x
6245	if (Call->hasAllowReassoc() &&
6246	match(Op0, m_Intrinsic<Intrinsic::log>(m_Value(X))))
6247	return X;
6248	break;
6249	case Intrinsic::exp2:
6250	// exp2(log2(x)) -> x
6251	if (Call->hasAllowReassoc() &&
6252	match(Op0, m_Intrinsic<Intrinsic::log2>(m_Value(V&: X))))
6253	return X;
6254	break;
6255	case Intrinsic::exp10:
6256	// exp10(log10(x)) -> x
6257	if (Call->hasAllowReassoc() &&
6258	match(Op0, m_Intrinsic<Intrinsic::log10>(m_Value(X))))
6259	return X;
6260	break;
6261	case Intrinsic::log:
6262	// log(exp(x)) -> x
6263	if (Call->hasAllowReassoc() &&
6264	match(Op0, m_Intrinsic<Intrinsic::exp>(m_Value(X))))
6265	return X;
6266	break;
6267	case Intrinsic::log2:
6268	// log2(exp2(x)) -> x
6269	if (Call->hasAllowReassoc() &&
6270	(match(Op0, m_Intrinsic<Intrinsic::exp2>(m_Value(X))) ||
6271	match(Op0,
6272	m_Intrinsic<Intrinsic::pow>(m_SpecificFP(2.0), m_Value(X)))))
6273	return X;
6274	break;
6275	case Intrinsic::log10:
6276	// log10(pow(10.0, x)) -> x
6277	// log10(exp10(x)) -> x
6278	if (Call->hasAllowReassoc() &&
6279	(match(Op0, m_Intrinsic<Intrinsic::exp10>(m_Value(X))) ||
6280	match(Op0,
6281	m_Intrinsic<Intrinsic::pow>(m_SpecificFP(10.0), m_Value(X)))))
6282	return X;
6283	break;
6284	case Intrinsic::experimental_vector_reverse:
6285	// experimental.vector.reverse(experimental.vector.reverse(x)) -> x
6286	if (match(V: Op0, P: m_VecReverse(Op0: m_Value(V&: X))))
6287	return X;
6288	// experimental.vector.reverse(splat(X)) -> splat(X)
6289	if (isSplatValue(V: Op0))
6290	return Op0;
6291	break;
6292	case Intrinsic::frexp: {
6293	// Frexp is idempotent with the added complication of the struct return.
6294	if (match(V: Op0, P: m_ExtractValue<0>(V: m_Value(V&: X)))) {
6295	if (match(X, m_Intrinsic<Intrinsic::frexp>(m_Value())))
6296	return X;
6297	}
6298
6299	break;
6300	}
6301	default:
6302	break;
6303	}
6304
6305	return nullptr;
6306	}
6307
6308	/// Given a min/max intrinsic, see if it can be removed based on having an
6309	/// operand that is another min/max intrinsic with shared operand(s). The caller
6310	/// is expected to swap the operand arguments to handle commutation.
6311	static Value *foldMinMaxSharedOp(Intrinsic::ID IID, Value *Op0, Value *Op1) {
6312	Value *X, *Y;
6313	if (!match(V: Op0, P: m_MaxOrMin(L: m_Value(V&: X), R: m_Value(V&: Y))))
6314	return nullptr;
6315
6316	auto *MM0 = dyn_cast<IntrinsicInst>(Val: Op0);
6317	if (!MM0)
6318	return nullptr;
6319	Intrinsic::ID IID0 = MM0->getIntrinsicID();
6320
6321	if (Op1 == X || Op1 == Y ||
6322	match(V: Op1, P: m_c_MaxOrMin(L: m_Specific(V: X), R: m_Specific(V: Y)))) {
6323	// max (max X, Y), X --> max X, Y
6324	if (IID0 == IID)
6325	return MM0;
6326	// max (min X, Y), X --> X
6327	if (IID0 == getInverseMinMaxIntrinsic(MinMaxID: IID))
6328	return Op1;
6329	}
6330	return nullptr;
6331	}
6332
6333	/// Given a min/max intrinsic, see if it can be removed based on having an
6334	/// operand that is another min/max intrinsic with shared operand(s). The caller
6335	/// is expected to swap the operand arguments to handle commutation.
6336	static Value *foldMinimumMaximumSharedOp(Intrinsic::ID IID, Value *Op0,
6337	Value *Op1) {
6338	assert((IID == Intrinsic::maxnum || IID == Intrinsic::minnum ||
6339	IID == Intrinsic::maximum || IID == Intrinsic::minimum) &&
6340	"Unsupported intrinsic");
6341
6342	auto *M0 = dyn_cast<IntrinsicInst>(Val: Op0);
6343	// If Op0 is not the same intrinsic as IID, do not process.
6344	// This is a difference with integer min/max handling. We do not process the
6345	// case like max(min(X,Y),min(X,Y)) => min(X,Y). But it can be handled by GVN.
6346	if (!M0 || M0->getIntrinsicID() != IID)
6347	return nullptr;
6348	Value *X0 = M0->getOperand(i_nocapture: 0);
6349	Value *Y0 = M0->getOperand(i_nocapture: 1);
6350	// Simple case, m(m(X,Y), X) => m(X, Y)
6351	// m(m(X,Y), Y) => m(X, Y)
6352	// For minimum/maximum, X is NaN => m(NaN, Y) == NaN and m(NaN, NaN) == NaN.
6353	// For minimum/maximum, Y is NaN => m(X, NaN) == NaN and m(NaN, NaN) == NaN.
6354	// For minnum/maxnum, X is NaN => m(NaN, Y) == Y and m(Y, Y) == Y.
6355	// For minnum/maxnum, Y is NaN => m(X, NaN) == X and m(X, NaN) == X.
6356	if (X0 == Op1 || Y0 == Op1)
6357	return M0;
6358
6359	auto *M1 = dyn_cast<IntrinsicInst>(Val: Op1);
6360	if (!M1)
6361	return nullptr;
6362	Value *X1 = M1->getOperand(i_nocapture: 0);
6363	Value *Y1 = M1->getOperand(i_nocapture: 1);
6364	Intrinsic::ID IID1 = M1->getIntrinsicID();
6365	// we have a case m(m(X,Y),m'(X,Y)) taking into account m' is commutative.
6366	// if m' is m or inversion of m => m(m(X,Y),m'(X,Y)) == m(X,Y).
6367	// For minimum/maximum, X is NaN => m(NaN,Y) == m'(NaN, Y) == NaN.
6368	// For minimum/maximum, Y is NaN => m(X,NaN) == m'(X, NaN) == NaN.
6369	// For minnum/maxnum, X is NaN => m(NaN,Y) == m'(NaN, Y) == Y.
6370	// For minnum/maxnum, Y is NaN => m(X,NaN) == m'(X, NaN) == X.
6371	if ((X0 == X1 && Y0 == Y1) || (X0 == Y1 && Y0 == X1))
6372	if (IID1 == IID || getInverseMinMaxIntrinsic(MinMaxID: IID1) == IID)
6373	return M0;
6374
6375	return nullptr;
6376	}
6377
6378	Value *llvm::simplifyBinaryIntrinsic(Intrinsic::ID IID, Type *ReturnType,
6379	Value *Op0, Value *Op1,
6380	const SimplifyQuery &Q,
6381	const CallBase *Call) {
6382	unsigned BitWidth = ReturnType->getScalarSizeInBits();
6383	switch (IID) {
6384	case Intrinsic::abs:
6385	// abs(abs(x)) -> abs(x). We don't need to worry about the nsw arg here.
6386	// It is always ok to pick the earlier abs. We'll just lose nsw if its only
6387	// on the outer abs.
6388	if (match(Op0, m_Intrinsic<Intrinsic::abs>(m_Value(), m_Value())))
6389	return Op0;
6390	break;
6391
6392	case Intrinsic::cttz: {
6393	Value *X;
6394	if (match(V: Op0, P: m_Shl(L: m_One(), R: m_Value(V&: X))))
6395	return X;
6396	break;
6397	}
6398	case Intrinsic::ctlz: {
6399	Value *X;
6400	if (match(V: Op0, P: m_LShr(L: m_Negative(), R: m_Value(V&: X))))
6401	return X;
6402	if (match(V: Op0, P: m_AShr(L: m_Negative(), R: m_Value())))
6403	return Constant::getNullValue(Ty: ReturnType);
6404	break;
6405	}
6406	case Intrinsic::ptrmask: {
6407	if (isa<PoisonValue>(Val: Op0) || isa<PoisonValue>(Val: Op1))
6408	return PoisonValue::get(T: Op0->getType());
6409
6410	// NOTE: We can't apply this simplifications based on the value of Op1
6411	// because we need to preserve provenance.
6412	if (Q.isUndefValue(V: Op0) || match(V: Op0, P: m_Zero()))
6413	return Constant::getNullValue(Ty: Op0->getType());
6414
6415	assert(Op1->getType()->getScalarSizeInBits() ==
6416	Q.DL.getIndexTypeSizeInBits(Op0->getType()) &&
6417	"Invalid mask width");
6418	// If index-width (mask size) is less than pointer-size then mask is
6419	// 1-extended.
6420	if (match(V: Op1, P: m_PtrToInt(Op: m_Specific(V: Op0))))
6421	return Op0;
6422
6423	// NOTE: We may have attributes associated with the return value of the
6424	// llvm.ptrmask intrinsic that will be lost when we just return the
6425	// operand. We should try to preserve them.
6426	if (match(V: Op1, P: m_AllOnes()) || Q.isUndefValue(V: Op1))
6427	return Op0;
6428
6429	Constant *C;
6430	if (match(V: Op1, P: m_ImmConstant(C))) {
6431	KnownBits PtrKnown = computeKnownBits(V: Op0, /*Depth=*/0, Q);
6432	// See if we only masking off bits we know are already zero due to
6433	// alignment.
6434	APInt IrrelevantPtrBits =
6435	PtrKnown.Zero.zextOrTrunc(width: C->getType()->getScalarSizeInBits());
6436	C = ConstantFoldBinaryOpOperands(
6437	Opcode: Instruction::Or, LHS: C, RHS: ConstantInt::get(Ty: C->getType(), V: IrrelevantPtrBits),
6438	DL: Q.DL);
6439	if (C != nullptr && C->isAllOnesValue())
6440	return Op0;
6441	}
6442	break;
6443	}
6444	case Intrinsic::smax:
6445	case Intrinsic::smin:
6446	case Intrinsic::umax:
6447	case Intrinsic::umin: {
6448	// If the arguments are the same, this is a no-op.
6449	if (Op0 == Op1)
6450	return Op0;
6451
6452	// Canonicalize immediate constant operand as Op1.
6453	if (match(V: Op0, P: m_ImmConstant()))
6454	std::swap(a&: Op0, b&: Op1);
6455
6456	// Assume undef is the limit value.
6457	if (Q.isUndefValue(V: Op1))
6458	return ConstantInt::get(
6459	Ty: ReturnType, V: MinMaxIntrinsic::getSaturationPoint(ID: IID, numBits: BitWidth));
6460
6461	const APInt *C;
6462	if (match(V: Op1, P: m_APIntAllowPoison(Res&: C))) {
6463	// Clamp to limit value. For example:
6464	// umax(i8 %x, i8 255) --> 255
6465	if (*C == MinMaxIntrinsic::getSaturationPoint(ID: IID, numBits: BitWidth))
6466	return ConstantInt::get(Ty: ReturnType, V: *C);
6467
6468	// If the constant op is the opposite of the limit value, the other must
6469	// be larger/smaller or equal. For example:
6470	// umin(i8 %x, i8 255) --> %x
6471	if (*C == MinMaxIntrinsic::getSaturationPoint(
6472	ID: getInverseMinMaxIntrinsic(MinMaxID: IID), numBits: BitWidth))
6473	return Op0;
6474
6475	// Remove nested call if constant operands allow it. Example:
6476	// max (max X, 7), 5 -> max X, 7
6477	auto *MinMax0 = dyn_cast<IntrinsicInst>(Val: Op0);
6478	if (MinMax0 && MinMax0->getIntrinsicID() == IID) {
6479	// TODO: loosen undef/splat restrictions for vector constants.
6480	Value *M00 = MinMax0->getOperand(i_nocapture: 0), *M01 = MinMax0->getOperand(i_nocapture: 1);
6481	const APInt *InnerC;
6482	if ((match(V: M00, P: m_APInt(Res&: InnerC)) || match(V: M01, P: m_APInt(Res&: InnerC))) &&
6483	ICmpInst::compare(LHS: *InnerC, RHS: *C,
6484	Pred: ICmpInst::getNonStrictPredicate(
6485	pred: MinMaxIntrinsic::getPredicate(ID: IID))))
6486	return Op0;
6487	}
6488	}
6489
6490	if (Value *V = foldMinMaxSharedOp(IID, Op0, Op1))
6491	return V;
6492	if (Value *V = foldMinMaxSharedOp(IID, Op0: Op1, Op1: Op0))
6493	return V;
6494
6495	ICmpInst::Predicate Pred =
6496	ICmpInst::getNonStrictPredicate(pred: MinMaxIntrinsic::getPredicate(ID: IID));
6497	if (isICmpTrue(Pred, LHS: Op0, RHS: Op1, Q: Q.getWithoutUndef(), MaxRecurse: RecursionLimit))
6498	return Op0;
6499	if (isICmpTrue(Pred, LHS: Op1, RHS: Op0, Q: Q.getWithoutUndef(), MaxRecurse: RecursionLimit))
6500	return Op1;
6501
6502	break;
6503	}
6504	case Intrinsic::usub_with_overflow:
6505	case Intrinsic::ssub_with_overflow:
6506	// X - X -> { 0, false }
6507	// X - undef -> { 0, false }
6508	// undef - X -> { 0, false }
6509	if (Op0 == Op1 || Q.isUndefValue(V: Op0) || Q.isUndefValue(V: Op1))
6510	return Constant::getNullValue(Ty: ReturnType);
6511	break;
6512	case Intrinsic::uadd_with_overflow:
6513	case Intrinsic::sadd_with_overflow:
6514	// X + undef -> { -1, false }
6515	// undef + x -> { -1, false }
6516	if (Q.isUndefValue(V: Op0) || Q.isUndefValue(V: Op1)) {
6517	return ConstantStruct::get(
6518	T: cast<StructType>(Val: ReturnType),
6519	V: {Constant::getAllOnesValue(Ty: ReturnType->getStructElementType(N: 0)),
6520	Constant::getNullValue(Ty: ReturnType->getStructElementType(N: 1))});
6521	}
6522	break;
6523	case Intrinsic::umul_with_overflow:
6524	case Intrinsic::smul_with_overflow:
6525	// 0 * X -> { 0, false }
6526	// X * 0 -> { 0, false }
6527	if (match(V: Op0, P: m_Zero()) || match(V: Op1, P: m_Zero()))
6528	return Constant::getNullValue(Ty: ReturnType);
6529	// undef * X -> { 0, false }
6530	// X * undef -> { 0, false }
6531	if (Q.isUndefValue(V: Op0) || Q.isUndefValue(V: Op1))
6532	return Constant::getNullValue(Ty: ReturnType);
6533	break;
6534	case Intrinsic::uadd_sat:
6535	// sat(MAX + X) -> MAX
6536	// sat(X + MAX) -> MAX
6537	if (match(V: Op0, P: m_AllOnes()) || match(V: Op1, P: m_AllOnes()))
6538	return Constant::getAllOnesValue(Ty: ReturnType);
6539	[[fallthrough]];
6540	case Intrinsic::sadd_sat:
6541	// sat(X + undef) -> -1
6542	// sat(undef + X) -> -1
6543	// For unsigned: Assume undef is MAX, thus we saturate to MAX (-1).
6544	// For signed: Assume undef is ~X, in which case X + ~X = -1.
6545	if (Q.isUndefValue(V: Op0) || Q.isUndefValue(V: Op1))
6546	return Constant::getAllOnesValue(Ty: ReturnType);
6547
6548	// X + 0 -> X
6549	if (match(V: Op1, P: m_Zero()))
6550	return Op0;
6551	// 0 + X -> X
6552	if (match(V: Op0, P: m_Zero()))
6553	return Op1;
6554	break;
6555	case Intrinsic::usub_sat:
6556	// sat(0 - X) -> 0, sat(X - MAX) -> 0
6557	if (match(V: Op0, P: m_Zero()) || match(V: Op1, P: m_AllOnes()))
6558	return Constant::getNullValue(Ty: ReturnType);
6559	[[fallthrough]];
6560	case Intrinsic::ssub_sat:
6561	// X - X -> 0, X - undef -> 0, undef - X -> 0
6562	if (Op0 == Op1 || Q.isUndefValue(V: Op0) || Q.isUndefValue(V: Op1))
6563	return Constant::getNullValue(Ty: ReturnType);
6564	// X - 0 -> X
6565	if (match(V: Op1, P: m_Zero()))
6566	return Op0;
6567	break;
6568	case Intrinsic::load_relative:
6569	if (auto *C0 = dyn_cast<Constant>(Val: Op0))
6570	if (auto *C1 = dyn_cast<Constant>(Val: Op1))
6571	return simplifyRelativeLoad(Ptr: C0, Offset: C1, DL: Q.DL);
6572	break;
6573	case Intrinsic::powi:
6574	if (auto *Power = dyn_cast<ConstantInt>(Val: Op1)) {
6575	// powi(x, 0) -> 1.0
6576	if (Power->isZero())
6577	return ConstantFP::get(Ty: Op0->getType(), V: 1.0);
6578	// powi(x, 1) -> x
6579	if (Power->isOne())
6580	return Op0;
6581	}
6582	break;
6583	case Intrinsic::ldexp:
6584	return simplifyLdexp(Op0, Op1, Q, IsStrict: false);
6585	case Intrinsic::copysign:
6586	// copysign X, X --> X
6587	if (Op0 == Op1)
6588	return Op0;
6589	// copysign -X, X --> X
6590	// copysign X, -X --> -X
6591	if (match(V: Op0, P: m_FNeg(X: m_Specific(V: Op1))) ||
6592	match(V: Op1, P: m_FNeg(X: m_Specific(V: Op0))))
6593	return Op1;
6594	break;
6595	case Intrinsic::is_fpclass: {
6596	if (isa<PoisonValue>(Val: Op0))
6597	return PoisonValue::get(T: ReturnType);
6598
6599	uint64_t Mask = cast<ConstantInt>(Val: Op1)->getZExtValue();
6600	// If all tests are made, it doesn't matter what the value is.
6601	if ((Mask & fcAllFlags) == fcAllFlags)
6602	return ConstantInt::get(Ty: ReturnType, V: true);
6603	if ((Mask & fcAllFlags) == 0)
6604	return ConstantInt::get(Ty: ReturnType, V: false);
6605	if (Q.isUndefValue(V: Op0))
6606	return UndefValue::get(T: ReturnType);
6607	break;
6608	}
6609	case Intrinsic::maxnum:
6610	case Intrinsic::minnum:
6611	case Intrinsic::maximum:
6612	case Intrinsic::minimum: {
6613	// If the arguments are the same, this is a no-op.
6614	if (Op0 == Op1)
6615	return Op0;
6616
6617	// Canonicalize constant operand as Op1.
6618	if (isa<Constant>(Val: Op0))
6619	std::swap(a&: Op0, b&: Op1);
6620
6621	// If an argument is undef, return the other argument.
6622	if (Q.isUndefValue(V: Op1))
6623	return Op0;
6624
6625	bool PropagateNaN = IID == Intrinsic::minimum || IID == Intrinsic::maximum;
6626	bool IsMin = IID == Intrinsic::minimum || IID == Intrinsic::minnum;
6627
6628	// minnum(X, nan) -> X
6629	// maxnum(X, nan) -> X
6630	// minimum(X, nan) -> nan
6631	// maximum(X, nan) -> nan
6632	if (match(V: Op1, P: m_NaN()))
6633	return PropagateNaN ? propagateNaN(In: cast<Constant>(Val: Op1)) : Op0;
6634
6635	// In the following folds, inf can be replaced with the largest finite
6636	// float, if the ninf flag is set.
6637	const APFloat *C;
6638	if (match(V: Op1, P: m_APFloat(Res&: C)) &&
6639	(C->isInfinity() || (Call && Call->hasNoInfs() && C->isLargest()))) {
6640	// minnum(X, -inf) -> -inf
6641	// maxnum(X, +inf) -> +inf
6642	// minimum(X, -inf) -> -inf if nnan
6643	// maximum(X, +inf) -> +inf if nnan
6644	if (C->isNegative() == IsMin &&
6645	(!PropagateNaN || (Call && Call->hasNoNaNs())))
6646	return ConstantFP::get(Ty: ReturnType, V: *C);
6647
6648	// minnum(X, +inf) -> X if nnan
6649	// maxnum(X, -inf) -> X if nnan
6650	// minimum(X, +inf) -> X
6651	// maximum(X, -inf) -> X
6652	if (C->isNegative() != IsMin &&
6653	(PropagateNaN || (Call && Call->hasNoNaNs())))
6654	return Op0;
6655	}
6656
6657	// Min/max of the same operation with common operand:
6658	// m(m(X, Y)), X --> m(X, Y) (4 commuted variants)
6659	if (Value *V = foldMinimumMaximumSharedOp(IID, Op0, Op1))
6660	return V;
6661	if (Value *V = foldMinimumMaximumSharedOp(IID, Op0: Op1, Op1: Op0))
6662	return V;
6663
6664	break;
6665	}
6666	case Intrinsic::vector_extract: {
6667	// (extract_vector (insert_vector _, X, 0), 0) -> X
6668	unsigned IdxN = cast<ConstantInt>(Val: Op1)->getZExtValue();
6669	Value *X = nullptr;
6670	if (match(Op0, m_Intrinsic<Intrinsic::vector_insert>(m_Value(), m_Value(X),
6671	m_Zero())) &&
6672	IdxN == 0 && X->getType() == ReturnType)
6673	return X;
6674
6675	break;
6676	}
6677	default:
6678	break;
6679	}
6680
6681	return nullptr;
6682	}
6683
6684	static Value *simplifyIntrinsic(CallBase *Call, Value *Callee,
6685	ArrayRef<Value *> Args,
6686	const SimplifyQuery &Q) {
6687	// Operand bundles should not be in Args.
6688	assert(Call->arg_size() == Args.size());
6689	unsigned NumOperands = Args.size();
6690	Function *F = cast<Function>(Val: Callee);
6691	Intrinsic::ID IID = F->getIntrinsicID();
6692
6693	// Most of the intrinsics with no operands have some kind of side effect.
6694	// Don't simplify.
6695	if (!NumOperands) {
6696	switch (IID) {
6697	case Intrinsic::vscale: {
6698	Type *RetTy = F->getReturnType();
6699	ConstantRange CR = getVScaleRange(F: Call->getFunction(), BitWidth: 64);
6700	if (const APInt *C = CR.getSingleElement())
6701	return ConstantInt::get(Ty: RetTy, V: C->getZExtValue());
6702	return nullptr;
6703	}
6704	default:
6705	return nullptr;
6706	}
6707	}
6708
6709	if (NumOperands == 1)
6710	return simplifyUnaryIntrinsic(F, Op0: Args[0], Q, Call);
6711
6712	if (NumOperands == 2)
6713	return simplifyBinaryIntrinsic(IID, ReturnType: F->getReturnType(), Op0: Args[0], Op1: Args[1], Q,
6714	Call);
6715
6716	// Handle intrinsics with 3 or more arguments.
6717	switch (IID) {
6718	case Intrinsic::masked_load:
6719	case Intrinsic::masked_gather: {
6720	Value *MaskArg = Args[2];
6721	Value *PassthruArg = Args[3];
6722	// If the mask is all zeros or undef, the "passthru" argument is the result.
6723	if (maskIsAllZeroOrUndef(Mask: MaskArg))
6724	return PassthruArg;
6725	return nullptr;
6726	}
6727	case Intrinsic::fshl:
6728	case Intrinsic::fshr: {
6729	Value *Op0 = Args[0], *Op1 = Args[1], *ShAmtArg = Args[2];
6730
6731	// If both operands are undef, the result is undef.
6732	if (Q.isUndefValue(V: Op0) && Q.isUndefValue(V: Op1))
6733	return UndefValue::get(T: F->getReturnType());
6734
6735	// If shift amount is undef, assume it is zero.
6736	if (Q.isUndefValue(ShAmtArg))
6737	return Args[IID == Intrinsic::fshl ? 0 : 1];
6738
6739	const APInt *ShAmtC;
6740	if (match(V: ShAmtArg, P: m_APInt(Res&: ShAmtC))) {
6741	// If there's effectively no shift, return the 1st arg or 2nd arg.
6742	APInt BitWidth = APInt(ShAmtC->getBitWidth(), ShAmtC->getBitWidth());
6743	if (ShAmtC->urem(BitWidth).isZero())
6744	return Args[IID == Intrinsic::fshl ? 0 : 1];
6745	}
6746
6747	// Rotating zero by anything is zero.
6748	if (match(V: Op0, P: m_Zero()) && match(V: Op1, P: m_Zero()))
6749	return ConstantInt::getNullValue(Ty: F->getReturnType());
6750
6751	// Rotating -1 by anything is -1.
6752	if (match(V: Op0, P: m_AllOnes()) && match(V: Op1, P: m_AllOnes()))
6753	return ConstantInt::getAllOnesValue(Ty: F->getReturnType());
6754
6755	return nullptr;
6756	}
6757	case Intrinsic::experimental_constrained_fma: {
6758	auto *FPI = cast<ConstrainedFPIntrinsic>(Val: Call);
6759	if (Value *V = simplifyFPOp(Ops: Args, FMF: {}, Q, ExBehavior: *FPI->getExceptionBehavior(),
6760	Rounding: *FPI->getRoundingMode()))
6761	return V;
6762	return nullptr;
6763	}
6764	case Intrinsic::fma:
6765	case Intrinsic::fmuladd: {
6766	if (Value *V = simplifyFPOp(Ops: Args, FMF: {}, Q, ExBehavior: fp::ebIgnore,
6767	Rounding: RoundingMode::NearestTiesToEven))
6768	return V;
6769	return nullptr;
6770	}
6771	case Intrinsic::smul_fix:
6772	case Intrinsic::smul_fix_sat: {
6773	Value *Op0 = Args[0];
6774	Value *Op1 = Args[1];
6775	Value *Op2 = Args[2];
6776	Type *ReturnType = F->getReturnType();
6777
6778	// Canonicalize constant operand as Op1 (ConstantFolding handles the case
6779	// when both Op0 and Op1 are constant so we do not care about that special
6780	// case here).
6781	if (isa<Constant>(Val: Op0))
6782	std::swap(a&: Op0, b&: Op1);
6783
6784	// X * 0 -> 0
6785	if (match(V: Op1, P: m_Zero()))
6786	return Constant::getNullValue(Ty: ReturnType);
6787
6788	// X * undef -> 0
6789	if (Q.isUndefValue(V: Op1))
6790	return Constant::getNullValue(Ty: ReturnType);
6791
6792	// X * (1 << Scale) -> X
6793	APInt ScaledOne =
6794	APInt::getOneBitSet(numBits: ReturnType->getScalarSizeInBits(),
6795	BitNo: cast<ConstantInt>(Val: Op2)->getZExtValue());
6796	if (ScaledOne.isNonNegative() && match(V: Op1, P: m_SpecificInt(V: ScaledOne)))
6797	return Op0;
6798
6799	return nullptr;
6800	}
6801	case Intrinsic::vector_insert: {
6802	Value *Vec = Args[0];
6803	Value *SubVec = Args[1];
6804	Value *Idx = Args[2];
6805	Type *ReturnType = F->getReturnType();
6806
6807	// (insert_vector Y, (extract_vector X, 0), 0) -> X
6808	// where: Y is X, or Y is undef
6809	unsigned IdxN = cast<ConstantInt>(Val: Idx)->getZExtValue();
6810	Value *X = nullptr;
6811	if (match(SubVec,
6812	m_Intrinsic<Intrinsic::vector_extract>(m_Value(X), m_Zero())) &&
6813	(Q.isUndefValue(Vec) || Vec == X) && IdxN == 0 &&
6814	X->getType() == ReturnType)
6815	return X;
6816
6817	return nullptr;
6818	}
6819	case Intrinsic::experimental_constrained_fadd: {
6820	auto *FPI = cast<ConstrainedFPIntrinsic>(Val: Call);
6821	return simplifyFAddInst(Op0: Args[0], Op1: Args[1], FMF: FPI->getFastMathFlags(), Q,
6822	ExBehavior: *FPI->getExceptionBehavior(),
6823	Rounding: *FPI->getRoundingMode());
6824	}
6825	case Intrinsic::experimental_constrained_fsub: {
6826	auto *FPI = cast<ConstrainedFPIntrinsic>(Val: Call);
6827	return simplifyFSubInst(Op0: Args[0], Op1: Args[1], FMF: FPI->getFastMathFlags(), Q,
6828	ExBehavior: *FPI->getExceptionBehavior(),
6829	Rounding: *FPI->getRoundingMode());
6830	}
6831	case Intrinsic::experimental_constrained_fmul: {
6832	auto *FPI = cast<ConstrainedFPIntrinsic>(Val: Call);
6833	return simplifyFMulInst(Op0: Args[0], Op1: Args[1], FMF: FPI->getFastMathFlags(), Q,
6834	ExBehavior: *FPI->getExceptionBehavior(),
6835	Rounding: *FPI->getRoundingMode());
6836	}
6837	case Intrinsic::experimental_constrained_fdiv: {
6838	auto *FPI = cast<ConstrainedFPIntrinsic>(Val: Call);
6839	return simplifyFDivInst(Op0: Args[0], Op1: Args[1], FMF: FPI->getFastMathFlags(), Q,
6840	ExBehavior: *FPI->getExceptionBehavior(),
6841	Rounding: *FPI->getRoundingMode());
6842	}
6843	case Intrinsic::experimental_constrained_frem: {
6844	auto *FPI = cast<ConstrainedFPIntrinsic>(Val: Call);
6845	return simplifyFRemInst(Op0: Args[0], Op1: Args[1], FMF: FPI->getFastMathFlags(), Q,
6846	ExBehavior: *FPI->getExceptionBehavior(),
6847	Rounding: *FPI->getRoundingMode());
6848	}
6849	case Intrinsic::experimental_constrained_ldexp:
6850	return simplifyLdexp(Op0: Args[0], Op1: Args[1], Q, IsStrict: true);
6851	case Intrinsic::experimental_gc_relocate: {
6852	GCRelocateInst &GCR = *cast<GCRelocateInst>(Val: Call);
6853	Value *DerivedPtr = GCR.getDerivedPtr();
6854	Value *BasePtr = GCR.getBasePtr();
6855
6856	// Undef is undef, even after relocation.
6857	if (isa<UndefValue>(Val: DerivedPtr) || isa<UndefValue>(Val: BasePtr)) {
6858	return UndefValue::get(T: GCR.getType());
6859	}
6860
6861	if (auto *PT = dyn_cast<PointerType>(Val: GCR.getType())) {
6862	// For now, the assumption is that the relocation of null will be null
6863	// for most any collector. If this ever changes, a corresponding hook
6864	// should be added to GCStrategy and this code should check it first.
6865	if (isa<ConstantPointerNull>(Val: DerivedPtr)) {
6866	// Use null-pointer of gc_relocate's type to replace it.
6867	return ConstantPointerNull::get(T: PT);
6868	}
6869	}
6870	return nullptr;
6871	}
6872	default:
6873	return nullptr;
6874	}
6875	}
6876
6877	static Value *tryConstantFoldCall(CallBase *Call, Value *Callee,
6878	ArrayRef<Value *> Args,
6879	const SimplifyQuery &Q) {
6880	auto *F = dyn_cast<Function>(Val: Callee);
6881	if (!F || !canConstantFoldCallTo(Call, F))
6882	return nullptr;
6883
6884	SmallVector<Constant *, 4> ConstantArgs;
6885	ConstantArgs.reserve(N: Args.size());
6886	for (Value *Arg : Args) {
6887	Constant *C = dyn_cast<Constant>(Val: Arg);
6888	if (!C) {
6889	if (isa<MetadataAsValue>(Val: Arg))
6890	continue;
6891	return nullptr;
6892	}
6893	ConstantArgs.push_back(Elt: C);
6894	}
6895
6896	return ConstantFoldCall(Call, F, Operands: ConstantArgs, TLI: Q.TLI);
6897	}
6898
6899	Value *llvm::simplifyCall(CallBase *Call, Value *Callee, ArrayRef<Value *> Args,
6900	const SimplifyQuery &Q) {
6901	// Args should not contain operand bundle operands.
6902	assert(Call->arg_size() == Args.size());
6903
6904	// musttail calls can only be simplified if they are also DCEd.
6905	// As we can't guarantee this here, don't simplify them.
6906	if (Call->isMustTailCall())
6907	return nullptr;
6908
6909	// call undef -> poison
6910	// call null -> poison
6911	if (isa<UndefValue>(Val: Callee) || isa<ConstantPointerNull>(Val: Callee))
6912	return PoisonValue::get(T: Call->getType());
6913
6914	if (Value *V = tryConstantFoldCall(Call, Callee, Args, Q))
6915	return V;
6916
6917	auto *F = dyn_cast<Function>(Val: Callee);
6918	if (F && F->isIntrinsic())
6919	if (Value *Ret = simplifyIntrinsic(Call, Callee, Args, Q))
6920	return Ret;
6921
6922	return nullptr;
6923	}
6924
6925	Value *llvm::simplifyConstrainedFPCall(CallBase *Call, const SimplifyQuery &Q) {
6926	assert(isa<ConstrainedFPIntrinsic>(Call));
6927	SmallVector<Value *, 4> Args(Call->args());
6928	if (Value *V = tryConstantFoldCall(Call, Callee: Call->getCalledOperand(), Args, Q))
6929	return V;
6930	if (Value *Ret = simplifyIntrinsic(Call, Callee: Call->getCalledOperand(), Args, Q))
6931	return Ret;
6932	return nullptr;
6933	}
6934
6935	/// Given operands for a Freeze, see if we can fold the result.
6936	static Value *simplifyFreezeInst(Value *Op0, const SimplifyQuery &Q) {
6937	// Use a utility function defined in ValueTracking.
6938	if (llvm::isGuaranteedNotToBeUndefOrPoison(V: Op0, AC: Q.AC, CtxI: Q.CxtI, DT: Q.DT))
6939	return Op0;
6940	// We have room for improvement.
6941	return nullptr;
6942	}
6943
6944	Value *llvm::simplifyFreezeInst(Value *Op0, const SimplifyQuery &Q) {
6945	return ::simplifyFreezeInst(Op0, Q);
6946	}
6947
6948	Value *llvm::simplifyLoadInst(LoadInst *LI, Value *PtrOp,
6949	const SimplifyQuery &Q) {
6950	if (LI->isVolatile())
6951	return nullptr;
6952
6953	if (auto *PtrOpC = dyn_cast<Constant>(Val: PtrOp))
6954	return ConstantFoldLoadFromConstPtr(C: PtrOpC, Ty: LI->getType(), DL: Q.DL);
6955
6956	// We can only fold the load if it is from a constant global with definitive
6957	// initializer. Skip expensive logic if this is not the case.
6958	auto *GV = dyn_cast<GlobalVariable>(Val: getUnderlyingObject(V: PtrOp));
6959	if (!GV || !GV->isConstant() || !GV->hasDefinitiveInitializer())
6960	return nullptr;
6961
6962	// If GlobalVariable's initializer is uniform, then return the constant
6963	// regardless of its offset.
6964	if (Constant *C = ConstantFoldLoadFromUniformValue(C: GV->getInitializer(),
6965	Ty: LI->getType(), DL: Q.DL))
6966	return C;
6967
6968	// Try to convert operand into a constant by stripping offsets while looking
6969	// through invariant.group intrinsics.
6970	APInt Offset(Q.DL.getIndexTypeSizeInBits(Ty: PtrOp->getType()), 0);
6971	PtrOp = PtrOp->stripAndAccumulateConstantOffsets(
6972	DL: Q.DL, Offset, /* AllowNonInbounts */ AllowNonInbounds: true,
6973	/* AllowInvariantGroup */ true);
6974	if (PtrOp == GV) {
6975	// Index size may have changed due to address space casts.
6976	Offset = Offset.sextOrTrunc(width: Q.DL.getIndexTypeSizeInBits(Ty: PtrOp->getType()));
6977	return ConstantFoldLoadFromConstPtr(C: GV, Ty: LI->getType(), Offset: std::move(Offset),
6978	DL: Q.DL);
6979	}
6980
6981	return nullptr;
6982	}
6983
6984	/// See if we can compute a simplified version of this instruction.
6985	/// If not, this returns null.
6986
6987	static Value *simplifyInstructionWithOperands(Instruction *I,
6988	ArrayRef<Value *> NewOps,
6989	const SimplifyQuery &SQ,
6990	unsigned MaxRecurse) {
6991	assert(I->getFunction() && "instruction should be inserted in a function");
6992	assert((!SQ.CxtI || SQ.CxtI->getFunction() == I->getFunction()) &&
6993	"context instruction should be in the same function");
6994
6995	const SimplifyQuery Q = SQ.CxtI ? SQ : SQ.getWithInstruction(I);
6996
6997	switch (I->getOpcode()) {
6998	default:
6999	if (llvm::all_of(Range&: NewOps, P: [](Value *V) { return isa<Constant>(Val: V); })) {
7000	SmallVector<Constant *, 8> NewConstOps(NewOps.size());
7001	transform(Range&: NewOps, d_first: NewConstOps.begin(),
7002	F: [](Value *V) { return cast<Constant>(Val: V); });
7003	return ConstantFoldInstOperands(I, Ops: NewConstOps, DL: Q.DL, TLI: Q.TLI);
7004	}
7005	return nullptr;
7006	case Instruction::FNeg:
7007	return simplifyFNegInst(Op: NewOps[0], FMF: I->getFastMathFlags(), Q, MaxRecurse);
7008	case Instruction::FAdd:
7009	return simplifyFAddInst(Op0: NewOps[0], Op1: NewOps[1], FMF: I->getFastMathFlags(), Q,
7010	MaxRecurse);
7011	case Instruction::Add:
7012	return simplifyAddInst(
7013	Op0: NewOps[0], Op1: NewOps[1], IsNSW: Q.IIQ.hasNoSignedWrap(Op: cast<BinaryOperator>(Val: I)),
7014	IsNUW: Q.IIQ.hasNoUnsignedWrap(Op: cast<BinaryOperator>(Val: I)), Q, MaxRecurse);
7015	case Instruction::FSub:
7016	return simplifyFSubInst(Op0: NewOps[0], Op1: NewOps[1], FMF: I->getFastMathFlags(), Q,
7017	MaxRecurse);
7018	case Instruction::Sub:
7019	return simplifySubInst(
7020	Op0: NewOps[0], Op1: NewOps[1], IsNSW: Q.IIQ.hasNoSignedWrap(Op: cast<BinaryOperator>(Val: I)),
7021	IsNUW: Q.IIQ.hasNoUnsignedWrap(Op: cast<BinaryOperator>(Val: I)), Q, MaxRecurse);
7022	case Instruction::FMul:
7023	return simplifyFMulInst(Op0: NewOps[0], Op1: NewOps[1], FMF: I->getFastMathFlags(), Q,
7024	MaxRecurse);
7025	case Instruction::Mul:
7026	return simplifyMulInst(
7027	Op0: NewOps[0], Op1: NewOps[1], IsNSW: Q.IIQ.hasNoSignedWrap(Op: cast<BinaryOperator>(Val: I)),
7028	IsNUW: Q.IIQ.hasNoUnsignedWrap(Op: cast<BinaryOperator>(Val: I)), Q, MaxRecurse);
7029	case Instruction::SDiv:
7030	return simplifySDivInst(Op0: NewOps[0], Op1: NewOps[1],
7031	IsExact: Q.IIQ.isExact(Op: cast<BinaryOperator>(Val: I)), Q,
7032	MaxRecurse);
7033	case Instruction::UDiv:
7034	return simplifyUDivInst(Op0: NewOps[0], Op1: NewOps[1],
7035	IsExact: Q.IIQ.isExact(Op: cast<BinaryOperator>(Val: I)), Q,
7036	MaxRecurse);
7037	case Instruction::FDiv:
7038	return simplifyFDivInst(Op0: NewOps[0], Op1: NewOps[1], FMF: I->getFastMathFlags(), Q,
7039	MaxRecurse);
7040	case Instruction::SRem:
7041	return simplifySRemInst(Op0: NewOps[0], Op1: NewOps[1], Q, MaxRecurse);
7042	case Instruction::URem:
7043	return simplifyURemInst(Op0: NewOps[0], Op1: NewOps[1], Q, MaxRecurse);
7044	case Instruction::FRem:
7045	return simplifyFRemInst(Op0: NewOps[0], Op1: NewOps[1], FMF: I->getFastMathFlags(), Q,
7046	MaxRecurse);
7047	case Instruction::Shl:
7048	return simplifyShlInst(
7049	Op0: NewOps[0], Op1: NewOps[1], IsNSW: Q.IIQ.hasNoSignedWrap(Op: cast<BinaryOperator>(Val: I)),
7050	IsNUW: Q.IIQ.hasNoUnsignedWrap(Op: cast<BinaryOperator>(Val: I)), Q, MaxRecurse);
7051	case Instruction::LShr:
7052	return simplifyLShrInst(Op0: NewOps[0], Op1: NewOps[1],
7053	IsExact: Q.IIQ.isExact(Op: cast<BinaryOperator>(Val: I)), Q,
7054	MaxRecurse);
7055	case Instruction::AShr:
7056	return simplifyAShrInst(Op0: NewOps[0], Op1: NewOps[1],
7057	IsExact: Q.IIQ.isExact(Op: cast<BinaryOperator>(Val: I)), Q,
7058	MaxRecurse);
7059	case Instruction::And:
7060	return simplifyAndInst(Op0: NewOps[0], Op1: NewOps[1], Q, MaxRecurse);
7061	case Instruction::Or:
7062	return simplifyOrInst(Op0: NewOps[0], Op1: NewOps[1], Q, MaxRecurse);
7063	case Instruction::Xor:
7064	return simplifyXorInst(Op0: NewOps[0], Op1: NewOps[1], Q, MaxRecurse);
7065	case Instruction::ICmp:
7066	return simplifyICmpInst(Predicate: cast<ICmpInst>(Val: I)->getPredicate(), LHS: NewOps[0],
7067	RHS: NewOps[1], Q, MaxRecurse);
7068	case Instruction::FCmp:
7069	return simplifyFCmpInst(Predicate: cast<FCmpInst>(Val: I)->getPredicate(), LHS: NewOps[0],
7070	RHS: NewOps[1], FMF: I->getFastMathFlags(), Q, MaxRecurse);
7071	case Instruction::Select:
7072	return simplifySelectInst(Cond: NewOps[0], TrueVal: NewOps[1], FalseVal: NewOps[2], Q, MaxRecurse);
7073	break;
7074	case Instruction::GetElementPtr: {
7075	auto *GEPI = cast<GetElementPtrInst>(Val: I);
7076	return simplifyGEPInst(SrcTy: GEPI->getSourceElementType(), Ptr: NewOps[0],
7077	Indices: ArrayRef(NewOps).slice(N: 1), InBounds: GEPI->isInBounds(), Q,
7078	MaxRecurse);
7079	}
7080	case Instruction::InsertValue: {
7081	InsertValueInst *IV = cast<InsertValueInst>(Val: I);
7082	return simplifyInsertValueInst(Agg: NewOps[0], Val: NewOps[1], Idxs: IV->getIndices(), Q,
7083	MaxRecurse);
7084	}
7085	case Instruction::InsertElement:
7086	return simplifyInsertElementInst(Vec: NewOps[0], Val: NewOps[1], Idx: NewOps[2], Q);
7087	case Instruction::ExtractValue: {
7088	auto *EVI = cast<ExtractValueInst>(Val: I);
7089	return simplifyExtractValueInst(Agg: NewOps[0], Idxs: EVI->getIndices(), Q,
7090	MaxRecurse);
7091	}
7092	case Instruction::ExtractElement:
7093	return simplifyExtractElementInst(Vec: NewOps[0], Idx: NewOps[1], Q, MaxRecurse);
7094	case Instruction::ShuffleVector: {
7095	auto *SVI = cast<ShuffleVectorInst>(Val: I);
7096	return simplifyShuffleVectorInst(Op0: NewOps[0], Op1: NewOps[1],
7097	Mask: SVI->getShuffleMask(), RetTy: SVI->getType(), Q,
7098	MaxRecurse);
7099	}
7100	case Instruction::PHI:
7101	return simplifyPHINode(PN: cast<PHINode>(Val: I), IncomingValues: NewOps, Q);
7102	case Instruction::Call:
7103	return simplifyCall(
7104	Call: cast<CallInst>(Val: I), Callee: NewOps.back(),
7105	Args: NewOps.drop_back(N: 1 + cast<CallInst>(Val: I)->getNumTotalBundleOperands()), Q);
7106	case Instruction::Freeze:
7107	return llvm::simplifyFreezeInst(Op0: NewOps[0], Q);
7108	#define HANDLE_CAST_INST(num, opc, clas) case Instruction::opc:
7109	#include "llvm/IR/Instruction.def"
7110	#undef HANDLE_CAST_INST
7111	return simplifyCastInst(CastOpc: I->getOpcode(), Op: NewOps[0], Ty: I->getType(), Q,
7112	MaxRecurse);
7113	case Instruction::Alloca:
7114	// No simplifications for Alloca and it can't be constant folded.
7115	return nullptr;
7116	case Instruction::Load:
7117	return simplifyLoadInst(LI: cast<LoadInst>(Val: I), PtrOp: NewOps[0], Q);
7118	}
7119	}
7120
7121	Value *llvm::simplifyInstructionWithOperands(Instruction *I,
7122	ArrayRef<Value *> NewOps,
7123	const SimplifyQuery &SQ) {
7124	assert(NewOps.size() == I->getNumOperands() &&
7125	"Number of operands should match the instruction!");
7126	return ::simplifyInstructionWithOperands(I, NewOps, SQ, MaxRecurse: RecursionLimit);
7127	}
7128
7129	Value *llvm::simplifyInstruction(Instruction *I, const SimplifyQuery &SQ) {
7130	SmallVector<Value *, 8> Ops(I->operands());
7131	Value *Result = ::simplifyInstructionWithOperands(I, NewOps: Ops, SQ, MaxRecurse: RecursionLimit);
7132
7133	/// If called on unreachable code, the instruction may simplify to itself.
7134	/// Make life easier for users by detecting that case here, and returning a
7135	/// safe value instead.
7136	return Result == I ? UndefValue::get(T: I->getType()) : Result;
7137	}
7138
7139	/// Implementation of recursive simplification through an instruction's
7140	/// uses.
7141	///
7142	/// This is the common implementation of the recursive simplification routines.
7143	/// If we have a pre-simplified value in 'SimpleV', that is forcibly used to
7144	/// replace the instruction 'I'. Otherwise, we simply add 'I' to the list of
7145	/// instructions to process and attempt to simplify it using
7146	/// InstructionSimplify. Recursively visited users which could not be
7147	/// simplified themselves are to the optional UnsimplifiedUsers set for
7148	/// further processing by the caller.
7149	///
7150	/// This routine returns 'true' only when *it* simplifies something. The passed
7151	/// in simplified value does not count toward this.
7152	static bool replaceAndRecursivelySimplifyImpl(
7153	Instruction *I, Value *SimpleV, const TargetLibraryInfo *TLI,
7154	const DominatorTree *DT, AssumptionCache *AC,
7155	SmallSetVector<Instruction *, 8> *UnsimplifiedUsers = nullptr) {
7156	bool Simplified = false;
7157	SmallSetVector<Instruction *, 8> Worklist;
7158	const DataLayout &DL = I->getModule()->getDataLayout();
7159
7160	// If we have an explicit value to collapse to, do that round of the
7161	// simplification loop by hand initially.
7162	if (SimpleV) {
7163	for (User *U : I->users())
7164	if (U != I)
7165	Worklist.insert(X: cast<Instruction>(Val: U));
7166
7167	// Replace the instruction with its simplified value.
7168	I->replaceAllUsesWith(V: SimpleV);
7169
7170	if (!I->isEHPad() && !I->isTerminator() && !I->mayHaveSideEffects())
7171	I->eraseFromParent();
7172	} else {
7173	Worklist.insert(X: I);
7174	}
7175
7176	// Note that we must test the size on each iteration, the worklist can grow.
7177	for (unsigned Idx = 0; Idx != Worklist.size(); ++Idx) {
7178	I = Worklist[Idx];
7179
7180	// See if this instruction simplifies.
7181	SimpleV = simplifyInstruction(I, SQ: {DL, TLI, DT, AC});
7182	if (!SimpleV) {
7183	if (UnsimplifiedUsers)
7184	UnsimplifiedUsers->insert(X: I);
7185	continue;
7186	}
7187
7188	Simplified = true;
7189
7190	// Stash away all the uses of the old instruction so we can check them for
7191	// recursive simplifications after a RAUW. This is cheaper than checking all
7192	// uses of To on the recursive step in most cases.
7193	for (User *U : I->users())
7194	Worklist.insert(X: cast<Instruction>(Val: U));
7195
7196	// Replace the instruction with its simplified value.
7197	I->replaceAllUsesWith(V: SimpleV);
7198
7199	if (!I->isEHPad() && !I->isTerminator() && !I->mayHaveSideEffects())
7200	I->eraseFromParent();
7201	}
7202	return Simplified;
7203	}
7204
7205	bool llvm::replaceAndRecursivelySimplify(
7206	Instruction *I, Value *SimpleV, const TargetLibraryInfo *TLI,
7207	const DominatorTree *DT, AssumptionCache *AC,
7208	SmallSetVector<Instruction *, 8> *UnsimplifiedUsers) {
7209	assert(I != SimpleV && "replaceAndRecursivelySimplify(X,X) is not valid!");
7210	assert(SimpleV && "Must provide a simplified value.");
7211	return replaceAndRecursivelySimplifyImpl(I, SimpleV, TLI, DT, AC,
7212	UnsimplifiedUsers);
7213	}
7214
7215	namespace llvm {
7216	const SimplifyQuery getBestSimplifyQuery(Pass &P, Function &F) {
7217	auto *DTWP = P.getAnalysisIfAvailable<DominatorTreeWrapperPass>();
7218	auto *DT = DTWP ? &DTWP->getDomTree() : nullptr;
7219	auto *TLIWP = P.getAnalysisIfAvailable<TargetLibraryInfoWrapperPass>();
7220	auto *TLI = TLIWP ? &TLIWP->getTLI(F) : nullptr;
7221	auto *ACWP = P.getAnalysisIfAvailable<AssumptionCacheTracker>();
7222	auto *AC = ACWP ? &ACWP->getAssumptionCache(F) : nullptr;
7223	return {F.getParent()->getDataLayout(), TLI, DT, AC};
7224	}
7225
7226	const SimplifyQuery getBestSimplifyQuery(LoopStandardAnalysisResults &AR,
7227	const DataLayout &DL) {
7228	return {DL, &AR.TLI, &AR.DT, &AR.AC};
7229	}
7230
7231	template <class T, class... TArgs>
7232	const SimplifyQuery getBestSimplifyQuery(AnalysisManager<T, TArgs...> &AM,
7233	Function &F) {
7234	auto *DT = AM.template getCachedResult<DominatorTreeAnalysis>(F);
7235	auto *TLI = AM.template getCachedResult<TargetLibraryAnalysis>(F);
7236	auto *AC = AM.template getCachedResult<AssumptionAnalysis>(F);
7237	return {F.getParent()->getDataLayout(), TLI, DT, AC};
7238	}
7239	template const SimplifyQuery getBestSimplifyQuery(AnalysisManager<Function> &,
7240	Function &);
7241
7242	bool SimplifyQuery::isUndefValue(Value *V) const {
7243	if (!CanUseUndef)
7244	return false;
7245
7246	return match(V, P: m_Undef());
7247	}
7248
7249	} // namespace llvm
7250
7251	void InstSimplifyFolder::anchor() {}
7252