1 | //===- InstCombineCompares.cpp --------------------------------------------===// |
2 | // |
3 | // Part of the LLVM Project, under the Apache License v2.0 with LLVM Exceptions. |
4 | // See https://llvm.org/LICENSE.txt for license information. |
5 | // SPDX-License-Identifier: Apache-2.0 WITH LLVM-exception |
6 | // |
7 | //===----------------------------------------------------------------------===// |
8 | // |
9 | // This file implements the visitICmp and visitFCmp functions. |
10 | // |
11 | //===----------------------------------------------------------------------===// |
12 | |
13 | #include "InstCombineInternal.h" |
14 | #include "llvm/ADT/APSInt.h" |
15 | #include "llvm/ADT/ScopeExit.h" |
16 | #include "llvm/ADT/SetVector.h" |
17 | #include "llvm/ADT/Statistic.h" |
18 | #include "llvm/Analysis/CaptureTracking.h" |
19 | #include "llvm/Analysis/CmpInstAnalysis.h" |
20 | #include "llvm/Analysis/ConstantFolding.h" |
21 | #include "llvm/Analysis/InstructionSimplify.h" |
22 | #include "llvm/Analysis/Utils/Local.h" |
23 | #include "llvm/Analysis/VectorUtils.h" |
24 | #include "llvm/IR/ConstantRange.h" |
25 | #include "llvm/IR/DataLayout.h" |
26 | #include "llvm/IR/InstrTypes.h" |
27 | #include "llvm/IR/IntrinsicInst.h" |
28 | #include "llvm/IR/PatternMatch.h" |
29 | #include "llvm/Support/KnownBits.h" |
30 | #include "llvm/Transforms/InstCombine/InstCombiner.h" |
31 | #include <bitset> |
32 | |
33 | using namespace llvm; |
34 | using namespace PatternMatch; |
35 | |
36 | #define DEBUG_TYPE "instcombine" |
37 | |
38 | // How many times is a select replaced by one of its operands? |
39 | STATISTIC(NumSel, "Number of select opts" ); |
40 | |
41 | |
42 | /// Compute Result = In1+In2, returning true if the result overflowed for this |
43 | /// type. |
44 | static bool addWithOverflow(APInt &Result, const APInt &In1, |
45 | const APInt &In2, bool IsSigned = false) { |
46 | bool Overflow; |
47 | if (IsSigned) |
48 | Result = In1.sadd_ov(RHS: In2, Overflow); |
49 | else |
50 | Result = In1.uadd_ov(RHS: In2, Overflow); |
51 | |
52 | return Overflow; |
53 | } |
54 | |
55 | /// Compute Result = In1-In2, returning true if the result overflowed for this |
56 | /// type. |
57 | static bool subWithOverflow(APInt &Result, const APInt &In1, |
58 | const APInt &In2, bool IsSigned = false) { |
59 | bool Overflow; |
60 | if (IsSigned) |
61 | Result = In1.ssub_ov(RHS: In2, Overflow); |
62 | else |
63 | Result = In1.usub_ov(RHS: In2, Overflow); |
64 | |
65 | return Overflow; |
66 | } |
67 | |
68 | /// Given an icmp instruction, return true if any use of this comparison is a |
69 | /// branch on sign bit comparison. |
70 | static bool hasBranchUse(ICmpInst &I) { |
71 | for (auto *U : I.users()) |
72 | if (isa<BranchInst>(Val: U)) |
73 | return true; |
74 | return false; |
75 | } |
76 | |
77 | /// Returns true if the exploded icmp can be expressed as a signed comparison |
78 | /// to zero and updates the predicate accordingly. |
79 | /// The signedness of the comparison is preserved. |
80 | /// TODO: Refactor with decomposeBitTestICmp()? |
81 | static bool isSignTest(ICmpInst::Predicate &Pred, const APInt &C) { |
82 | if (!ICmpInst::isSigned(predicate: Pred)) |
83 | return false; |
84 | |
85 | if (C.isZero()) |
86 | return ICmpInst::isRelational(P: Pred); |
87 | |
88 | if (C.isOne()) { |
89 | if (Pred == ICmpInst::ICMP_SLT) { |
90 | Pred = ICmpInst::ICMP_SLE; |
91 | return true; |
92 | } |
93 | } else if (C.isAllOnes()) { |
94 | if (Pred == ICmpInst::ICMP_SGT) { |
95 | Pred = ICmpInst::ICMP_SGE; |
96 | return true; |
97 | } |
98 | } |
99 | |
100 | return false; |
101 | } |
102 | |
103 | /// This is called when we see this pattern: |
104 | /// cmp pred (load (gep GV, ...)), cmpcst |
105 | /// where GV is a global variable with a constant initializer. Try to simplify |
106 | /// this into some simple computation that does not need the load. For example |
107 | /// we can optimize "icmp eq (load (gep "foo", 0, i)), 0" into "icmp eq i, 3". |
108 | /// |
109 | /// If AndCst is non-null, then the loaded value is masked with that constant |
110 | /// before doing the comparison. This handles cases like "A[i]&4 == 0". |
111 | Instruction *InstCombinerImpl::foldCmpLoadFromIndexedGlobal( |
112 | LoadInst *LI, GetElementPtrInst *GEP, GlobalVariable *GV, CmpInst &ICI, |
113 | ConstantInt *AndCst) { |
114 | if (LI->isVolatile() || LI->getType() != GEP->getResultElementType() || |
115 | GV->getValueType() != GEP->getSourceElementType() || !GV->isConstant() || |
116 | !GV->hasDefinitiveInitializer()) |
117 | return nullptr; |
118 | |
119 | Constant *Init = GV->getInitializer(); |
120 | if (!isa<ConstantArray>(Val: Init) && !isa<ConstantDataArray>(Val: Init)) |
121 | return nullptr; |
122 | |
123 | uint64_t ArrayElementCount = Init->getType()->getArrayNumElements(); |
124 | // Don't blow up on huge arrays. |
125 | if (ArrayElementCount > MaxArraySizeForCombine) |
126 | return nullptr; |
127 | |
128 | // There are many forms of this optimization we can handle, for now, just do |
129 | // the simple index into a single-dimensional array. |
130 | // |
131 | // Require: GEP GV, 0, i {{, constant indices}} |
132 | if (GEP->getNumOperands() < 3 || !isa<ConstantInt>(Val: GEP->getOperand(i_nocapture: 1)) || |
133 | !cast<ConstantInt>(Val: GEP->getOperand(i_nocapture: 1))->isZero() || |
134 | isa<Constant>(Val: GEP->getOperand(i_nocapture: 2))) |
135 | return nullptr; |
136 | |
137 | // Check that indices after the variable are constants and in-range for the |
138 | // type they index. Collect the indices. This is typically for arrays of |
139 | // structs. |
140 | SmallVector<unsigned, 4> LaterIndices; |
141 | |
142 | Type *EltTy = Init->getType()->getArrayElementType(); |
143 | for (unsigned i = 3, e = GEP->getNumOperands(); i != e; ++i) { |
144 | ConstantInt *Idx = dyn_cast<ConstantInt>(Val: GEP->getOperand(i_nocapture: i)); |
145 | if (!Idx) |
146 | return nullptr; // Variable index. |
147 | |
148 | uint64_t IdxVal = Idx->getZExtValue(); |
149 | if ((unsigned)IdxVal != IdxVal) |
150 | return nullptr; // Too large array index. |
151 | |
152 | if (StructType *STy = dyn_cast<StructType>(Val: EltTy)) |
153 | EltTy = STy->getElementType(N: IdxVal); |
154 | else if (ArrayType *ATy = dyn_cast<ArrayType>(Val: EltTy)) { |
155 | if (IdxVal >= ATy->getNumElements()) |
156 | return nullptr; |
157 | EltTy = ATy->getElementType(); |
158 | } else { |
159 | return nullptr; // Unknown type. |
160 | } |
161 | |
162 | LaterIndices.push_back(Elt: IdxVal); |
163 | } |
164 | |
165 | enum { Overdefined = -3, Undefined = -2 }; |
166 | |
167 | // Variables for our state machines. |
168 | |
169 | // FirstTrueElement/SecondTrueElement - Used to emit a comparison of the form |
170 | // "i == 47 | i == 87", where 47 is the first index the condition is true for, |
171 | // and 87 is the second (and last) index. FirstTrueElement is -2 when |
172 | // undefined, otherwise set to the first true element. SecondTrueElement is |
173 | // -2 when undefined, -3 when overdefined and >= 0 when that index is true. |
174 | int FirstTrueElement = Undefined, SecondTrueElement = Undefined; |
175 | |
176 | // FirstFalseElement/SecondFalseElement - Used to emit a comparison of the |
177 | // form "i != 47 & i != 87". Same state transitions as for true elements. |
178 | int FirstFalseElement = Undefined, SecondFalseElement = Undefined; |
179 | |
180 | /// TrueRangeEnd/FalseRangeEnd - In conjunction with First*Element, these |
181 | /// define a state machine that triggers for ranges of values that the index |
182 | /// is true or false for. This triggers on things like "abbbbc"[i] == 'b'. |
183 | /// This is -2 when undefined, -3 when overdefined, and otherwise the last |
184 | /// index in the range (inclusive). We use -2 for undefined here because we |
185 | /// use relative comparisons and don't want 0-1 to match -1. |
186 | int TrueRangeEnd = Undefined, FalseRangeEnd = Undefined; |
187 | |
188 | // MagicBitvector - This is a magic bitvector where we set a bit if the |
189 | // comparison is true for element 'i'. If there are 64 elements or less in |
190 | // the array, this will fully represent all the comparison results. |
191 | uint64_t MagicBitvector = 0; |
192 | |
193 | // Scan the array and see if one of our patterns matches. |
194 | Constant *CompareRHS = cast<Constant>(Val: ICI.getOperand(i_nocapture: 1)); |
195 | for (unsigned i = 0, e = ArrayElementCount; i != e; ++i) { |
196 | Constant *Elt = Init->getAggregateElement(Elt: i); |
197 | if (!Elt) |
198 | return nullptr; |
199 | |
200 | // If this is indexing an array of structures, get the structure element. |
201 | if (!LaterIndices.empty()) { |
202 | Elt = ConstantFoldExtractValueInstruction(Agg: Elt, Idxs: LaterIndices); |
203 | if (!Elt) |
204 | return nullptr; |
205 | } |
206 | |
207 | // If the element is masked, handle it. |
208 | if (AndCst) { |
209 | Elt = ConstantFoldBinaryOpOperands(Opcode: Instruction::And, LHS: Elt, RHS: AndCst, DL); |
210 | if (!Elt) |
211 | return nullptr; |
212 | } |
213 | |
214 | // Find out if the comparison would be true or false for the i'th element. |
215 | Constant *C = ConstantFoldCompareInstOperands(Predicate: ICI.getPredicate(), LHS: Elt, |
216 | RHS: CompareRHS, DL, TLI: &TLI); |
217 | // If the result is undef for this element, ignore it. |
218 | if (isa<UndefValue>(Val: C)) { |
219 | // Extend range state machines to cover this element in case there is an |
220 | // undef in the middle of the range. |
221 | if (TrueRangeEnd == (int)i - 1) |
222 | TrueRangeEnd = i; |
223 | if (FalseRangeEnd == (int)i - 1) |
224 | FalseRangeEnd = i; |
225 | continue; |
226 | } |
227 | |
228 | // If we can't compute the result for any of the elements, we have to give |
229 | // up evaluating the entire conditional. |
230 | if (!isa<ConstantInt>(Val: C)) |
231 | return nullptr; |
232 | |
233 | // Otherwise, we know if the comparison is true or false for this element, |
234 | // update our state machines. |
235 | bool IsTrueForElt = !cast<ConstantInt>(Val: C)->isZero(); |
236 | |
237 | // State machine for single/double/range index comparison. |
238 | if (IsTrueForElt) { |
239 | // Update the TrueElement state machine. |
240 | if (FirstTrueElement == Undefined) |
241 | FirstTrueElement = TrueRangeEnd = i; // First true element. |
242 | else { |
243 | // Update double-compare state machine. |
244 | if (SecondTrueElement == Undefined) |
245 | SecondTrueElement = i; |
246 | else |
247 | SecondTrueElement = Overdefined; |
248 | |
249 | // Update range state machine. |
250 | if (TrueRangeEnd == (int)i - 1) |
251 | TrueRangeEnd = i; |
252 | else |
253 | TrueRangeEnd = Overdefined; |
254 | } |
255 | } else { |
256 | // Update the FalseElement state machine. |
257 | if (FirstFalseElement == Undefined) |
258 | FirstFalseElement = FalseRangeEnd = i; // First false element. |
259 | else { |
260 | // Update double-compare state machine. |
261 | if (SecondFalseElement == Undefined) |
262 | SecondFalseElement = i; |
263 | else |
264 | SecondFalseElement = Overdefined; |
265 | |
266 | // Update range state machine. |
267 | if (FalseRangeEnd == (int)i - 1) |
268 | FalseRangeEnd = i; |
269 | else |
270 | FalseRangeEnd = Overdefined; |
271 | } |
272 | } |
273 | |
274 | // If this element is in range, update our magic bitvector. |
275 | if (i < 64 && IsTrueForElt) |
276 | MagicBitvector |= 1ULL << i; |
277 | |
278 | // If all of our states become overdefined, bail out early. Since the |
279 | // predicate is expensive, only check it every 8 elements. This is only |
280 | // really useful for really huge arrays. |
281 | if ((i & 8) == 0 && i >= 64 && SecondTrueElement == Overdefined && |
282 | SecondFalseElement == Overdefined && TrueRangeEnd == Overdefined && |
283 | FalseRangeEnd == Overdefined) |
284 | return nullptr; |
285 | } |
286 | |
287 | // Now that we've scanned the entire array, emit our new comparison(s). We |
288 | // order the state machines in complexity of the generated code. |
289 | Value *Idx = GEP->getOperand(i_nocapture: 2); |
290 | |
291 | // If the index is larger than the pointer offset size of the target, truncate |
292 | // the index down like the GEP would do implicitly. We don't have to do this |
293 | // for an inbounds GEP because the index can't be out of range. |
294 | if (!GEP->isInBounds()) { |
295 | Type *PtrIdxTy = DL.getIndexType(PtrTy: GEP->getType()); |
296 | unsigned OffsetSize = PtrIdxTy->getIntegerBitWidth(); |
297 | if (Idx->getType()->getPrimitiveSizeInBits().getFixedValue() > OffsetSize) |
298 | Idx = Builder.CreateTrunc(V: Idx, DestTy: PtrIdxTy); |
299 | } |
300 | |
301 | // If inbounds keyword is not present, Idx * ElementSize can overflow. |
302 | // Let's assume that ElementSize is 2 and the wanted value is at offset 0. |
303 | // Then, there are two possible values for Idx to match offset 0: |
304 | // 0x00..00, 0x80..00. |
305 | // Emitting 'icmp eq Idx, 0' isn't correct in this case because the |
306 | // comparison is false if Idx was 0x80..00. |
307 | // We need to erase the highest countTrailingZeros(ElementSize) bits of Idx. |
308 | unsigned ElementSize = |
309 | DL.getTypeAllocSize(Ty: Init->getType()->getArrayElementType()); |
310 | auto MaskIdx = [&](Value *Idx) { |
311 | if (!GEP->isInBounds() && llvm::countr_zero(Val: ElementSize) != 0) { |
312 | Value *Mask = ConstantInt::get(Ty: Idx->getType(), V: -1); |
313 | Mask = Builder.CreateLShr(LHS: Mask, RHS: llvm::countr_zero(Val: ElementSize)); |
314 | Idx = Builder.CreateAnd(LHS: Idx, RHS: Mask); |
315 | } |
316 | return Idx; |
317 | }; |
318 | |
319 | // If the comparison is only true for one or two elements, emit direct |
320 | // comparisons. |
321 | if (SecondTrueElement != Overdefined) { |
322 | Idx = MaskIdx(Idx); |
323 | // None true -> false. |
324 | if (FirstTrueElement == Undefined) |
325 | return replaceInstUsesWith(I&: ICI, V: Builder.getFalse()); |
326 | |
327 | Value *FirstTrueIdx = ConstantInt::get(Ty: Idx->getType(), V: FirstTrueElement); |
328 | |
329 | // True for one element -> 'i == 47'. |
330 | if (SecondTrueElement == Undefined) |
331 | return new ICmpInst(ICmpInst::ICMP_EQ, Idx, FirstTrueIdx); |
332 | |
333 | // True for two elements -> 'i == 47 | i == 72'. |
334 | Value *C1 = Builder.CreateICmpEQ(LHS: Idx, RHS: FirstTrueIdx); |
335 | Value *SecondTrueIdx = ConstantInt::get(Ty: Idx->getType(), V: SecondTrueElement); |
336 | Value *C2 = Builder.CreateICmpEQ(LHS: Idx, RHS: SecondTrueIdx); |
337 | return BinaryOperator::CreateOr(V1: C1, V2: C2); |
338 | } |
339 | |
340 | // If the comparison is only false for one or two elements, emit direct |
341 | // comparisons. |
342 | if (SecondFalseElement != Overdefined) { |
343 | Idx = MaskIdx(Idx); |
344 | // None false -> true. |
345 | if (FirstFalseElement == Undefined) |
346 | return replaceInstUsesWith(I&: ICI, V: Builder.getTrue()); |
347 | |
348 | Value *FirstFalseIdx = ConstantInt::get(Ty: Idx->getType(), V: FirstFalseElement); |
349 | |
350 | // False for one element -> 'i != 47'. |
351 | if (SecondFalseElement == Undefined) |
352 | return new ICmpInst(ICmpInst::ICMP_NE, Idx, FirstFalseIdx); |
353 | |
354 | // False for two elements -> 'i != 47 & i != 72'. |
355 | Value *C1 = Builder.CreateICmpNE(LHS: Idx, RHS: FirstFalseIdx); |
356 | Value *SecondFalseIdx = |
357 | ConstantInt::get(Ty: Idx->getType(), V: SecondFalseElement); |
358 | Value *C2 = Builder.CreateICmpNE(LHS: Idx, RHS: SecondFalseIdx); |
359 | return BinaryOperator::CreateAnd(V1: C1, V2: C2); |
360 | } |
361 | |
362 | // If the comparison can be replaced with a range comparison for the elements |
363 | // where it is true, emit the range check. |
364 | if (TrueRangeEnd != Overdefined) { |
365 | assert(TrueRangeEnd != FirstTrueElement && "Should emit single compare" ); |
366 | Idx = MaskIdx(Idx); |
367 | |
368 | // Generate (i-FirstTrue) <u (TrueRangeEnd-FirstTrue+1). |
369 | if (FirstTrueElement) { |
370 | Value *Offs = ConstantInt::get(Ty: Idx->getType(), V: -FirstTrueElement); |
371 | Idx = Builder.CreateAdd(LHS: Idx, RHS: Offs); |
372 | } |
373 | |
374 | Value *End = |
375 | ConstantInt::get(Ty: Idx->getType(), V: TrueRangeEnd - FirstTrueElement + 1); |
376 | return new ICmpInst(ICmpInst::ICMP_ULT, Idx, End); |
377 | } |
378 | |
379 | // False range check. |
380 | if (FalseRangeEnd != Overdefined) { |
381 | assert(FalseRangeEnd != FirstFalseElement && "Should emit single compare" ); |
382 | Idx = MaskIdx(Idx); |
383 | // Generate (i-FirstFalse) >u (FalseRangeEnd-FirstFalse). |
384 | if (FirstFalseElement) { |
385 | Value *Offs = ConstantInt::get(Ty: Idx->getType(), V: -FirstFalseElement); |
386 | Idx = Builder.CreateAdd(LHS: Idx, RHS: Offs); |
387 | } |
388 | |
389 | Value *End = |
390 | ConstantInt::get(Ty: Idx->getType(), V: FalseRangeEnd - FirstFalseElement); |
391 | return new ICmpInst(ICmpInst::ICMP_UGT, Idx, End); |
392 | } |
393 | |
394 | // If a magic bitvector captures the entire comparison state |
395 | // of this load, replace it with computation that does: |
396 | // ((magic_cst >> i) & 1) != 0 |
397 | { |
398 | Type *Ty = nullptr; |
399 | |
400 | // Look for an appropriate type: |
401 | // - The type of Idx if the magic fits |
402 | // - The smallest fitting legal type |
403 | if (ArrayElementCount <= Idx->getType()->getIntegerBitWidth()) |
404 | Ty = Idx->getType(); |
405 | else |
406 | Ty = DL.getSmallestLegalIntType(C&: Init->getContext(), Width: ArrayElementCount); |
407 | |
408 | if (Ty) { |
409 | Idx = MaskIdx(Idx); |
410 | Value *V = Builder.CreateIntCast(V: Idx, DestTy: Ty, isSigned: false); |
411 | V = Builder.CreateLShr(LHS: ConstantInt::get(Ty, V: MagicBitvector), RHS: V); |
412 | V = Builder.CreateAnd(LHS: ConstantInt::get(Ty, V: 1), RHS: V); |
413 | return new ICmpInst(ICmpInst::ICMP_NE, V, ConstantInt::get(Ty, V: 0)); |
414 | } |
415 | } |
416 | |
417 | return nullptr; |
418 | } |
419 | |
420 | /// Returns true if we can rewrite Start as a GEP with pointer Base |
421 | /// and some integer offset. The nodes that need to be re-written |
422 | /// for this transformation will be added to Explored. |
423 | static bool canRewriteGEPAsOffset(Value *Start, Value *Base, |
424 | const DataLayout &DL, |
425 | SetVector<Value *> &Explored) { |
426 | SmallVector<Value *, 16> WorkList(1, Start); |
427 | Explored.insert(X: Base); |
428 | |
429 | // The following traversal gives us an order which can be used |
430 | // when doing the final transformation. Since in the final |
431 | // transformation we create the PHI replacement instructions first, |
432 | // we don't have to get them in any particular order. |
433 | // |
434 | // However, for other instructions we will have to traverse the |
435 | // operands of an instruction first, which means that we have to |
436 | // do a post-order traversal. |
437 | while (!WorkList.empty()) { |
438 | SetVector<PHINode *> PHIs; |
439 | |
440 | while (!WorkList.empty()) { |
441 | if (Explored.size() >= 100) |
442 | return false; |
443 | |
444 | Value *V = WorkList.back(); |
445 | |
446 | if (Explored.contains(key: V)) { |
447 | WorkList.pop_back(); |
448 | continue; |
449 | } |
450 | |
451 | if (!isa<GetElementPtrInst>(Val: V) && !isa<PHINode>(Val: V)) |
452 | // We've found some value that we can't explore which is different from |
453 | // the base. Therefore we can't do this transformation. |
454 | return false; |
455 | |
456 | if (auto *GEP = dyn_cast<GEPOperator>(Val: V)) { |
457 | // Only allow inbounds GEPs with at most one variable offset. |
458 | auto IsNonConst = [](Value *V) { return !isa<ConstantInt>(Val: V); }; |
459 | if (!GEP->isInBounds() || count_if(Range: GEP->indices(), P: IsNonConst) > 1) |
460 | return false; |
461 | |
462 | if (!Explored.contains(key: GEP->getOperand(i_nocapture: 0))) |
463 | WorkList.push_back(Elt: GEP->getOperand(i_nocapture: 0)); |
464 | } |
465 | |
466 | if (WorkList.back() == V) { |
467 | WorkList.pop_back(); |
468 | // We've finished visiting this node, mark it as such. |
469 | Explored.insert(X: V); |
470 | } |
471 | |
472 | if (auto *PN = dyn_cast<PHINode>(Val: V)) { |
473 | // We cannot transform PHIs on unsplittable basic blocks. |
474 | if (isa<CatchSwitchInst>(Val: PN->getParent()->getTerminator())) |
475 | return false; |
476 | Explored.insert(X: PN); |
477 | PHIs.insert(X: PN); |
478 | } |
479 | } |
480 | |
481 | // Explore the PHI nodes further. |
482 | for (auto *PN : PHIs) |
483 | for (Value *Op : PN->incoming_values()) |
484 | if (!Explored.contains(key: Op)) |
485 | WorkList.push_back(Elt: Op); |
486 | } |
487 | |
488 | // Make sure that we can do this. Since we can't insert GEPs in a basic |
489 | // block before a PHI node, we can't easily do this transformation if |
490 | // we have PHI node users of transformed instructions. |
491 | for (Value *Val : Explored) { |
492 | for (Value *Use : Val->uses()) { |
493 | |
494 | auto *PHI = dyn_cast<PHINode>(Val: Use); |
495 | auto *Inst = dyn_cast<Instruction>(Val); |
496 | |
497 | if (Inst == Base || Inst == PHI || !Inst || !PHI || |
498 | !Explored.contains(key: PHI)) |
499 | continue; |
500 | |
501 | if (PHI->getParent() == Inst->getParent()) |
502 | return false; |
503 | } |
504 | } |
505 | return true; |
506 | } |
507 | |
508 | // Sets the appropriate insert point on Builder where we can add |
509 | // a replacement Instruction for V (if that is possible). |
510 | static void setInsertionPoint(IRBuilder<> &Builder, Value *V, |
511 | bool Before = true) { |
512 | if (auto *PHI = dyn_cast<PHINode>(Val: V)) { |
513 | BasicBlock *Parent = PHI->getParent(); |
514 | Builder.SetInsertPoint(TheBB: Parent, IP: Parent->getFirstInsertionPt()); |
515 | return; |
516 | } |
517 | if (auto *I = dyn_cast<Instruction>(Val: V)) { |
518 | if (!Before) |
519 | I = &*std::next(x: I->getIterator()); |
520 | Builder.SetInsertPoint(I); |
521 | return; |
522 | } |
523 | if (auto *A = dyn_cast<Argument>(Val: V)) { |
524 | // Set the insertion point in the entry block. |
525 | BasicBlock &Entry = A->getParent()->getEntryBlock(); |
526 | Builder.SetInsertPoint(TheBB: &Entry, IP: Entry.getFirstInsertionPt()); |
527 | return; |
528 | } |
529 | // Otherwise, this is a constant and we don't need to set a new |
530 | // insertion point. |
531 | assert(isa<Constant>(V) && "Setting insertion point for unknown value!" ); |
532 | } |
533 | |
534 | /// Returns a re-written value of Start as an indexed GEP using Base as a |
535 | /// pointer. |
536 | static Value *rewriteGEPAsOffset(Value *Start, Value *Base, |
537 | const DataLayout &DL, |
538 | SetVector<Value *> &Explored, |
539 | InstCombiner &IC) { |
540 | // Perform all the substitutions. This is a bit tricky because we can |
541 | // have cycles in our use-def chains. |
542 | // 1. Create the PHI nodes without any incoming values. |
543 | // 2. Create all the other values. |
544 | // 3. Add the edges for the PHI nodes. |
545 | // 4. Emit GEPs to get the original pointers. |
546 | // 5. Remove the original instructions. |
547 | Type *IndexType = IntegerType::get( |
548 | C&: Base->getContext(), NumBits: DL.getIndexTypeSizeInBits(Ty: Start->getType())); |
549 | |
550 | DenseMap<Value *, Value *> NewInsts; |
551 | NewInsts[Base] = ConstantInt::getNullValue(Ty: IndexType); |
552 | |
553 | // Create the new PHI nodes, without adding any incoming values. |
554 | for (Value *Val : Explored) { |
555 | if (Val == Base) |
556 | continue; |
557 | // Create empty phi nodes. This avoids cyclic dependencies when creating |
558 | // the remaining instructions. |
559 | if (auto *PHI = dyn_cast<PHINode>(Val)) |
560 | NewInsts[PHI] = |
561 | PHINode::Create(Ty: IndexType, NumReservedValues: PHI->getNumIncomingValues(), |
562 | NameStr: PHI->getName() + ".idx" , InsertBefore: PHI->getIterator()); |
563 | } |
564 | IRBuilder<> Builder(Base->getContext()); |
565 | |
566 | // Create all the other instructions. |
567 | for (Value *Val : Explored) { |
568 | if (NewInsts.contains(Val)) |
569 | continue; |
570 | |
571 | if (auto *GEP = dyn_cast<GEPOperator>(Val)) { |
572 | setInsertionPoint(Builder, V: GEP); |
573 | Value *Op = NewInsts[GEP->getOperand(i_nocapture: 0)]; |
574 | Value *OffsetV = emitGEPOffset(Builder: &Builder, DL, GEP); |
575 | if (isa<ConstantInt>(Val: Op) && cast<ConstantInt>(Val: Op)->isZero()) |
576 | NewInsts[GEP] = OffsetV; |
577 | else |
578 | NewInsts[GEP] = Builder.CreateNSWAdd( |
579 | LHS: Op, RHS: OffsetV, Name: GEP->getOperand(i_nocapture: 0)->getName() + ".add" ); |
580 | continue; |
581 | } |
582 | if (isa<PHINode>(Val)) |
583 | continue; |
584 | |
585 | llvm_unreachable("Unexpected instruction type" ); |
586 | } |
587 | |
588 | // Add the incoming values to the PHI nodes. |
589 | for (Value *Val : Explored) { |
590 | if (Val == Base) |
591 | continue; |
592 | // All the instructions have been created, we can now add edges to the |
593 | // phi nodes. |
594 | if (auto *PHI = dyn_cast<PHINode>(Val)) { |
595 | PHINode *NewPhi = static_cast<PHINode *>(NewInsts[PHI]); |
596 | for (unsigned I = 0, E = PHI->getNumIncomingValues(); I < E; ++I) { |
597 | Value *NewIncoming = PHI->getIncomingValue(i: I); |
598 | |
599 | if (NewInsts.contains(Val: NewIncoming)) |
600 | NewIncoming = NewInsts[NewIncoming]; |
601 | |
602 | NewPhi->addIncoming(V: NewIncoming, BB: PHI->getIncomingBlock(i: I)); |
603 | } |
604 | } |
605 | } |
606 | |
607 | for (Value *Val : Explored) { |
608 | if (Val == Base) |
609 | continue; |
610 | |
611 | setInsertionPoint(Builder, V: Val, Before: false); |
612 | // Create GEP for external users. |
613 | Value *NewVal = Builder.CreateInBoundsGEP( |
614 | Ty: Builder.getInt8Ty(), Ptr: Base, IdxList: NewInsts[Val], Name: Val->getName() + ".ptr" ); |
615 | IC.replaceInstUsesWith(I&: *cast<Instruction>(Val), V: NewVal); |
616 | // Add old instruction to worklist for DCE. We don't directly remove it |
617 | // here because the original compare is one of the users. |
618 | IC.addToWorklist(I: cast<Instruction>(Val)); |
619 | } |
620 | |
621 | return NewInsts[Start]; |
622 | } |
623 | |
624 | /// Converts (CMP GEPLHS, RHS) if this change would make RHS a constant. |
625 | /// We can look through PHIs, GEPs and casts in order to determine a common base |
626 | /// between GEPLHS and RHS. |
627 | static Instruction *transformToIndexedCompare(GEPOperator *GEPLHS, Value *RHS, |
628 | ICmpInst::Predicate Cond, |
629 | const DataLayout &DL, |
630 | InstCombiner &IC) { |
631 | // FIXME: Support vector of pointers. |
632 | if (GEPLHS->getType()->isVectorTy()) |
633 | return nullptr; |
634 | |
635 | if (!GEPLHS->hasAllConstantIndices()) |
636 | return nullptr; |
637 | |
638 | APInt Offset(DL.getIndexTypeSizeInBits(Ty: GEPLHS->getType()), 0); |
639 | Value *PtrBase = |
640 | GEPLHS->stripAndAccumulateConstantOffsets(DL, Offset, |
641 | /*AllowNonInbounds*/ false); |
642 | |
643 | // Bail if we looked through addrspacecast. |
644 | if (PtrBase->getType() != GEPLHS->getType()) |
645 | return nullptr; |
646 | |
647 | // The set of nodes that will take part in this transformation. |
648 | SetVector<Value *> Nodes; |
649 | |
650 | if (!canRewriteGEPAsOffset(Start: RHS, Base: PtrBase, DL, Explored&: Nodes)) |
651 | return nullptr; |
652 | |
653 | // We know we can re-write this as |
654 | // ((gep Ptr, OFFSET1) cmp (gep Ptr, OFFSET2) |
655 | // Since we've only looked through inbouds GEPs we know that we |
656 | // can't have overflow on either side. We can therefore re-write |
657 | // this as: |
658 | // OFFSET1 cmp OFFSET2 |
659 | Value *NewRHS = rewriteGEPAsOffset(Start: RHS, Base: PtrBase, DL, Explored&: Nodes, IC); |
660 | |
661 | // RewriteGEPAsOffset has replaced RHS and all of its uses with a re-written |
662 | // GEP having PtrBase as the pointer base, and has returned in NewRHS the |
663 | // offset. Since Index is the offset of LHS to the base pointer, we will now |
664 | // compare the offsets instead of comparing the pointers. |
665 | return new ICmpInst(ICmpInst::getSignedPredicate(pred: Cond), |
666 | IC.Builder.getInt(AI: Offset), NewRHS); |
667 | } |
668 | |
669 | /// Fold comparisons between a GEP instruction and something else. At this point |
670 | /// we know that the GEP is on the LHS of the comparison. |
671 | Instruction *InstCombinerImpl::foldGEPICmp(GEPOperator *GEPLHS, Value *RHS, |
672 | ICmpInst::Predicate Cond, |
673 | Instruction &I) { |
674 | // Don't transform signed compares of GEPs into index compares. Even if the |
675 | // GEP is inbounds, the final add of the base pointer can have signed overflow |
676 | // and would change the result of the icmp. |
677 | // e.g. "&foo[0] <s &foo[1]" can't be folded to "true" because "foo" could be |
678 | // the maximum signed value for the pointer type. |
679 | if (ICmpInst::isSigned(predicate: Cond)) |
680 | return nullptr; |
681 | |
682 | // Look through bitcasts and addrspacecasts. We do not however want to remove |
683 | // 0 GEPs. |
684 | if (!isa<GetElementPtrInst>(Val: RHS)) |
685 | RHS = RHS->stripPointerCasts(); |
686 | |
687 | Value *PtrBase = GEPLHS->getOperand(i_nocapture: 0); |
688 | if (PtrBase == RHS && (GEPLHS->isInBounds() || ICmpInst::isEquality(P: Cond))) { |
689 | // ((gep Ptr, OFFSET) cmp Ptr) ---> (OFFSET cmp 0). |
690 | Value *Offset = EmitGEPOffset(GEP: GEPLHS); |
691 | return new ICmpInst(ICmpInst::getSignedPredicate(pred: Cond), Offset, |
692 | Constant::getNullValue(Ty: Offset->getType())); |
693 | } |
694 | |
695 | if (GEPLHS->isInBounds() && ICmpInst::isEquality(P: Cond) && |
696 | isa<Constant>(Val: RHS) && cast<Constant>(Val: RHS)->isNullValue() && |
697 | !NullPointerIsDefined(F: I.getFunction(), |
698 | AS: RHS->getType()->getPointerAddressSpace())) { |
699 | // For most address spaces, an allocation can't be placed at null, but null |
700 | // itself is treated as a 0 size allocation in the in bounds rules. Thus, |
701 | // the only valid inbounds address derived from null, is null itself. |
702 | // Thus, we have four cases to consider: |
703 | // 1) Base == nullptr, Offset == 0 -> inbounds, null |
704 | // 2) Base == nullptr, Offset != 0 -> poison as the result is out of bounds |
705 | // 3) Base != nullptr, Offset == (-base) -> poison (crossing allocations) |
706 | // 4) Base != nullptr, Offset != (-base) -> nonnull (and possibly poison) |
707 | // |
708 | // (Note if we're indexing a type of size 0, that simply collapses into one |
709 | // of the buckets above.) |
710 | // |
711 | // In general, we're allowed to make values less poison (i.e. remove |
712 | // sources of full UB), so in this case, we just select between the two |
713 | // non-poison cases (1 and 4 above). |
714 | // |
715 | // For vectors, we apply the same reasoning on a per-lane basis. |
716 | auto *Base = GEPLHS->getPointerOperand(); |
717 | if (GEPLHS->getType()->isVectorTy() && Base->getType()->isPointerTy()) { |
718 | auto EC = cast<VectorType>(Val: GEPLHS->getType())->getElementCount(); |
719 | Base = Builder.CreateVectorSplat(EC, V: Base); |
720 | } |
721 | return new ICmpInst(Cond, Base, |
722 | ConstantExpr::getPointerBitCastOrAddrSpaceCast( |
723 | C: cast<Constant>(Val: RHS), Ty: Base->getType())); |
724 | } else if (GEPOperator *GEPRHS = dyn_cast<GEPOperator>(Val: RHS)) { |
725 | // If the base pointers are different, but the indices are the same, just |
726 | // compare the base pointer. |
727 | if (PtrBase != GEPRHS->getOperand(i_nocapture: 0)) { |
728 | bool IndicesTheSame = |
729 | GEPLHS->getNumOperands() == GEPRHS->getNumOperands() && |
730 | GEPLHS->getPointerOperand()->getType() == |
731 | GEPRHS->getPointerOperand()->getType() && |
732 | GEPLHS->getSourceElementType() == GEPRHS->getSourceElementType(); |
733 | if (IndicesTheSame) |
734 | for (unsigned i = 1, e = GEPLHS->getNumOperands(); i != e; ++i) |
735 | if (GEPLHS->getOperand(i_nocapture: i) != GEPRHS->getOperand(i_nocapture: i)) { |
736 | IndicesTheSame = false; |
737 | break; |
738 | } |
739 | |
740 | // If all indices are the same, just compare the base pointers. |
741 | Type *BaseType = GEPLHS->getOperand(i_nocapture: 0)->getType(); |
742 | if (IndicesTheSame && CmpInst::makeCmpResultType(opnd_type: BaseType) == I.getType()) |
743 | return new ICmpInst(Cond, GEPLHS->getOperand(i_nocapture: 0), GEPRHS->getOperand(i_nocapture: 0)); |
744 | |
745 | // If we're comparing GEPs with two base pointers that only differ in type |
746 | // and both GEPs have only constant indices or just one use, then fold |
747 | // the compare with the adjusted indices. |
748 | // FIXME: Support vector of pointers. |
749 | if (GEPLHS->isInBounds() && GEPRHS->isInBounds() && |
750 | (GEPLHS->hasAllConstantIndices() || GEPLHS->hasOneUse()) && |
751 | (GEPRHS->hasAllConstantIndices() || GEPRHS->hasOneUse()) && |
752 | PtrBase->stripPointerCasts() == |
753 | GEPRHS->getOperand(i_nocapture: 0)->stripPointerCasts() && |
754 | !GEPLHS->getType()->isVectorTy()) { |
755 | Value *LOffset = EmitGEPOffset(GEP: GEPLHS); |
756 | Value *ROffset = EmitGEPOffset(GEP: GEPRHS); |
757 | |
758 | // If we looked through an addrspacecast between different sized address |
759 | // spaces, the LHS and RHS pointers are different sized |
760 | // integers. Truncate to the smaller one. |
761 | Type *LHSIndexTy = LOffset->getType(); |
762 | Type *RHSIndexTy = ROffset->getType(); |
763 | if (LHSIndexTy != RHSIndexTy) { |
764 | if (LHSIndexTy->getPrimitiveSizeInBits().getFixedValue() < |
765 | RHSIndexTy->getPrimitiveSizeInBits().getFixedValue()) { |
766 | ROffset = Builder.CreateTrunc(V: ROffset, DestTy: LHSIndexTy); |
767 | } else |
768 | LOffset = Builder.CreateTrunc(V: LOffset, DestTy: RHSIndexTy); |
769 | } |
770 | |
771 | Value *Cmp = Builder.CreateICmp(P: ICmpInst::getSignedPredicate(pred: Cond), |
772 | LHS: LOffset, RHS: ROffset); |
773 | return replaceInstUsesWith(I, V: Cmp); |
774 | } |
775 | |
776 | // Otherwise, the base pointers are different and the indices are |
777 | // different. Try convert this to an indexed compare by looking through |
778 | // PHIs/casts. |
779 | return transformToIndexedCompare(GEPLHS, RHS, Cond, DL, IC&: *this); |
780 | } |
781 | |
782 | bool GEPsInBounds = GEPLHS->isInBounds() && GEPRHS->isInBounds(); |
783 | if (GEPLHS->getNumOperands() == GEPRHS->getNumOperands() && |
784 | GEPLHS->getSourceElementType() == GEPRHS->getSourceElementType()) { |
785 | // If the GEPs only differ by one index, compare it. |
786 | unsigned NumDifferences = 0; // Keep track of # differences. |
787 | unsigned DiffOperand = 0; // The operand that differs. |
788 | for (unsigned i = 1, e = GEPRHS->getNumOperands(); i != e; ++i) |
789 | if (GEPLHS->getOperand(i_nocapture: i) != GEPRHS->getOperand(i_nocapture: i)) { |
790 | Type *LHSType = GEPLHS->getOperand(i_nocapture: i)->getType(); |
791 | Type *RHSType = GEPRHS->getOperand(i_nocapture: i)->getType(); |
792 | // FIXME: Better support for vector of pointers. |
793 | if (LHSType->getPrimitiveSizeInBits() != |
794 | RHSType->getPrimitiveSizeInBits() || |
795 | (GEPLHS->getType()->isVectorTy() && |
796 | (!LHSType->isVectorTy() || !RHSType->isVectorTy()))) { |
797 | // Irreconcilable differences. |
798 | NumDifferences = 2; |
799 | break; |
800 | } |
801 | |
802 | if (NumDifferences++) break; |
803 | DiffOperand = i; |
804 | } |
805 | |
806 | if (NumDifferences == 0) // SAME GEP? |
807 | return replaceInstUsesWith(I, // No comparison is needed here. |
808 | V: ConstantInt::get(Ty: I.getType(), V: ICmpInst::isTrueWhenEqual(predicate: Cond))); |
809 | |
810 | else if (NumDifferences == 1 && GEPsInBounds) { |
811 | Value *LHSV = GEPLHS->getOperand(i_nocapture: DiffOperand); |
812 | Value *RHSV = GEPRHS->getOperand(i_nocapture: DiffOperand); |
813 | // Make sure we do a signed comparison here. |
814 | return new ICmpInst(ICmpInst::getSignedPredicate(pred: Cond), LHSV, RHSV); |
815 | } |
816 | } |
817 | |
818 | if (GEPsInBounds || CmpInst::isEquality(pred: Cond)) { |
819 | auto EmitGEPOffsetAndRewrite = [&](GEPOperator *GEP) { |
820 | IRBuilderBase::InsertPointGuard Guard(Builder); |
821 | auto *Inst = dyn_cast<Instruction>(Val: GEP); |
822 | if (Inst) |
823 | Builder.SetInsertPoint(Inst); |
824 | |
825 | Value *Offset = EmitGEPOffset(GEP); |
826 | // If a non-trivial GEP has other uses, rewrite it to avoid duplicating |
827 | // the offset arithmetic. |
828 | if (Inst && !GEP->hasOneUse() && !GEP->hasAllConstantIndices() && |
829 | !GEP->getSourceElementType()->isIntegerTy(Bitwidth: 8)) { |
830 | replaceInstUsesWith(I&: *Inst, |
831 | V: Builder.CreateGEP(Ty: Builder.getInt8Ty(), |
832 | Ptr: GEP->getPointerOperand(), |
833 | IdxList: Offset, Name: "" , IsInBounds: GEPsInBounds)); |
834 | eraseInstFromFunction(I&: *Inst); |
835 | } |
836 | return Offset; |
837 | }; |
838 | |
839 | // ((gep Ptr, OFFSET1) cmp (gep Ptr, OFFSET2) ---> (OFFSET1 cmp OFFSET2) |
840 | Value *L = EmitGEPOffsetAndRewrite(GEPLHS); |
841 | Value *R = EmitGEPOffsetAndRewrite(GEPRHS); |
842 | return new ICmpInst(ICmpInst::getSignedPredicate(pred: Cond), L, R); |
843 | } |
844 | } |
845 | |
846 | // Try convert this to an indexed compare by looking through PHIs/casts as a |
847 | // last resort. |
848 | return transformToIndexedCompare(GEPLHS, RHS, Cond, DL, IC&: *this); |
849 | } |
850 | |
851 | bool InstCombinerImpl::foldAllocaCmp(AllocaInst *Alloca) { |
852 | // It would be tempting to fold away comparisons between allocas and any |
853 | // pointer not based on that alloca (e.g. an argument). However, even |
854 | // though such pointers cannot alias, they can still compare equal. |
855 | // |
856 | // But LLVM doesn't specify where allocas get their memory, so if the alloca |
857 | // doesn't escape we can argue that it's impossible to guess its value, and we |
858 | // can therefore act as if any such guesses are wrong. |
859 | // |
860 | // However, we need to ensure that this folding is consistent: We can't fold |
861 | // one comparison to false, and then leave a different comparison against the |
862 | // same value alone (as it might evaluate to true at runtime, leading to a |
863 | // contradiction). As such, this code ensures that all comparisons are folded |
864 | // at the same time, and there are no other escapes. |
865 | |
866 | struct CmpCaptureTracker : public CaptureTracker { |
867 | AllocaInst *Alloca; |
868 | bool Captured = false; |
869 | /// The value of the map is a bit mask of which icmp operands the alloca is |
870 | /// used in. |
871 | SmallMapVector<ICmpInst *, unsigned, 4> ICmps; |
872 | |
873 | CmpCaptureTracker(AllocaInst *Alloca) : Alloca(Alloca) {} |
874 | |
875 | void tooManyUses() override { Captured = true; } |
876 | |
877 | bool captured(const Use *U) override { |
878 | auto *ICmp = dyn_cast<ICmpInst>(Val: U->getUser()); |
879 | // We need to check that U is based *only* on the alloca, and doesn't |
880 | // have other contributions from a select/phi operand. |
881 | // TODO: We could check whether getUnderlyingObjects() reduces to one |
882 | // object, which would allow looking through phi nodes. |
883 | if (ICmp && ICmp->isEquality() && getUnderlyingObject(V: *U) == Alloca) { |
884 | // Collect equality icmps of the alloca, and don't treat them as |
885 | // captures. |
886 | auto Res = ICmps.insert(KV: {ICmp, 0}); |
887 | Res.first->second |= 1u << U->getOperandNo(); |
888 | return false; |
889 | } |
890 | |
891 | Captured = true; |
892 | return true; |
893 | } |
894 | }; |
895 | |
896 | CmpCaptureTracker Tracker(Alloca); |
897 | PointerMayBeCaptured(V: Alloca, Tracker: &Tracker); |
898 | if (Tracker.Captured) |
899 | return false; |
900 | |
901 | bool Changed = false; |
902 | for (auto [ICmp, Operands] : Tracker.ICmps) { |
903 | switch (Operands) { |
904 | case 1: |
905 | case 2: { |
906 | // The alloca is only used in one icmp operand. Assume that the |
907 | // equality is false. |
908 | auto *Res = ConstantInt::get( |
909 | Ty: ICmp->getType(), V: ICmp->getPredicate() == ICmpInst::ICMP_NE); |
910 | replaceInstUsesWith(I&: *ICmp, V: Res); |
911 | eraseInstFromFunction(I&: *ICmp); |
912 | Changed = true; |
913 | break; |
914 | } |
915 | case 3: |
916 | // Both icmp operands are based on the alloca, so this is comparing |
917 | // pointer offsets, without leaking any information about the address |
918 | // of the alloca. Ignore such comparisons. |
919 | break; |
920 | default: |
921 | llvm_unreachable("Cannot happen" ); |
922 | } |
923 | } |
924 | |
925 | return Changed; |
926 | } |
927 | |
928 | /// Fold "icmp pred (X+C), X". |
929 | Instruction *InstCombinerImpl::foldICmpAddOpConst(Value *X, const APInt &C, |
930 | ICmpInst::Predicate Pred) { |
931 | // From this point on, we know that (X+C <= X) --> (X+C < X) because C != 0, |
932 | // so the values can never be equal. Similarly for all other "or equals" |
933 | // operators. |
934 | assert(!!C && "C should not be zero!" ); |
935 | |
936 | // (X+1) <u X --> X >u (MAXUINT-1) --> X == 255 |
937 | // (X+2) <u X --> X >u (MAXUINT-2) --> X > 253 |
938 | // (X+MAXUINT) <u X --> X >u (MAXUINT-MAXUINT) --> X != 0 |
939 | if (Pred == ICmpInst::ICMP_ULT || Pred == ICmpInst::ICMP_ULE) { |
940 | Constant *R = ConstantInt::get(Ty: X->getType(), |
941 | V: APInt::getMaxValue(numBits: C.getBitWidth()) - C); |
942 | return new ICmpInst(ICmpInst::ICMP_UGT, X, R); |
943 | } |
944 | |
945 | // (X+1) >u X --> X <u (0-1) --> X != 255 |
946 | // (X+2) >u X --> X <u (0-2) --> X <u 254 |
947 | // (X+MAXUINT) >u X --> X <u (0-MAXUINT) --> X <u 1 --> X == 0 |
948 | if (Pred == ICmpInst::ICMP_UGT || Pred == ICmpInst::ICMP_UGE) |
949 | return new ICmpInst(ICmpInst::ICMP_ULT, X, |
950 | ConstantInt::get(Ty: X->getType(), V: -C)); |
951 | |
952 | APInt SMax = APInt::getSignedMaxValue(numBits: C.getBitWidth()); |
953 | |
954 | // (X+ 1) <s X --> X >s (MAXSINT-1) --> X == 127 |
955 | // (X+ 2) <s X --> X >s (MAXSINT-2) --> X >s 125 |
956 | // (X+MAXSINT) <s X --> X >s (MAXSINT-MAXSINT) --> X >s 0 |
957 | // (X+MINSINT) <s X --> X >s (MAXSINT-MINSINT) --> X >s -1 |
958 | // (X+ -2) <s X --> X >s (MAXSINT- -2) --> X >s 126 |
959 | // (X+ -1) <s X --> X >s (MAXSINT- -1) --> X != 127 |
960 | if (Pred == ICmpInst::ICMP_SLT || Pred == ICmpInst::ICMP_SLE) |
961 | return new ICmpInst(ICmpInst::ICMP_SGT, X, |
962 | ConstantInt::get(Ty: X->getType(), V: SMax - C)); |
963 | |
964 | // (X+ 1) >s X --> X <s (MAXSINT-(1-1)) --> X != 127 |
965 | // (X+ 2) >s X --> X <s (MAXSINT-(2-1)) --> X <s 126 |
966 | // (X+MAXSINT) >s X --> X <s (MAXSINT-(MAXSINT-1)) --> X <s 1 |
967 | // (X+MINSINT) >s X --> X <s (MAXSINT-(MINSINT-1)) --> X <s -2 |
968 | // (X+ -2) >s X --> X <s (MAXSINT-(-2-1)) --> X <s -126 |
969 | // (X+ -1) >s X --> X <s (MAXSINT-(-1-1)) --> X == -128 |
970 | |
971 | assert(Pred == ICmpInst::ICMP_SGT || Pred == ICmpInst::ICMP_SGE); |
972 | return new ICmpInst(ICmpInst::ICMP_SLT, X, |
973 | ConstantInt::get(Ty: X->getType(), V: SMax - (C - 1))); |
974 | } |
975 | |
976 | /// Handle "(icmp eq/ne (ashr/lshr AP2, A), AP1)" -> |
977 | /// (icmp eq/ne A, Log2(AP2/AP1)) -> |
978 | /// (icmp eq/ne A, Log2(AP2) - Log2(AP1)). |
979 | Instruction *InstCombinerImpl::foldICmpShrConstConst(ICmpInst &I, Value *A, |
980 | const APInt &AP1, |
981 | const APInt &AP2) { |
982 | assert(I.isEquality() && "Cannot fold icmp gt/lt" ); |
983 | |
984 | auto getICmp = [&I](CmpInst::Predicate Pred, Value *LHS, Value *RHS) { |
985 | if (I.getPredicate() == I.ICMP_NE) |
986 | Pred = CmpInst::getInversePredicate(pred: Pred); |
987 | return new ICmpInst(Pred, LHS, RHS); |
988 | }; |
989 | |
990 | // Don't bother doing any work for cases which InstSimplify handles. |
991 | if (AP2.isZero()) |
992 | return nullptr; |
993 | |
994 | bool IsAShr = isa<AShrOperator>(Val: I.getOperand(i_nocapture: 0)); |
995 | if (IsAShr) { |
996 | if (AP2.isAllOnes()) |
997 | return nullptr; |
998 | if (AP2.isNegative() != AP1.isNegative()) |
999 | return nullptr; |
1000 | if (AP2.sgt(RHS: AP1)) |
1001 | return nullptr; |
1002 | } |
1003 | |
1004 | if (!AP1) |
1005 | // 'A' must be large enough to shift out the highest set bit. |
1006 | return getICmp(I.ICMP_UGT, A, |
1007 | ConstantInt::get(Ty: A->getType(), V: AP2.logBase2())); |
1008 | |
1009 | if (AP1 == AP2) |
1010 | return getICmp(I.ICMP_EQ, A, ConstantInt::getNullValue(Ty: A->getType())); |
1011 | |
1012 | int Shift; |
1013 | if (IsAShr && AP1.isNegative()) |
1014 | Shift = AP1.countl_one() - AP2.countl_one(); |
1015 | else |
1016 | Shift = AP1.countl_zero() - AP2.countl_zero(); |
1017 | |
1018 | if (Shift > 0) { |
1019 | if (IsAShr && AP1 == AP2.ashr(ShiftAmt: Shift)) { |
1020 | // There are multiple solutions if we are comparing against -1 and the LHS |
1021 | // of the ashr is not a power of two. |
1022 | if (AP1.isAllOnes() && !AP2.isPowerOf2()) |
1023 | return getICmp(I.ICMP_UGE, A, ConstantInt::get(Ty: A->getType(), V: Shift)); |
1024 | return getICmp(I.ICMP_EQ, A, ConstantInt::get(Ty: A->getType(), V: Shift)); |
1025 | } else if (AP1 == AP2.lshr(shiftAmt: Shift)) { |
1026 | return getICmp(I.ICMP_EQ, A, ConstantInt::get(Ty: A->getType(), V: Shift)); |
1027 | } |
1028 | } |
1029 | |
1030 | // Shifting const2 will never be equal to const1. |
1031 | // FIXME: This should always be handled by InstSimplify? |
1032 | auto *TorF = ConstantInt::get(Ty: I.getType(), V: I.getPredicate() == I.ICMP_NE); |
1033 | return replaceInstUsesWith(I, V: TorF); |
1034 | } |
1035 | |
1036 | /// Handle "(icmp eq/ne (shl AP2, A), AP1)" -> |
1037 | /// (icmp eq/ne A, TrailingZeros(AP1) - TrailingZeros(AP2)). |
1038 | Instruction *InstCombinerImpl::foldICmpShlConstConst(ICmpInst &I, Value *A, |
1039 | const APInt &AP1, |
1040 | const APInt &AP2) { |
1041 | assert(I.isEquality() && "Cannot fold icmp gt/lt" ); |
1042 | |
1043 | auto getICmp = [&I](CmpInst::Predicate Pred, Value *LHS, Value *RHS) { |
1044 | if (I.getPredicate() == I.ICMP_NE) |
1045 | Pred = CmpInst::getInversePredicate(pred: Pred); |
1046 | return new ICmpInst(Pred, LHS, RHS); |
1047 | }; |
1048 | |
1049 | // Don't bother doing any work for cases which InstSimplify handles. |
1050 | if (AP2.isZero()) |
1051 | return nullptr; |
1052 | |
1053 | unsigned AP2TrailingZeros = AP2.countr_zero(); |
1054 | |
1055 | if (!AP1 && AP2TrailingZeros != 0) |
1056 | return getICmp( |
1057 | I.ICMP_UGE, A, |
1058 | ConstantInt::get(Ty: A->getType(), V: AP2.getBitWidth() - AP2TrailingZeros)); |
1059 | |
1060 | if (AP1 == AP2) |
1061 | return getICmp(I.ICMP_EQ, A, ConstantInt::getNullValue(Ty: A->getType())); |
1062 | |
1063 | // Get the distance between the lowest bits that are set. |
1064 | int Shift = AP1.countr_zero() - AP2TrailingZeros; |
1065 | |
1066 | if (Shift > 0 && AP2.shl(shiftAmt: Shift) == AP1) |
1067 | return getICmp(I.ICMP_EQ, A, ConstantInt::get(Ty: A->getType(), V: Shift)); |
1068 | |
1069 | // Shifting const2 will never be equal to const1. |
1070 | // FIXME: This should always be handled by InstSimplify? |
1071 | auto *TorF = ConstantInt::get(Ty: I.getType(), V: I.getPredicate() == I.ICMP_NE); |
1072 | return replaceInstUsesWith(I, V: TorF); |
1073 | } |
1074 | |
1075 | /// The caller has matched a pattern of the form: |
1076 | /// I = icmp ugt (add (add A, B), CI2), CI1 |
1077 | /// If this is of the form: |
1078 | /// sum = a + b |
1079 | /// if (sum+128 >u 255) |
1080 | /// Then replace it with llvm.sadd.with.overflow.i8. |
1081 | /// |
1082 | static Instruction *processUGT_ADDCST_ADD(ICmpInst &I, Value *A, Value *B, |
1083 | ConstantInt *CI2, ConstantInt *CI1, |
1084 | InstCombinerImpl &IC) { |
1085 | // The transformation we're trying to do here is to transform this into an |
1086 | // llvm.sadd.with.overflow. To do this, we have to replace the original add |
1087 | // with a narrower add, and discard the add-with-constant that is part of the |
1088 | // range check (if we can't eliminate it, this isn't profitable). |
1089 | |
1090 | // In order to eliminate the add-with-constant, the compare can be its only |
1091 | // use. |
1092 | Instruction *AddWithCst = cast<Instruction>(Val: I.getOperand(i_nocapture: 0)); |
1093 | if (!AddWithCst->hasOneUse()) |
1094 | return nullptr; |
1095 | |
1096 | // If CI2 is 2^7, 2^15, 2^31, then it might be an sadd.with.overflow. |
1097 | if (!CI2->getValue().isPowerOf2()) |
1098 | return nullptr; |
1099 | unsigned NewWidth = CI2->getValue().countr_zero(); |
1100 | if (NewWidth != 7 && NewWidth != 15 && NewWidth != 31) |
1101 | return nullptr; |
1102 | |
1103 | // The width of the new add formed is 1 more than the bias. |
1104 | ++NewWidth; |
1105 | |
1106 | // Check to see that CI1 is an all-ones value with NewWidth bits. |
1107 | if (CI1->getBitWidth() == NewWidth || |
1108 | CI1->getValue() != APInt::getLowBitsSet(numBits: CI1->getBitWidth(), loBitsSet: NewWidth)) |
1109 | return nullptr; |
1110 | |
1111 | // This is only really a signed overflow check if the inputs have been |
1112 | // sign-extended; check for that condition. For example, if CI2 is 2^31 and |
1113 | // the operands of the add are 64 bits wide, we need at least 33 sign bits. |
1114 | if (IC.ComputeMaxSignificantBits(Op: A, Depth: 0, CxtI: &I) > NewWidth || |
1115 | IC.ComputeMaxSignificantBits(Op: B, Depth: 0, CxtI: &I) > NewWidth) |
1116 | return nullptr; |
1117 | |
1118 | // In order to replace the original add with a narrower |
1119 | // llvm.sadd.with.overflow, the only uses allowed are the add-with-constant |
1120 | // and truncates that discard the high bits of the add. Verify that this is |
1121 | // the case. |
1122 | Instruction *OrigAdd = cast<Instruction>(Val: AddWithCst->getOperand(i: 0)); |
1123 | for (User *U : OrigAdd->users()) { |
1124 | if (U == AddWithCst) |
1125 | continue; |
1126 | |
1127 | // Only accept truncates for now. We would really like a nice recursive |
1128 | // predicate like SimplifyDemandedBits, but which goes downwards the use-def |
1129 | // chain to see which bits of a value are actually demanded. If the |
1130 | // original add had another add which was then immediately truncated, we |
1131 | // could still do the transformation. |
1132 | TruncInst *TI = dyn_cast<TruncInst>(Val: U); |
1133 | if (!TI || TI->getType()->getPrimitiveSizeInBits() > NewWidth) |
1134 | return nullptr; |
1135 | } |
1136 | |
1137 | // If the pattern matches, truncate the inputs to the narrower type and |
1138 | // use the sadd_with_overflow intrinsic to efficiently compute both the |
1139 | // result and the overflow bit. |
1140 | Type *NewType = IntegerType::get(C&: OrigAdd->getContext(), NumBits: NewWidth); |
1141 | Function *F = Intrinsic::getDeclaration( |
1142 | M: I.getModule(), Intrinsic::id: sadd_with_overflow, Tys: NewType); |
1143 | |
1144 | InstCombiner::BuilderTy &Builder = IC.Builder; |
1145 | |
1146 | // Put the new code above the original add, in case there are any uses of the |
1147 | // add between the add and the compare. |
1148 | Builder.SetInsertPoint(OrigAdd); |
1149 | |
1150 | Value *TruncA = Builder.CreateTrunc(V: A, DestTy: NewType, Name: A->getName() + ".trunc" ); |
1151 | Value *TruncB = Builder.CreateTrunc(V: B, DestTy: NewType, Name: B->getName() + ".trunc" ); |
1152 | CallInst *Call = Builder.CreateCall(Callee: F, Args: {TruncA, TruncB}, Name: "sadd" ); |
1153 | Value *Add = Builder.CreateExtractValue(Agg: Call, Idxs: 0, Name: "sadd.result" ); |
1154 | Value *ZExt = Builder.CreateZExt(V: Add, DestTy: OrigAdd->getType()); |
1155 | |
1156 | // The inner add was the result of the narrow add, zero extended to the |
1157 | // wider type. Replace it with the result computed by the intrinsic. |
1158 | IC.replaceInstUsesWith(I&: *OrigAdd, V: ZExt); |
1159 | IC.eraseInstFromFunction(I&: *OrigAdd); |
1160 | |
1161 | // The original icmp gets replaced with the overflow value. |
1162 | return ExtractValueInst::Create(Agg: Call, Idxs: 1, NameStr: "sadd.overflow" ); |
1163 | } |
1164 | |
1165 | /// If we have: |
1166 | /// icmp eq/ne (urem/srem %x, %y), 0 |
1167 | /// iff %y is a power-of-two, we can replace this with a bit test: |
1168 | /// icmp eq/ne (and %x, (add %y, -1)), 0 |
1169 | Instruction *InstCombinerImpl::foldIRemByPowerOfTwoToBitTest(ICmpInst &I) { |
1170 | // This fold is only valid for equality predicates. |
1171 | if (!I.isEquality()) |
1172 | return nullptr; |
1173 | ICmpInst::Predicate Pred; |
1174 | Value *X, *Y, *Zero; |
1175 | if (!match(V: &I, P: m_ICmp(Pred, L: m_OneUse(SubPattern: m_IRem(L: m_Value(V&: X), R: m_Value(V&: Y))), |
1176 | R: m_CombineAnd(L: m_Zero(), R: m_Value(V&: Zero))))) |
1177 | return nullptr; |
1178 | if (!isKnownToBeAPowerOfTwo(V: Y, /*OrZero*/ true, Depth: 0, CxtI: &I)) |
1179 | return nullptr; |
1180 | // This may increase instruction count, we don't enforce that Y is a constant. |
1181 | Value *Mask = Builder.CreateAdd(LHS: Y, RHS: Constant::getAllOnesValue(Ty: Y->getType())); |
1182 | Value *Masked = Builder.CreateAnd(LHS: X, RHS: Mask); |
1183 | return ICmpInst::Create(Op: Instruction::ICmp, Pred, S1: Masked, S2: Zero); |
1184 | } |
1185 | |
1186 | /// Fold equality-comparison between zero and any (maybe truncated) right-shift |
1187 | /// by one-less-than-bitwidth into a sign test on the original value. |
1188 | Instruction *InstCombinerImpl::foldSignBitTest(ICmpInst &I) { |
1189 | Instruction *Val; |
1190 | ICmpInst::Predicate Pred; |
1191 | if (!I.isEquality() || !match(V: &I, P: m_ICmp(Pred, L: m_Instruction(I&: Val), R: m_Zero()))) |
1192 | return nullptr; |
1193 | |
1194 | Value *X; |
1195 | Type *XTy; |
1196 | |
1197 | Constant *C; |
1198 | if (match(V: Val, P: m_TruncOrSelf(Op: m_Shr(L: m_Value(V&: X), R: m_Constant(C))))) { |
1199 | XTy = X->getType(); |
1200 | unsigned XBitWidth = XTy->getScalarSizeInBits(); |
1201 | if (!match(V: C, P: m_SpecificInt_ICMP(Predicate: ICmpInst::Predicate::ICMP_EQ, |
1202 | Threshold: APInt(XBitWidth, XBitWidth - 1)))) |
1203 | return nullptr; |
1204 | } else if (isa<BinaryOperator>(Val) && |
1205 | (X = reassociateShiftAmtsOfTwoSameDirectionShifts( |
1206 | Sh0: cast<BinaryOperator>(Val), SQ: SQ.getWithInstruction(I: Val), |
1207 | /*AnalyzeForSignBitExtraction=*/true))) { |
1208 | XTy = X->getType(); |
1209 | } else |
1210 | return nullptr; |
1211 | |
1212 | return ICmpInst::Create(Op: Instruction::ICmp, |
1213 | Pred: Pred == ICmpInst::ICMP_EQ ? ICmpInst::ICMP_SGE |
1214 | : ICmpInst::ICMP_SLT, |
1215 | S1: X, S2: ConstantInt::getNullValue(Ty: XTy)); |
1216 | } |
1217 | |
1218 | // Handle icmp pred X, 0 |
1219 | Instruction *InstCombinerImpl::foldICmpWithZero(ICmpInst &Cmp) { |
1220 | CmpInst::Predicate Pred = Cmp.getPredicate(); |
1221 | if (!match(V: Cmp.getOperand(i_nocapture: 1), P: m_Zero())) |
1222 | return nullptr; |
1223 | |
1224 | // (icmp sgt smin(PosA, B) 0) -> (icmp sgt B 0) |
1225 | if (Pred == ICmpInst::ICMP_SGT) { |
1226 | Value *A, *B; |
1227 | if (match(V: Cmp.getOperand(i_nocapture: 0), P: m_SMin(L: m_Value(V&: A), R: m_Value(V&: B)))) { |
1228 | if (isKnownPositive(V: A, SQ: SQ.getWithInstruction(I: &Cmp))) |
1229 | return new ICmpInst(Pred, B, Cmp.getOperand(i_nocapture: 1)); |
1230 | if (isKnownPositive(V: B, SQ: SQ.getWithInstruction(I: &Cmp))) |
1231 | return new ICmpInst(Pred, A, Cmp.getOperand(i_nocapture: 1)); |
1232 | } |
1233 | } |
1234 | |
1235 | if (Instruction *New = foldIRemByPowerOfTwoToBitTest(I&: Cmp)) |
1236 | return New; |
1237 | |
1238 | // Given: |
1239 | // icmp eq/ne (urem %x, %y), 0 |
1240 | // Iff %x has 0 or 1 bits set, and %y has at least 2 bits set, omit 'urem': |
1241 | // icmp eq/ne %x, 0 |
1242 | Value *X, *Y; |
1243 | if (match(V: Cmp.getOperand(i_nocapture: 0), P: m_URem(L: m_Value(V&: X), R: m_Value(V&: Y))) && |
1244 | ICmpInst::isEquality(P: Pred)) { |
1245 | KnownBits XKnown = computeKnownBits(V: X, Depth: 0, CxtI: &Cmp); |
1246 | KnownBits YKnown = computeKnownBits(V: Y, Depth: 0, CxtI: &Cmp); |
1247 | if (XKnown.countMaxPopulation() == 1 && YKnown.countMinPopulation() >= 2) |
1248 | return new ICmpInst(Pred, X, Cmp.getOperand(i_nocapture: 1)); |
1249 | } |
1250 | |
1251 | // (icmp eq/ne (mul X Y)) -> (icmp eq/ne X/Y) if we know about whether X/Y are |
1252 | // odd/non-zero/there is no overflow. |
1253 | if (match(V: Cmp.getOperand(i_nocapture: 0), P: m_Mul(L: m_Value(V&: X), R: m_Value(V&: Y))) && |
1254 | ICmpInst::isEquality(P: Pred)) { |
1255 | |
1256 | KnownBits XKnown = computeKnownBits(V: X, Depth: 0, CxtI: &Cmp); |
1257 | // if X % 2 != 0 |
1258 | // (icmp eq/ne Y) |
1259 | if (XKnown.countMaxTrailingZeros() == 0) |
1260 | return new ICmpInst(Pred, Y, Cmp.getOperand(i_nocapture: 1)); |
1261 | |
1262 | KnownBits YKnown = computeKnownBits(V: Y, Depth: 0, CxtI: &Cmp); |
1263 | // if Y % 2 != 0 |
1264 | // (icmp eq/ne X) |
1265 | if (YKnown.countMaxTrailingZeros() == 0) |
1266 | return new ICmpInst(Pred, X, Cmp.getOperand(i_nocapture: 1)); |
1267 | |
1268 | auto *BO0 = cast<OverflowingBinaryOperator>(Val: Cmp.getOperand(i_nocapture: 0)); |
1269 | if (BO0->hasNoUnsignedWrap() || BO0->hasNoSignedWrap()) { |
1270 | const SimplifyQuery Q = SQ.getWithInstruction(I: &Cmp); |
1271 | // `isKnownNonZero` does more analysis than just `!KnownBits.One.isZero()` |
1272 | // but to avoid unnecessary work, first just if this is an obvious case. |
1273 | |
1274 | // if X non-zero and NoOverflow(X * Y) |
1275 | // (icmp eq/ne Y) |
1276 | if (!XKnown.One.isZero() || isKnownNonZero(V: X, Q)) |
1277 | return new ICmpInst(Pred, Y, Cmp.getOperand(i_nocapture: 1)); |
1278 | |
1279 | // if Y non-zero and NoOverflow(X * Y) |
1280 | // (icmp eq/ne X) |
1281 | if (!YKnown.One.isZero() || isKnownNonZero(V: Y, Q)) |
1282 | return new ICmpInst(Pred, X, Cmp.getOperand(i_nocapture: 1)); |
1283 | } |
1284 | // Note, we are skipping cases: |
1285 | // if Y % 2 != 0 AND X % 2 != 0 |
1286 | // (false/true) |
1287 | // if X non-zero and Y non-zero and NoOverflow(X * Y) |
1288 | // (false/true) |
1289 | // Those can be simplified later as we would have already replaced the (icmp |
1290 | // eq/ne (mul X, Y)) with (icmp eq/ne X/Y) and if X/Y is known non-zero that |
1291 | // will fold to a constant elsewhere. |
1292 | } |
1293 | return nullptr; |
1294 | } |
1295 | |
1296 | /// Fold icmp Pred X, C. |
1297 | /// TODO: This code structure does not make sense. The saturating add fold |
1298 | /// should be moved to some other helper and extended as noted below (it is also |
1299 | /// possible that code has been made unnecessary - do we canonicalize IR to |
1300 | /// overflow/saturating intrinsics or not?). |
1301 | Instruction *InstCombinerImpl::foldICmpWithConstant(ICmpInst &Cmp) { |
1302 | // Match the following pattern, which is a common idiom when writing |
1303 | // overflow-safe integer arithmetic functions. The source performs an addition |
1304 | // in wider type and explicitly checks for overflow using comparisons against |
1305 | // INT_MIN and INT_MAX. Simplify by using the sadd_with_overflow intrinsic. |
1306 | // |
1307 | // TODO: This could probably be generalized to handle other overflow-safe |
1308 | // operations if we worked out the formulas to compute the appropriate magic |
1309 | // constants. |
1310 | // |
1311 | // sum = a + b |
1312 | // if (sum+128 >u 255) ... -> llvm.sadd.with.overflow.i8 |
1313 | CmpInst::Predicate Pred = Cmp.getPredicate(); |
1314 | Value *Op0 = Cmp.getOperand(i_nocapture: 0), *Op1 = Cmp.getOperand(i_nocapture: 1); |
1315 | Value *A, *B; |
1316 | ConstantInt *CI, *CI2; // I = icmp ugt (add (add A, B), CI2), CI |
1317 | if (Pred == ICmpInst::ICMP_UGT && match(V: Op1, P: m_ConstantInt(CI)) && |
1318 | match(V: Op0, P: m_Add(L: m_Add(L: m_Value(V&: A), R: m_Value(V&: B)), R: m_ConstantInt(CI&: CI2)))) |
1319 | if (Instruction *Res = processUGT_ADDCST_ADD(I&: Cmp, A, B, CI2, CI1: CI, IC&: *this)) |
1320 | return Res; |
1321 | |
1322 | // icmp(phi(C1, C2, ...), C) -> phi(icmp(C1, C), icmp(C2, C), ...). |
1323 | Constant *C = dyn_cast<Constant>(Val: Op1); |
1324 | if (!C) |
1325 | return nullptr; |
1326 | |
1327 | if (auto *Phi = dyn_cast<PHINode>(Val: Op0)) |
1328 | if (all_of(Range: Phi->operands(), P: [](Value *V) { return isa<Constant>(Val: V); })) { |
1329 | SmallVector<Constant *> Ops; |
1330 | for (Value *V : Phi->incoming_values()) { |
1331 | Constant *Res = |
1332 | ConstantFoldCompareInstOperands(Predicate: Pred, LHS: cast<Constant>(Val: V), RHS: C, DL); |
1333 | if (!Res) |
1334 | return nullptr; |
1335 | Ops.push_back(Elt: Res); |
1336 | } |
1337 | Builder.SetInsertPoint(Phi); |
1338 | PHINode *NewPhi = Builder.CreatePHI(Ty: Cmp.getType(), NumReservedValues: Phi->getNumOperands()); |
1339 | for (auto [V, Pred] : zip(t&: Ops, u: Phi->blocks())) |
1340 | NewPhi->addIncoming(V, BB: Pred); |
1341 | return replaceInstUsesWith(I&: Cmp, V: NewPhi); |
1342 | } |
1343 | |
1344 | if (Instruction *R = tryFoldInstWithCtpopWithNot(I: &Cmp)) |
1345 | return R; |
1346 | |
1347 | return nullptr; |
1348 | } |
1349 | |
1350 | /// Canonicalize icmp instructions based on dominating conditions. |
1351 | Instruction *InstCombinerImpl::foldICmpWithDominatingICmp(ICmpInst &Cmp) { |
1352 | // We already checked simple implication in InstSimplify, only handle complex |
1353 | // cases here. |
1354 | Value *X = Cmp.getOperand(i_nocapture: 0), *Y = Cmp.getOperand(i_nocapture: 1); |
1355 | const APInt *C; |
1356 | if (!match(V: Y, P: m_APInt(Res&: C))) |
1357 | return nullptr; |
1358 | |
1359 | CmpInst::Predicate Pred = Cmp.getPredicate(); |
1360 | ConstantRange CR = ConstantRange::makeExactICmpRegion(Pred, Other: *C); |
1361 | |
1362 | auto handleDomCond = [&](ICmpInst::Predicate DomPred, |
1363 | const APInt *DomC) -> Instruction * { |
1364 | // We have 2 compares of a variable with constants. Calculate the constant |
1365 | // ranges of those compares to see if we can transform the 2nd compare: |
1366 | // DomBB: |
1367 | // DomCond = icmp DomPred X, DomC |
1368 | // br DomCond, CmpBB, FalseBB |
1369 | // CmpBB: |
1370 | // Cmp = icmp Pred X, C |
1371 | ConstantRange DominatingCR = |
1372 | ConstantRange::makeExactICmpRegion(Pred: DomPred, Other: *DomC); |
1373 | ConstantRange Intersection = DominatingCR.intersectWith(CR); |
1374 | ConstantRange Difference = DominatingCR.difference(CR); |
1375 | if (Intersection.isEmptySet()) |
1376 | return replaceInstUsesWith(I&: Cmp, V: Builder.getFalse()); |
1377 | if (Difference.isEmptySet()) |
1378 | return replaceInstUsesWith(I&: Cmp, V: Builder.getTrue()); |
1379 | |
1380 | // Canonicalizing a sign bit comparison that gets used in a branch, |
1381 | // pessimizes codegen by generating branch on zero instruction instead |
1382 | // of a test and branch. So we avoid canonicalizing in such situations |
1383 | // because test and branch instruction has better branch displacement |
1384 | // than compare and branch instruction. |
1385 | bool UnusedBit; |
1386 | bool IsSignBit = isSignBitCheck(Pred, RHS: *C, TrueIfSigned&: UnusedBit); |
1387 | if (Cmp.isEquality() || (IsSignBit && hasBranchUse(I&: Cmp))) |
1388 | return nullptr; |
1389 | |
1390 | // Avoid an infinite loop with min/max canonicalization. |
1391 | // TODO: This will be unnecessary if we canonicalize to min/max intrinsics. |
1392 | if (Cmp.hasOneUse() && |
1393 | match(V: Cmp.user_back(), P: m_MaxOrMin(L: m_Value(), R: m_Value()))) |
1394 | return nullptr; |
1395 | |
1396 | if (const APInt *EqC = Intersection.getSingleElement()) |
1397 | return new ICmpInst(ICmpInst::ICMP_EQ, X, Builder.getInt(AI: *EqC)); |
1398 | if (const APInt *NeC = Difference.getSingleElement()) |
1399 | return new ICmpInst(ICmpInst::ICMP_NE, X, Builder.getInt(AI: *NeC)); |
1400 | return nullptr; |
1401 | }; |
1402 | |
1403 | for (BranchInst *BI : DC.conditionsFor(V: X)) { |
1404 | ICmpInst::Predicate DomPred; |
1405 | const APInt *DomC; |
1406 | if (!match(V: BI->getCondition(), |
1407 | P: m_ICmp(Pred&: DomPred, L: m_Specific(V: X), R: m_APInt(Res&: DomC)))) |
1408 | continue; |
1409 | |
1410 | BasicBlockEdge Edge0(BI->getParent(), BI->getSuccessor(i: 0)); |
1411 | if (DT.dominates(BBE: Edge0, BB: Cmp.getParent())) { |
1412 | if (auto *V = handleDomCond(DomPred, DomC)) |
1413 | return V; |
1414 | } else { |
1415 | BasicBlockEdge Edge1(BI->getParent(), BI->getSuccessor(i: 1)); |
1416 | if (DT.dominates(BBE: Edge1, BB: Cmp.getParent())) |
1417 | if (auto *V = |
1418 | handleDomCond(CmpInst::getInversePredicate(pred: DomPred), DomC)) |
1419 | return V; |
1420 | } |
1421 | } |
1422 | |
1423 | return nullptr; |
1424 | } |
1425 | |
1426 | /// Fold icmp (trunc X), C. |
1427 | Instruction *InstCombinerImpl::foldICmpTruncConstant(ICmpInst &Cmp, |
1428 | TruncInst *Trunc, |
1429 | const APInt &C) { |
1430 | ICmpInst::Predicate Pred = Cmp.getPredicate(); |
1431 | Value *X = Trunc->getOperand(i_nocapture: 0); |
1432 | if (C.isOne() && C.getBitWidth() > 1) { |
1433 | // icmp slt trunc(signum(V)) 1 --> icmp slt V, 1 |
1434 | Value *V = nullptr; |
1435 | if (Pred == ICmpInst::ICMP_SLT && match(V: X, P: m_Signum(V: m_Value(V)))) |
1436 | return new ICmpInst(ICmpInst::ICMP_SLT, V, |
1437 | ConstantInt::get(Ty: V->getType(), V: 1)); |
1438 | } |
1439 | |
1440 | Type *SrcTy = X->getType(); |
1441 | unsigned DstBits = Trunc->getType()->getScalarSizeInBits(), |
1442 | SrcBits = SrcTy->getScalarSizeInBits(); |
1443 | |
1444 | // TODO: Handle any shifted constant by subtracting trailing zeros. |
1445 | // TODO: Handle non-equality predicates. |
1446 | Value *Y; |
1447 | if (Cmp.isEquality() && match(V: X, P: m_Shl(L: m_One(), R: m_Value(V&: Y)))) { |
1448 | // (trunc (1 << Y) to iN) == 0 --> Y u>= N |
1449 | // (trunc (1 << Y) to iN) != 0 --> Y u< N |
1450 | if (C.isZero()) { |
1451 | auto NewPred = (Pred == Cmp.ICMP_EQ) ? Cmp.ICMP_UGE : Cmp.ICMP_ULT; |
1452 | return new ICmpInst(NewPred, Y, ConstantInt::get(Ty: SrcTy, V: DstBits)); |
1453 | } |
1454 | // (trunc (1 << Y) to iN) == 2**C --> Y == C |
1455 | // (trunc (1 << Y) to iN) != 2**C --> Y != C |
1456 | if (C.isPowerOf2()) |
1457 | return new ICmpInst(Pred, Y, ConstantInt::get(Ty: SrcTy, V: C.logBase2())); |
1458 | } |
1459 | |
1460 | if (Cmp.isEquality() && Trunc->hasOneUse()) { |
1461 | // Canonicalize to a mask and wider compare if the wide type is suitable: |
1462 | // (trunc X to i8) == C --> (X & 0xff) == (zext C) |
1463 | if (!SrcTy->isVectorTy() && shouldChangeType(FromBitWidth: DstBits, ToBitWidth: SrcBits)) { |
1464 | Constant *Mask = |
1465 | ConstantInt::get(Ty: SrcTy, V: APInt::getLowBitsSet(numBits: SrcBits, loBitsSet: DstBits)); |
1466 | Value *And = Builder.CreateAnd(LHS: X, RHS: Mask); |
1467 | Constant *WideC = ConstantInt::get(Ty: SrcTy, V: C.zext(width: SrcBits)); |
1468 | return new ICmpInst(Pred, And, WideC); |
1469 | } |
1470 | |
1471 | // Simplify icmp eq (trunc x to i8), 42 -> icmp eq x, 42|highbits if all |
1472 | // of the high bits truncated out of x are known. |
1473 | KnownBits Known = computeKnownBits(V: X, Depth: 0, CxtI: &Cmp); |
1474 | |
1475 | // If all the high bits are known, we can do this xform. |
1476 | if ((Known.Zero | Known.One).countl_one() >= SrcBits - DstBits) { |
1477 | // Pull in the high bits from known-ones set. |
1478 | APInt NewRHS = C.zext(width: SrcBits); |
1479 | NewRHS |= Known.One & APInt::getHighBitsSet(numBits: SrcBits, hiBitsSet: SrcBits - DstBits); |
1480 | return new ICmpInst(Pred, X, ConstantInt::get(Ty: SrcTy, V: NewRHS)); |
1481 | } |
1482 | } |
1483 | |
1484 | // Look through truncated right-shift of the sign-bit for a sign-bit check: |
1485 | // trunc iN (ShOp >> ShAmtC) to i[N - ShAmtC] < 0 --> ShOp < 0 |
1486 | // trunc iN (ShOp >> ShAmtC) to i[N - ShAmtC] > -1 --> ShOp > -1 |
1487 | Value *ShOp; |
1488 | const APInt *ShAmtC; |
1489 | bool TrueIfSigned; |
1490 | if (isSignBitCheck(Pred, RHS: C, TrueIfSigned) && |
1491 | match(V: X, P: m_Shr(L: m_Value(V&: ShOp), R: m_APInt(Res&: ShAmtC))) && |
1492 | DstBits == SrcBits - ShAmtC->getZExtValue()) { |
1493 | return TrueIfSigned ? new ICmpInst(ICmpInst::ICMP_SLT, ShOp, |
1494 | ConstantInt::getNullValue(Ty: SrcTy)) |
1495 | : new ICmpInst(ICmpInst::ICMP_SGT, ShOp, |
1496 | ConstantInt::getAllOnesValue(Ty: SrcTy)); |
1497 | } |
1498 | |
1499 | return nullptr; |
1500 | } |
1501 | |
1502 | /// Fold icmp (trunc X), (trunc Y). |
1503 | /// Fold icmp (trunc X), (zext Y). |
1504 | Instruction * |
1505 | InstCombinerImpl::foldICmpTruncWithTruncOrExt(ICmpInst &Cmp, |
1506 | const SimplifyQuery &Q) { |
1507 | if (Cmp.isSigned()) |
1508 | return nullptr; |
1509 | |
1510 | Value *X, *Y; |
1511 | ICmpInst::Predicate Pred; |
1512 | bool YIsZext = false; |
1513 | // Try to match icmp (trunc X), (trunc Y) |
1514 | if (match(V: &Cmp, P: m_ICmp(Pred, L: m_Trunc(Op: m_Value(V&: X)), R: m_Trunc(Op: m_Value(V&: Y))))) { |
1515 | if (X->getType() != Y->getType() && |
1516 | (!Cmp.getOperand(i_nocapture: 0)->hasOneUse() || !Cmp.getOperand(i_nocapture: 1)->hasOneUse())) |
1517 | return nullptr; |
1518 | if (!isDesirableIntType(BitWidth: X->getType()->getScalarSizeInBits()) && |
1519 | isDesirableIntType(BitWidth: Y->getType()->getScalarSizeInBits())) { |
1520 | std::swap(a&: X, b&: Y); |
1521 | Pred = Cmp.getSwappedPredicate(pred: Pred); |
1522 | } |
1523 | } |
1524 | // Try to match icmp (trunc X), (zext Y) |
1525 | else if (match(V: &Cmp, P: m_c_ICmp(Pred, L: m_Trunc(Op: m_Value(V&: X)), |
1526 | R: m_OneUse(SubPattern: m_ZExt(Op: m_Value(V&: Y)))))) |
1527 | |
1528 | YIsZext = true; |
1529 | else |
1530 | return nullptr; |
1531 | |
1532 | Type *TruncTy = Cmp.getOperand(i_nocapture: 0)->getType(); |
1533 | unsigned TruncBits = TruncTy->getScalarSizeInBits(); |
1534 | |
1535 | // If this transform will end up changing from desirable types -> undesirable |
1536 | // types skip it. |
1537 | if (isDesirableIntType(BitWidth: TruncBits) && |
1538 | !isDesirableIntType(BitWidth: X->getType()->getScalarSizeInBits())) |
1539 | return nullptr; |
1540 | |
1541 | // Check if the trunc is unneeded. |
1542 | KnownBits KnownX = llvm::computeKnownBits(V: X, /*Depth*/ 0, Q); |
1543 | if (KnownX.countMaxActiveBits() > TruncBits) |
1544 | return nullptr; |
1545 | |
1546 | if (!YIsZext) { |
1547 | // If Y is also a trunc, make sure it is unneeded. |
1548 | KnownBits KnownY = llvm::computeKnownBits(V: Y, /*Depth*/ 0, Q); |
1549 | if (KnownY.countMaxActiveBits() > TruncBits) |
1550 | return nullptr; |
1551 | } |
1552 | |
1553 | Value *NewY = Builder.CreateZExtOrTrunc(V: Y, DestTy: X->getType()); |
1554 | return new ICmpInst(Pred, X, NewY); |
1555 | } |
1556 | |
1557 | /// Fold icmp (xor X, Y), C. |
1558 | Instruction *InstCombinerImpl::foldICmpXorConstant(ICmpInst &Cmp, |
1559 | BinaryOperator *Xor, |
1560 | const APInt &C) { |
1561 | if (Instruction *I = foldICmpXorShiftConst(Cmp, Xor, C)) |
1562 | return I; |
1563 | |
1564 | Value *X = Xor->getOperand(i_nocapture: 0); |
1565 | Value *Y = Xor->getOperand(i_nocapture: 1); |
1566 | const APInt *XorC; |
1567 | if (!match(V: Y, P: m_APInt(Res&: XorC))) |
1568 | return nullptr; |
1569 | |
1570 | // If this is a comparison that tests the signbit (X < 0) or (x > -1), |
1571 | // fold the xor. |
1572 | ICmpInst::Predicate Pred = Cmp.getPredicate(); |
1573 | bool TrueIfSigned = false; |
1574 | if (isSignBitCheck(Pred: Cmp.getPredicate(), RHS: C, TrueIfSigned)) { |
1575 | |
1576 | // If the sign bit of the XorCst is not set, there is no change to |
1577 | // the operation, just stop using the Xor. |
1578 | if (!XorC->isNegative()) |
1579 | return replaceOperand(I&: Cmp, OpNum: 0, V: X); |
1580 | |
1581 | // Emit the opposite comparison. |
1582 | if (TrueIfSigned) |
1583 | return new ICmpInst(ICmpInst::ICMP_SGT, X, |
1584 | ConstantInt::getAllOnesValue(Ty: X->getType())); |
1585 | else |
1586 | return new ICmpInst(ICmpInst::ICMP_SLT, X, |
1587 | ConstantInt::getNullValue(Ty: X->getType())); |
1588 | } |
1589 | |
1590 | if (Xor->hasOneUse()) { |
1591 | // (icmp u/s (xor X SignMask), C) -> (icmp s/u X, (xor C SignMask)) |
1592 | if (!Cmp.isEquality() && XorC->isSignMask()) { |
1593 | Pred = Cmp.getFlippedSignednessPredicate(); |
1594 | return new ICmpInst(Pred, X, ConstantInt::get(Ty: X->getType(), V: C ^ *XorC)); |
1595 | } |
1596 | |
1597 | // (icmp u/s (xor X ~SignMask), C) -> (icmp s/u X, (xor C ~SignMask)) |
1598 | if (!Cmp.isEquality() && XorC->isMaxSignedValue()) { |
1599 | Pred = Cmp.getFlippedSignednessPredicate(); |
1600 | Pred = Cmp.getSwappedPredicate(pred: Pred); |
1601 | return new ICmpInst(Pred, X, ConstantInt::get(Ty: X->getType(), V: C ^ *XorC)); |
1602 | } |
1603 | } |
1604 | |
1605 | // Mask constant magic can eliminate an 'xor' with unsigned compares. |
1606 | if (Pred == ICmpInst::ICMP_UGT) { |
1607 | // (xor X, ~C) >u C --> X <u ~C (when C+1 is a power of 2) |
1608 | if (*XorC == ~C && (C + 1).isPowerOf2()) |
1609 | return new ICmpInst(ICmpInst::ICMP_ULT, X, Y); |
1610 | // (xor X, C) >u C --> X >u C (when C+1 is a power of 2) |
1611 | if (*XorC == C && (C + 1).isPowerOf2()) |
1612 | return new ICmpInst(ICmpInst::ICMP_UGT, X, Y); |
1613 | } |
1614 | if (Pred == ICmpInst::ICMP_ULT) { |
1615 | // (xor X, -C) <u C --> X >u ~C (when C is a power of 2) |
1616 | if (*XorC == -C && C.isPowerOf2()) |
1617 | return new ICmpInst(ICmpInst::ICMP_UGT, X, |
1618 | ConstantInt::get(Ty: X->getType(), V: ~C)); |
1619 | // (xor X, C) <u C --> X >u ~C (when -C is a power of 2) |
1620 | if (*XorC == C && (-C).isPowerOf2()) |
1621 | return new ICmpInst(ICmpInst::ICMP_UGT, X, |
1622 | ConstantInt::get(Ty: X->getType(), V: ~C)); |
1623 | } |
1624 | return nullptr; |
1625 | } |
1626 | |
1627 | /// For power-of-2 C: |
1628 | /// ((X s>> ShiftC) ^ X) u< C --> (X + C) u< (C << 1) |
1629 | /// ((X s>> ShiftC) ^ X) u> (C - 1) --> (X + C) u> ((C << 1) - 1) |
1630 | Instruction *InstCombinerImpl::foldICmpXorShiftConst(ICmpInst &Cmp, |
1631 | BinaryOperator *Xor, |
1632 | const APInt &C) { |
1633 | CmpInst::Predicate Pred = Cmp.getPredicate(); |
1634 | APInt PowerOf2; |
1635 | if (Pred == ICmpInst::ICMP_ULT) |
1636 | PowerOf2 = C; |
1637 | else if (Pred == ICmpInst::ICMP_UGT && !C.isMaxValue()) |
1638 | PowerOf2 = C + 1; |
1639 | else |
1640 | return nullptr; |
1641 | if (!PowerOf2.isPowerOf2()) |
1642 | return nullptr; |
1643 | Value *X; |
1644 | const APInt *ShiftC; |
1645 | if (!match(V: Xor, P: m_OneUse(SubPattern: m_c_Xor(L: m_Value(V&: X), |
1646 | R: m_AShr(L: m_Deferred(V: X), R: m_APInt(Res&: ShiftC)))))) |
1647 | return nullptr; |
1648 | uint64_t Shift = ShiftC->getLimitedValue(); |
1649 | Type *XType = X->getType(); |
1650 | if (Shift == 0 || PowerOf2.isMinSignedValue()) |
1651 | return nullptr; |
1652 | Value *Add = Builder.CreateAdd(LHS: X, RHS: ConstantInt::get(Ty: XType, V: PowerOf2)); |
1653 | APInt Bound = |
1654 | Pred == ICmpInst::ICMP_ULT ? PowerOf2 << 1 : ((PowerOf2 << 1) - 1); |
1655 | return new ICmpInst(Pred, Add, ConstantInt::get(Ty: XType, V: Bound)); |
1656 | } |
1657 | |
1658 | /// Fold icmp (and (sh X, Y), C2), C1. |
1659 | Instruction *InstCombinerImpl::foldICmpAndShift(ICmpInst &Cmp, |
1660 | BinaryOperator *And, |
1661 | const APInt &C1, |
1662 | const APInt &C2) { |
1663 | BinaryOperator *Shift = dyn_cast<BinaryOperator>(Val: And->getOperand(i_nocapture: 0)); |
1664 | if (!Shift || !Shift->isShift()) |
1665 | return nullptr; |
1666 | |
1667 | // If this is: (X >> C3) & C2 != C1 (where any shift and any compare could |
1668 | // exist), turn it into (X & (C2 << C3)) != (C1 << C3). This happens a LOT in |
1669 | // code produced by the clang front-end, for bitfield access. |
1670 | // This seemingly simple opportunity to fold away a shift turns out to be |
1671 | // rather complicated. See PR17827 for details. |
1672 | unsigned ShiftOpcode = Shift->getOpcode(); |
1673 | bool IsShl = ShiftOpcode == Instruction::Shl; |
1674 | const APInt *C3; |
1675 | if (match(V: Shift->getOperand(i_nocapture: 1), P: m_APInt(Res&: C3))) { |
1676 | APInt NewAndCst, NewCmpCst; |
1677 | bool AnyCmpCstBitsShiftedOut; |
1678 | if (ShiftOpcode == Instruction::Shl) { |
1679 | // For a left shift, we can fold if the comparison is not signed. We can |
1680 | // also fold a signed comparison if the mask value and comparison value |
1681 | // are not negative. These constraints may not be obvious, but we can |
1682 | // prove that they are correct using an SMT solver. |
1683 | if (Cmp.isSigned() && (C2.isNegative() || C1.isNegative())) |
1684 | return nullptr; |
1685 | |
1686 | NewCmpCst = C1.lshr(ShiftAmt: *C3); |
1687 | NewAndCst = C2.lshr(ShiftAmt: *C3); |
1688 | AnyCmpCstBitsShiftedOut = NewCmpCst.shl(ShiftAmt: *C3) != C1; |
1689 | } else if (ShiftOpcode == Instruction::LShr) { |
1690 | // For a logical right shift, we can fold if the comparison is not signed. |
1691 | // We can also fold a signed comparison if the shifted mask value and the |
1692 | // shifted comparison value are not negative. These constraints may not be |
1693 | // obvious, but we can prove that they are correct using an SMT solver. |
1694 | NewCmpCst = C1.shl(ShiftAmt: *C3); |
1695 | NewAndCst = C2.shl(ShiftAmt: *C3); |
1696 | AnyCmpCstBitsShiftedOut = NewCmpCst.lshr(ShiftAmt: *C3) != C1; |
1697 | if (Cmp.isSigned() && (NewAndCst.isNegative() || NewCmpCst.isNegative())) |
1698 | return nullptr; |
1699 | } else { |
1700 | // For an arithmetic shift, check that both constants don't use (in a |
1701 | // signed sense) the top bits being shifted out. |
1702 | assert(ShiftOpcode == Instruction::AShr && "Unknown shift opcode" ); |
1703 | NewCmpCst = C1.shl(ShiftAmt: *C3); |
1704 | NewAndCst = C2.shl(ShiftAmt: *C3); |
1705 | AnyCmpCstBitsShiftedOut = NewCmpCst.ashr(ShiftAmt: *C3) != C1; |
1706 | if (NewAndCst.ashr(ShiftAmt: *C3) != C2) |
1707 | return nullptr; |
1708 | } |
1709 | |
1710 | if (AnyCmpCstBitsShiftedOut) { |
1711 | // If we shifted bits out, the fold is not going to work out. As a |
1712 | // special case, check to see if this means that the result is always |
1713 | // true or false now. |
1714 | if (Cmp.getPredicate() == ICmpInst::ICMP_EQ) |
1715 | return replaceInstUsesWith(I&: Cmp, V: ConstantInt::getFalse(Ty: Cmp.getType())); |
1716 | if (Cmp.getPredicate() == ICmpInst::ICMP_NE) |
1717 | return replaceInstUsesWith(I&: Cmp, V: ConstantInt::getTrue(Ty: Cmp.getType())); |
1718 | } else { |
1719 | Value *NewAnd = Builder.CreateAnd( |
1720 | LHS: Shift->getOperand(i_nocapture: 0), RHS: ConstantInt::get(Ty: And->getType(), V: NewAndCst)); |
1721 | return new ICmpInst(Cmp.getPredicate(), |
1722 | NewAnd, ConstantInt::get(Ty: And->getType(), V: NewCmpCst)); |
1723 | } |
1724 | } |
1725 | |
1726 | // Turn ((X >> Y) & C2) == 0 into (X & (C2 << Y)) == 0. The latter is |
1727 | // preferable because it allows the C2 << Y expression to be hoisted out of a |
1728 | // loop if Y is invariant and X is not. |
1729 | if (Shift->hasOneUse() && C1.isZero() && Cmp.isEquality() && |
1730 | !Shift->isArithmeticShift() && !isa<Constant>(Val: Shift->getOperand(i_nocapture: 0))) { |
1731 | // Compute C2 << Y. |
1732 | Value *NewShift = |
1733 | IsShl ? Builder.CreateLShr(LHS: And->getOperand(i_nocapture: 1), RHS: Shift->getOperand(i_nocapture: 1)) |
1734 | : Builder.CreateShl(LHS: And->getOperand(i_nocapture: 1), RHS: Shift->getOperand(i_nocapture: 1)); |
1735 | |
1736 | // Compute X & (C2 << Y). |
1737 | Value *NewAnd = Builder.CreateAnd(LHS: Shift->getOperand(i_nocapture: 0), RHS: NewShift); |
1738 | return replaceOperand(I&: Cmp, OpNum: 0, V: NewAnd); |
1739 | } |
1740 | |
1741 | return nullptr; |
1742 | } |
1743 | |
1744 | /// Fold icmp (and X, C2), C1. |
1745 | Instruction *InstCombinerImpl::foldICmpAndConstConst(ICmpInst &Cmp, |
1746 | BinaryOperator *And, |
1747 | const APInt &C1) { |
1748 | bool isICMP_NE = Cmp.getPredicate() == ICmpInst::ICMP_NE; |
1749 | |
1750 | // For vectors: icmp ne (and X, 1), 0 --> trunc X to N x i1 |
1751 | // TODO: We canonicalize to the longer form for scalars because we have |
1752 | // better analysis/folds for icmp, and codegen may be better with icmp. |
1753 | if (isICMP_NE && Cmp.getType()->isVectorTy() && C1.isZero() && |
1754 | match(V: And->getOperand(i_nocapture: 1), P: m_One())) |
1755 | return new TruncInst(And->getOperand(i_nocapture: 0), Cmp.getType()); |
1756 | |
1757 | const APInt *C2; |
1758 | Value *X; |
1759 | if (!match(V: And, P: m_And(L: m_Value(V&: X), R: m_APInt(Res&: C2)))) |
1760 | return nullptr; |
1761 | |
1762 | // Don't perform the following transforms if the AND has multiple uses |
1763 | if (!And->hasOneUse()) |
1764 | return nullptr; |
1765 | |
1766 | if (Cmp.isEquality() && C1.isZero()) { |
1767 | // Restrict this fold to single-use 'and' (PR10267). |
1768 | // Replace (and X, (1 << size(X)-1) != 0) with X s< 0 |
1769 | if (C2->isSignMask()) { |
1770 | Constant *Zero = Constant::getNullValue(Ty: X->getType()); |
1771 | auto NewPred = isICMP_NE ? ICmpInst::ICMP_SLT : ICmpInst::ICMP_SGE; |
1772 | return new ICmpInst(NewPred, X, Zero); |
1773 | } |
1774 | |
1775 | APInt NewC2 = *C2; |
1776 | KnownBits Know = computeKnownBits(V: And->getOperand(i_nocapture: 0), Depth: 0, CxtI: And); |
1777 | // Set high zeros of C2 to allow matching negated power-of-2. |
1778 | NewC2 = *C2 | APInt::getHighBitsSet(numBits: C2->getBitWidth(), |
1779 | hiBitsSet: Know.countMinLeadingZeros()); |
1780 | |
1781 | // Restrict this fold only for single-use 'and' (PR10267). |
1782 | // ((%x & C) == 0) --> %x u< (-C) iff (-C) is power of two. |
1783 | if (NewC2.isNegatedPowerOf2()) { |
1784 | Constant *NegBOC = ConstantInt::get(Ty: And->getType(), V: -NewC2); |
1785 | auto NewPred = isICMP_NE ? ICmpInst::ICMP_UGE : ICmpInst::ICMP_ULT; |
1786 | return new ICmpInst(NewPred, X, NegBOC); |
1787 | } |
1788 | } |
1789 | |
1790 | // If the LHS is an 'and' of a truncate and we can widen the and/compare to |
1791 | // the input width without changing the value produced, eliminate the cast: |
1792 | // |
1793 | // icmp (and (trunc W), C2), C1 -> icmp (and W, C2'), C1' |
1794 | // |
1795 | // We can do this transformation if the constants do not have their sign bits |
1796 | // set or if it is an equality comparison. Extending a relational comparison |
1797 | // when we're checking the sign bit would not work. |
1798 | Value *W; |
1799 | if (match(V: And->getOperand(i_nocapture: 0), P: m_OneUse(SubPattern: m_Trunc(Op: m_Value(V&: W)))) && |
1800 | (Cmp.isEquality() || (!C1.isNegative() && !C2->isNegative()))) { |
1801 | // TODO: Is this a good transform for vectors? Wider types may reduce |
1802 | // throughput. Should this transform be limited (even for scalars) by using |
1803 | // shouldChangeType()? |
1804 | if (!Cmp.getType()->isVectorTy()) { |
1805 | Type *WideType = W->getType(); |
1806 | unsigned WideScalarBits = WideType->getScalarSizeInBits(); |
1807 | Constant *ZextC1 = ConstantInt::get(Ty: WideType, V: C1.zext(width: WideScalarBits)); |
1808 | Constant *ZextC2 = ConstantInt::get(Ty: WideType, V: C2->zext(width: WideScalarBits)); |
1809 | Value *NewAnd = Builder.CreateAnd(LHS: W, RHS: ZextC2, Name: And->getName()); |
1810 | return new ICmpInst(Cmp.getPredicate(), NewAnd, ZextC1); |
1811 | } |
1812 | } |
1813 | |
1814 | if (Instruction *I = foldICmpAndShift(Cmp, And, C1, C2: *C2)) |
1815 | return I; |
1816 | |
1817 | // (icmp pred (and (or (lshr A, B), A), 1), 0) --> |
1818 | // (icmp pred (and A, (or (shl 1, B), 1), 0)) |
1819 | // |
1820 | // iff pred isn't signed |
1821 | if (!Cmp.isSigned() && C1.isZero() && And->getOperand(i_nocapture: 0)->hasOneUse() && |
1822 | match(V: And->getOperand(i_nocapture: 1), P: m_One())) { |
1823 | Constant *One = cast<Constant>(Val: And->getOperand(i_nocapture: 1)); |
1824 | Value *Or = And->getOperand(i_nocapture: 0); |
1825 | Value *A, *B, *LShr; |
1826 | if (match(V: Or, P: m_Or(L: m_Value(V&: LShr), R: m_Value(V&: A))) && |
1827 | match(V: LShr, P: m_LShr(L: m_Specific(V: A), R: m_Value(V&: B)))) { |
1828 | unsigned UsesRemoved = 0; |
1829 | if (And->hasOneUse()) |
1830 | ++UsesRemoved; |
1831 | if (Or->hasOneUse()) |
1832 | ++UsesRemoved; |
1833 | if (LShr->hasOneUse()) |
1834 | ++UsesRemoved; |
1835 | |
1836 | // Compute A & ((1 << B) | 1) |
1837 | unsigned RequireUsesRemoved = match(V: B, P: m_ImmConstant()) ? 1 : 3; |
1838 | if (UsesRemoved >= RequireUsesRemoved) { |
1839 | Value *NewOr = |
1840 | Builder.CreateOr(LHS: Builder.CreateShl(LHS: One, RHS: B, Name: LShr->getName(), |
1841 | /*HasNUW=*/true), |
1842 | RHS: One, Name: Or->getName()); |
1843 | Value *NewAnd = Builder.CreateAnd(LHS: A, RHS: NewOr, Name: And->getName()); |
1844 | return replaceOperand(I&: Cmp, OpNum: 0, V: NewAnd); |
1845 | } |
1846 | } |
1847 | } |
1848 | |
1849 | // (icmp eq (and (bitcast X to int), ExponentMask), ExponentMask) --> |
1850 | // llvm.is.fpclass(X, fcInf|fcNan) |
1851 | // (icmp ne (and (bitcast X to int), ExponentMask), ExponentMask) --> |
1852 | // llvm.is.fpclass(X, ~(fcInf|fcNan)) |
1853 | Value *V; |
1854 | if (!Cmp.getParent()->getParent()->hasFnAttribute( |
1855 | Attribute::NoImplicitFloat) && |
1856 | Cmp.isEquality() && |
1857 | match(V: X, P: m_OneUse(SubPattern: m_ElementWiseBitCast(Op: m_Value(V))))) { |
1858 | Type *FPType = V->getType()->getScalarType(); |
1859 | if (FPType->isIEEELikeFPTy() && C1 == *C2) { |
1860 | APInt ExponentMask = |
1861 | APFloat::getInf(Sem: FPType->getFltSemantics()).bitcastToAPInt(); |
1862 | if (C1 == ExponentMask) { |
1863 | unsigned Mask = FPClassTest::fcNan | FPClassTest::fcInf; |
1864 | if (isICMP_NE) |
1865 | Mask = ~Mask & fcAllFlags; |
1866 | return replaceInstUsesWith(I&: Cmp, V: Builder.createIsFPClass(FPNum: V, Test: Mask)); |
1867 | } |
1868 | } |
1869 | } |
1870 | |
1871 | return nullptr; |
1872 | } |
1873 | |
1874 | /// Fold icmp (and X, Y), C. |
1875 | Instruction *InstCombinerImpl::foldICmpAndConstant(ICmpInst &Cmp, |
1876 | BinaryOperator *And, |
1877 | const APInt &C) { |
1878 | if (Instruction *I = foldICmpAndConstConst(Cmp, And, C1: C)) |
1879 | return I; |
1880 | |
1881 | const ICmpInst::Predicate Pred = Cmp.getPredicate(); |
1882 | bool TrueIfNeg; |
1883 | if (isSignBitCheck(Pred, RHS: C, TrueIfSigned&: TrueIfNeg)) { |
1884 | // ((X - 1) & ~X) < 0 --> X == 0 |
1885 | // ((X - 1) & ~X) >= 0 --> X != 0 |
1886 | Value *X; |
1887 | if (match(V: And->getOperand(i_nocapture: 0), P: m_Add(L: m_Value(V&: X), R: m_AllOnes())) && |
1888 | match(V: And->getOperand(i_nocapture: 1), P: m_Not(V: m_Specific(V: X)))) { |
1889 | auto NewPred = TrueIfNeg ? CmpInst::ICMP_EQ : CmpInst::ICMP_NE; |
1890 | return new ICmpInst(NewPred, X, ConstantInt::getNullValue(Ty: X->getType())); |
1891 | } |
1892 | // (X & -X) < 0 --> X == MinSignedC |
1893 | // (X & -X) > -1 --> X != MinSignedC |
1894 | if (match(V: And, P: m_c_And(L: m_Neg(V: m_Value(V&: X)), R: m_Deferred(V: X)))) { |
1895 | Constant *MinSignedC = ConstantInt::get( |
1896 | Ty: X->getType(), |
1897 | V: APInt::getSignedMinValue(numBits: X->getType()->getScalarSizeInBits())); |
1898 | auto NewPred = TrueIfNeg ? CmpInst::ICMP_EQ : CmpInst::ICMP_NE; |
1899 | return new ICmpInst(NewPred, X, MinSignedC); |
1900 | } |
1901 | } |
1902 | |
1903 | // TODO: These all require that Y is constant too, so refactor with the above. |
1904 | |
1905 | // Try to optimize things like "A[i] & 42 == 0" to index computations. |
1906 | Value *X = And->getOperand(i_nocapture: 0); |
1907 | Value *Y = And->getOperand(i_nocapture: 1); |
1908 | if (auto *C2 = dyn_cast<ConstantInt>(Val: Y)) |
1909 | if (auto *LI = dyn_cast<LoadInst>(Val: X)) |
1910 | if (auto *GEP = dyn_cast<GetElementPtrInst>(Val: LI->getOperand(i_nocapture: 0))) |
1911 | if (auto *GV = dyn_cast<GlobalVariable>(Val: GEP->getOperand(i_nocapture: 0))) |
1912 | if (Instruction *Res = |
1913 | foldCmpLoadFromIndexedGlobal(LI, GEP, GV, ICI&: Cmp, AndCst: C2)) |
1914 | return Res; |
1915 | |
1916 | if (!Cmp.isEquality()) |
1917 | return nullptr; |
1918 | |
1919 | // X & -C == -C -> X > u ~C |
1920 | // X & -C != -C -> X <= u ~C |
1921 | // iff C is a power of 2 |
1922 | if (Cmp.getOperand(i_nocapture: 1) == Y && C.isNegatedPowerOf2()) { |
1923 | auto NewPred = |
1924 | Pred == CmpInst::ICMP_EQ ? CmpInst::ICMP_UGT : CmpInst::ICMP_ULE; |
1925 | return new ICmpInst(NewPred, X, SubOne(C: cast<Constant>(Val: Cmp.getOperand(i_nocapture: 1)))); |
1926 | } |
1927 | |
1928 | // If we are testing the intersection of 2 select-of-nonzero-constants with no |
1929 | // common bits set, it's the same as checking if exactly one select condition |
1930 | // is set: |
1931 | // ((A ? TC : FC) & (B ? TC : FC)) == 0 --> xor A, B |
1932 | // ((A ? TC : FC) & (B ? TC : FC)) != 0 --> not(xor A, B) |
1933 | // TODO: Generalize for non-constant values. |
1934 | // TODO: Handle signed/unsigned predicates. |
1935 | // TODO: Handle other bitwise logic connectors. |
1936 | // TODO: Extend to handle a non-zero compare constant. |
1937 | if (C.isZero() && (Pred == CmpInst::ICMP_EQ || And->hasOneUse())) { |
1938 | assert(Cmp.isEquality() && "Not expecting non-equality predicates" ); |
1939 | Value *A, *B; |
1940 | const APInt *TC, *FC; |
1941 | if (match(V: X, P: m_Select(C: m_Value(V&: A), L: m_APInt(Res&: TC), R: m_APInt(Res&: FC))) && |
1942 | match(V: Y, |
1943 | P: m_Select(C: m_Value(V&: B), L: m_SpecificInt(V: *TC), R: m_SpecificInt(V: *FC))) && |
1944 | !TC->isZero() && !FC->isZero() && !TC->intersects(RHS: *FC)) { |
1945 | Value *R = Builder.CreateXor(LHS: A, RHS: B); |
1946 | if (Pred == CmpInst::ICMP_NE) |
1947 | R = Builder.CreateNot(V: R); |
1948 | return replaceInstUsesWith(I&: Cmp, V: R); |
1949 | } |
1950 | } |
1951 | |
1952 | // ((zext i1 X) & Y) == 0 --> !((trunc Y) & X) |
1953 | // ((zext i1 X) & Y) != 0 --> ((trunc Y) & X) |
1954 | // ((zext i1 X) & Y) == 1 --> ((trunc Y) & X) |
1955 | // ((zext i1 X) & Y) != 1 --> !((trunc Y) & X) |
1956 | if (match(V: And, P: m_OneUse(SubPattern: m_c_And(L: m_OneUse(SubPattern: m_ZExt(Op: m_Value(V&: X))), R: m_Value(V&: Y)))) && |
1957 | X->getType()->isIntOrIntVectorTy(BitWidth: 1) && (C.isZero() || C.isOne())) { |
1958 | Value *TruncY = Builder.CreateTrunc(V: Y, DestTy: X->getType()); |
1959 | if (C.isZero() ^ (Pred == CmpInst::ICMP_NE)) { |
1960 | Value *And = Builder.CreateAnd(LHS: TruncY, RHS: X); |
1961 | return BinaryOperator::CreateNot(Op: And); |
1962 | } |
1963 | return BinaryOperator::CreateAnd(V1: TruncY, V2: X); |
1964 | } |
1965 | |
1966 | // (icmp eq/ne (and (shl -1, X), Y), 0) |
1967 | // -> (icmp eq/ne (lshr Y, X), 0) |
1968 | // We could technically handle any C == 0 or (C < 0 && isOdd(C)) but it seems |
1969 | // highly unlikely the non-zero case will ever show up in code. |
1970 | if (C.isZero() && |
1971 | match(V: And, P: m_OneUse(SubPattern: m_c_And(L: m_OneUse(SubPattern: m_Shl(L: m_AllOnes(), R: m_Value(V&: X))), |
1972 | R: m_Value(V&: Y))))) { |
1973 | Value *LShr = Builder.CreateLShr(LHS: Y, RHS: X); |
1974 | return new ICmpInst(Pred, LShr, Constant::getNullValue(Ty: LShr->getType())); |
1975 | } |
1976 | |
1977 | return nullptr; |
1978 | } |
1979 | |
1980 | /// Fold icmp eq/ne (or (xor/sub (X1, X2), xor/sub (X3, X4))), 0. |
1981 | static Value *foldICmpOrXorSubChain(ICmpInst &Cmp, BinaryOperator *Or, |
1982 | InstCombiner::BuilderTy &Builder) { |
1983 | // Are we using xors or subs to bitwise check for a pair or pairs of |
1984 | // (in)equalities? Convert to a shorter form that has more potential to be |
1985 | // folded even further. |
1986 | // ((X1 ^/- X2) || (X3 ^/- X4)) == 0 --> (X1 == X2) && (X3 == X4) |
1987 | // ((X1 ^/- X2) || (X3 ^/- X4)) != 0 --> (X1 != X2) || (X3 != X4) |
1988 | // ((X1 ^/- X2) || (X3 ^/- X4) || (X5 ^/- X6)) == 0 --> |
1989 | // (X1 == X2) && (X3 == X4) && (X5 == X6) |
1990 | // ((X1 ^/- X2) || (X3 ^/- X4) || (X5 ^/- X6)) != 0 --> |
1991 | // (X1 != X2) || (X3 != X4) || (X5 != X6) |
1992 | SmallVector<std::pair<Value *, Value *>, 2> CmpValues; |
1993 | SmallVector<Value *, 16> WorkList(1, Or); |
1994 | |
1995 | while (!WorkList.empty()) { |
1996 | auto MatchOrOperatorArgument = [&](Value *OrOperatorArgument) { |
1997 | Value *Lhs, *Rhs; |
1998 | |
1999 | if (match(V: OrOperatorArgument, |
2000 | P: m_OneUse(SubPattern: m_Xor(L: m_Value(V&: Lhs), R: m_Value(V&: Rhs))))) { |
2001 | CmpValues.emplace_back(Args&: Lhs, Args&: Rhs); |
2002 | return; |
2003 | } |
2004 | |
2005 | if (match(V: OrOperatorArgument, |
2006 | P: m_OneUse(SubPattern: m_Sub(L: m_Value(V&: Lhs), R: m_Value(V&: Rhs))))) { |
2007 | CmpValues.emplace_back(Args&: Lhs, Args&: Rhs); |
2008 | return; |
2009 | } |
2010 | |
2011 | WorkList.push_back(Elt: OrOperatorArgument); |
2012 | }; |
2013 | |
2014 | Value *CurrentValue = WorkList.pop_back_val(); |
2015 | Value *OrOperatorLhs, *OrOperatorRhs; |
2016 | |
2017 | if (!match(V: CurrentValue, |
2018 | P: m_Or(L: m_Value(V&: OrOperatorLhs), R: m_Value(V&: OrOperatorRhs)))) { |
2019 | return nullptr; |
2020 | } |
2021 | |
2022 | MatchOrOperatorArgument(OrOperatorRhs); |
2023 | MatchOrOperatorArgument(OrOperatorLhs); |
2024 | } |
2025 | |
2026 | ICmpInst::Predicate Pred = Cmp.getPredicate(); |
2027 | auto BOpc = Pred == CmpInst::ICMP_EQ ? Instruction::And : Instruction::Or; |
2028 | Value *LhsCmp = Builder.CreateICmp(P: Pred, LHS: CmpValues.rbegin()->first, |
2029 | RHS: CmpValues.rbegin()->second); |
2030 | |
2031 | for (auto It = CmpValues.rbegin() + 1; It != CmpValues.rend(); ++It) { |
2032 | Value *RhsCmp = Builder.CreateICmp(P: Pred, LHS: It->first, RHS: It->second); |
2033 | LhsCmp = Builder.CreateBinOp(Opc: BOpc, LHS: LhsCmp, RHS: RhsCmp); |
2034 | } |
2035 | |
2036 | return LhsCmp; |
2037 | } |
2038 | |
2039 | /// Fold icmp (or X, Y), C. |
2040 | Instruction *InstCombinerImpl::foldICmpOrConstant(ICmpInst &Cmp, |
2041 | BinaryOperator *Or, |
2042 | const APInt &C) { |
2043 | ICmpInst::Predicate Pred = Cmp.getPredicate(); |
2044 | if (C.isOne()) { |
2045 | // icmp slt signum(V) 1 --> icmp slt V, 1 |
2046 | Value *V = nullptr; |
2047 | if (Pred == ICmpInst::ICMP_SLT && match(V: Or, P: m_Signum(V: m_Value(V)))) |
2048 | return new ICmpInst(ICmpInst::ICMP_SLT, V, |
2049 | ConstantInt::get(Ty: V->getType(), V: 1)); |
2050 | } |
2051 | |
2052 | Value *OrOp0 = Or->getOperand(i_nocapture: 0), *OrOp1 = Or->getOperand(i_nocapture: 1); |
2053 | |
2054 | // (icmp eq/ne (or disjoint x, C0), C1) |
2055 | // -> (icmp eq/ne x, C0^C1) |
2056 | if (Cmp.isEquality() && match(V: OrOp1, P: m_ImmConstant()) && |
2057 | cast<PossiblyDisjointInst>(Val: Or)->isDisjoint()) { |
2058 | Value *NewC = |
2059 | Builder.CreateXor(LHS: OrOp1, RHS: ConstantInt::get(Ty: OrOp1->getType(), V: C)); |
2060 | return new ICmpInst(Pred, OrOp0, NewC); |
2061 | } |
2062 | |
2063 | const APInt *MaskC; |
2064 | if (match(V: OrOp1, P: m_APInt(Res&: MaskC)) && Cmp.isEquality()) { |
2065 | if (*MaskC == C && (C + 1).isPowerOf2()) { |
2066 | // X | C == C --> X <=u C |
2067 | // X | C != C --> X >u C |
2068 | // iff C+1 is a power of 2 (C is a bitmask of the low bits) |
2069 | Pred = (Pred == CmpInst::ICMP_EQ) ? CmpInst::ICMP_ULE : CmpInst::ICMP_UGT; |
2070 | return new ICmpInst(Pred, OrOp0, OrOp1); |
2071 | } |
2072 | |
2073 | // More general: canonicalize 'equality with set bits mask' to |
2074 | // 'equality with clear bits mask'. |
2075 | // (X | MaskC) == C --> (X & ~MaskC) == C ^ MaskC |
2076 | // (X | MaskC) != C --> (X & ~MaskC) != C ^ MaskC |
2077 | if (Or->hasOneUse()) { |
2078 | Value *And = Builder.CreateAnd(LHS: OrOp0, RHS: ~(*MaskC)); |
2079 | Constant *NewC = ConstantInt::get(Ty: Or->getType(), V: C ^ (*MaskC)); |
2080 | return new ICmpInst(Pred, And, NewC); |
2081 | } |
2082 | } |
2083 | |
2084 | // (X | (X-1)) s< 0 --> X s< 1 |
2085 | // (X | (X-1)) s> -1 --> X s> 0 |
2086 | Value *X; |
2087 | bool TrueIfSigned; |
2088 | if (isSignBitCheck(Pred, RHS: C, TrueIfSigned) && |
2089 | match(V: Or, P: m_c_Or(L: m_Add(L: m_Value(V&: X), R: m_AllOnes()), R: m_Deferred(V: X)))) { |
2090 | auto NewPred = TrueIfSigned ? ICmpInst::ICMP_SLT : ICmpInst::ICMP_SGT; |
2091 | Constant *NewC = ConstantInt::get(Ty: X->getType(), V: TrueIfSigned ? 1 : 0); |
2092 | return new ICmpInst(NewPred, X, NewC); |
2093 | } |
2094 | |
2095 | const APInt *OrC; |
2096 | // icmp(X | OrC, C) --> icmp(X, 0) |
2097 | if (C.isNonNegative() && match(V: Or, P: m_Or(L: m_Value(V&: X), R: m_APInt(Res&: OrC)))) { |
2098 | switch (Pred) { |
2099 | // X | OrC s< C --> X s< 0 iff OrC s>= C s>= 0 |
2100 | case ICmpInst::ICMP_SLT: |
2101 | // X | OrC s>= C --> X s>= 0 iff OrC s>= C s>= 0 |
2102 | case ICmpInst::ICMP_SGE: |
2103 | if (OrC->sge(RHS: C)) |
2104 | return new ICmpInst(Pred, X, ConstantInt::getNullValue(Ty: X->getType())); |
2105 | break; |
2106 | // X | OrC s<= C --> X s< 0 iff OrC s> C s>= 0 |
2107 | case ICmpInst::ICMP_SLE: |
2108 | // X | OrC s> C --> X s>= 0 iff OrC s> C s>= 0 |
2109 | case ICmpInst::ICMP_SGT: |
2110 | if (OrC->sgt(RHS: C)) |
2111 | return new ICmpInst(ICmpInst::getFlippedStrictnessPredicate(pred: Pred), X, |
2112 | ConstantInt::getNullValue(Ty: X->getType())); |
2113 | break; |
2114 | default: |
2115 | break; |
2116 | } |
2117 | } |
2118 | |
2119 | if (!Cmp.isEquality() || !C.isZero() || !Or->hasOneUse()) |
2120 | return nullptr; |
2121 | |
2122 | Value *P, *Q; |
2123 | if (match(V: Or, P: m_Or(L: m_PtrToInt(Op: m_Value(V&: P)), R: m_PtrToInt(Op: m_Value(V&: Q))))) { |
2124 | // Simplify icmp eq (or (ptrtoint P), (ptrtoint Q)), 0 |
2125 | // -> and (icmp eq P, null), (icmp eq Q, null). |
2126 | Value *CmpP = |
2127 | Builder.CreateICmp(P: Pred, LHS: P, RHS: ConstantInt::getNullValue(Ty: P->getType())); |
2128 | Value *CmpQ = |
2129 | Builder.CreateICmp(P: Pred, LHS: Q, RHS: ConstantInt::getNullValue(Ty: Q->getType())); |
2130 | auto BOpc = Pred == CmpInst::ICMP_EQ ? Instruction::And : Instruction::Or; |
2131 | return BinaryOperator::Create(Op: BOpc, S1: CmpP, S2: CmpQ); |
2132 | } |
2133 | |
2134 | if (Value *V = foldICmpOrXorSubChain(Cmp, Or, Builder)) |
2135 | return replaceInstUsesWith(I&: Cmp, V); |
2136 | |
2137 | return nullptr; |
2138 | } |
2139 | |
2140 | /// Fold icmp (mul X, Y), C. |
2141 | Instruction *InstCombinerImpl::foldICmpMulConstant(ICmpInst &Cmp, |
2142 | BinaryOperator *Mul, |
2143 | const APInt &C) { |
2144 | ICmpInst::Predicate Pred = Cmp.getPredicate(); |
2145 | Type *MulTy = Mul->getType(); |
2146 | Value *X = Mul->getOperand(i_nocapture: 0); |
2147 | |
2148 | // If there's no overflow: |
2149 | // X * X == 0 --> X == 0 |
2150 | // X * X != 0 --> X != 0 |
2151 | if (Cmp.isEquality() && C.isZero() && X == Mul->getOperand(i_nocapture: 1) && |
2152 | (Mul->hasNoUnsignedWrap() || Mul->hasNoSignedWrap())) |
2153 | return new ICmpInst(Pred, X, ConstantInt::getNullValue(Ty: MulTy)); |
2154 | |
2155 | const APInt *MulC; |
2156 | if (!match(V: Mul->getOperand(i_nocapture: 1), P: m_APInt(Res&: MulC))) |
2157 | return nullptr; |
2158 | |
2159 | // If this is a test of the sign bit and the multiply is sign-preserving with |
2160 | // a constant operand, use the multiply LHS operand instead: |
2161 | // (X * +MulC) < 0 --> X < 0 |
2162 | // (X * -MulC) < 0 --> X > 0 |
2163 | if (isSignTest(Pred, C) && Mul->hasNoSignedWrap()) { |
2164 | if (MulC->isNegative()) |
2165 | Pred = ICmpInst::getSwappedPredicate(pred: Pred); |
2166 | return new ICmpInst(Pred, X, ConstantInt::getNullValue(Ty: MulTy)); |
2167 | } |
2168 | |
2169 | if (MulC->isZero()) |
2170 | return nullptr; |
2171 | |
2172 | // If the multiply does not wrap or the constant is odd, try to divide the |
2173 | // compare constant by the multiplication factor. |
2174 | if (Cmp.isEquality()) { |
2175 | // (mul nsw X, MulC) eq/ne C --> X eq/ne C /s MulC |
2176 | if (Mul->hasNoSignedWrap() && C.srem(RHS: *MulC).isZero()) { |
2177 | Constant *NewC = ConstantInt::get(Ty: MulTy, V: C.sdiv(RHS: *MulC)); |
2178 | return new ICmpInst(Pred, X, NewC); |
2179 | } |
2180 | |
2181 | // C % MulC == 0 is weaker than we could use if MulC is odd because it |
2182 | // correct to transform if MulC * N == C including overflow. I.e with i8 |
2183 | // (icmp eq (mul X, 5), 101) -> (icmp eq X, 225) but since 101 % 5 != 0, we |
2184 | // miss that case. |
2185 | if (C.urem(RHS: *MulC).isZero()) { |
2186 | // (mul nuw X, MulC) eq/ne C --> X eq/ne C /u MulC |
2187 | // (mul X, OddC) eq/ne N * C --> X eq/ne N |
2188 | if ((*MulC & 1).isOne() || Mul->hasNoUnsignedWrap()) { |
2189 | Constant *NewC = ConstantInt::get(Ty: MulTy, V: C.udiv(RHS: *MulC)); |
2190 | return new ICmpInst(Pred, X, NewC); |
2191 | } |
2192 | } |
2193 | } |
2194 | |
2195 | // With a matching no-overflow guarantee, fold the constants: |
2196 | // (X * MulC) < C --> X < (C / MulC) |
2197 | // (X * MulC) > C --> X > (C / MulC) |
2198 | // TODO: Assert that Pred is not equal to SGE, SLE, UGE, ULE? |
2199 | Constant *NewC = nullptr; |
2200 | if (Mul->hasNoSignedWrap() && ICmpInst::isSigned(predicate: Pred)) { |
2201 | // MININT / -1 --> overflow. |
2202 | if (C.isMinSignedValue() && MulC->isAllOnes()) |
2203 | return nullptr; |
2204 | if (MulC->isNegative()) |
2205 | Pred = ICmpInst::getSwappedPredicate(pred: Pred); |
2206 | |
2207 | if (Pred == ICmpInst::ICMP_SLT || Pred == ICmpInst::ICMP_SGE) { |
2208 | NewC = ConstantInt::get( |
2209 | Ty: MulTy, V: APIntOps::RoundingSDiv(A: C, B: *MulC, RM: APInt::Rounding::UP)); |
2210 | } else { |
2211 | assert((Pred == ICmpInst::ICMP_SLE || Pred == ICmpInst::ICMP_SGT) && |
2212 | "Unexpected predicate" ); |
2213 | NewC = ConstantInt::get( |
2214 | Ty: MulTy, V: APIntOps::RoundingSDiv(A: C, B: *MulC, RM: APInt::Rounding::DOWN)); |
2215 | } |
2216 | } else if (Mul->hasNoUnsignedWrap() && ICmpInst::isUnsigned(predicate: Pred)) { |
2217 | if (Pred == ICmpInst::ICMP_ULT || Pred == ICmpInst::ICMP_UGE) { |
2218 | NewC = ConstantInt::get( |
2219 | Ty: MulTy, V: APIntOps::RoundingUDiv(A: C, B: *MulC, RM: APInt::Rounding::UP)); |
2220 | } else { |
2221 | assert((Pred == ICmpInst::ICMP_ULE || Pred == ICmpInst::ICMP_UGT) && |
2222 | "Unexpected predicate" ); |
2223 | NewC = ConstantInt::get( |
2224 | Ty: MulTy, V: APIntOps::RoundingUDiv(A: C, B: *MulC, RM: APInt::Rounding::DOWN)); |
2225 | } |
2226 | } |
2227 | |
2228 | return NewC ? new ICmpInst(Pred, X, NewC) : nullptr; |
2229 | } |
2230 | |
2231 | /// Fold icmp (shl 1, Y), C. |
2232 | static Instruction *foldICmpShlOne(ICmpInst &Cmp, Instruction *Shl, |
2233 | const APInt &C) { |
2234 | Value *Y; |
2235 | if (!match(V: Shl, P: m_Shl(L: m_One(), R: m_Value(V&: Y)))) |
2236 | return nullptr; |
2237 | |
2238 | Type *ShiftType = Shl->getType(); |
2239 | unsigned TypeBits = C.getBitWidth(); |
2240 | bool CIsPowerOf2 = C.isPowerOf2(); |
2241 | ICmpInst::Predicate Pred = Cmp.getPredicate(); |
2242 | if (Cmp.isUnsigned()) { |
2243 | // (1 << Y) pred C -> Y pred Log2(C) |
2244 | if (!CIsPowerOf2) { |
2245 | // (1 << Y) < 30 -> Y <= 4 |
2246 | // (1 << Y) <= 30 -> Y <= 4 |
2247 | // (1 << Y) >= 30 -> Y > 4 |
2248 | // (1 << Y) > 30 -> Y > 4 |
2249 | if (Pred == ICmpInst::ICMP_ULT) |
2250 | Pred = ICmpInst::ICMP_ULE; |
2251 | else if (Pred == ICmpInst::ICMP_UGE) |
2252 | Pred = ICmpInst::ICMP_UGT; |
2253 | } |
2254 | |
2255 | unsigned CLog2 = C.logBase2(); |
2256 | return new ICmpInst(Pred, Y, ConstantInt::get(Ty: ShiftType, V: CLog2)); |
2257 | } else if (Cmp.isSigned()) { |
2258 | Constant *BitWidthMinusOne = ConstantInt::get(Ty: ShiftType, V: TypeBits - 1); |
2259 | // (1 << Y) > 0 -> Y != 31 |
2260 | // (1 << Y) > C -> Y != 31 if C is negative. |
2261 | if (Pred == ICmpInst::ICMP_SGT && C.sle(RHS: 0)) |
2262 | return new ICmpInst(ICmpInst::ICMP_NE, Y, BitWidthMinusOne); |
2263 | |
2264 | // (1 << Y) < 0 -> Y == 31 |
2265 | // (1 << Y) < 1 -> Y == 31 |
2266 | // (1 << Y) < C -> Y == 31 if C is negative and not signed min. |
2267 | // Exclude signed min by subtracting 1 and lower the upper bound to 0. |
2268 | if (Pred == ICmpInst::ICMP_SLT && (C-1).sle(RHS: 0)) |
2269 | return new ICmpInst(ICmpInst::ICMP_EQ, Y, BitWidthMinusOne); |
2270 | } |
2271 | |
2272 | return nullptr; |
2273 | } |
2274 | |
2275 | /// Fold icmp (shl X, Y), C. |
2276 | Instruction *InstCombinerImpl::foldICmpShlConstant(ICmpInst &Cmp, |
2277 | BinaryOperator *Shl, |
2278 | const APInt &C) { |
2279 | const APInt *ShiftVal; |
2280 | if (Cmp.isEquality() && match(V: Shl->getOperand(i_nocapture: 0), P: m_APInt(Res&: ShiftVal))) |
2281 | return foldICmpShlConstConst(I&: Cmp, A: Shl->getOperand(i_nocapture: 1), AP1: C, AP2: *ShiftVal); |
2282 | |
2283 | ICmpInst::Predicate Pred = Cmp.getPredicate(); |
2284 | // (icmp pred (shl nuw&nsw X, Y), Csle0) |
2285 | // -> (icmp pred X, Csle0) |
2286 | // |
2287 | // The idea is the nuw/nsw essentially freeze the sign bit for the shift op |
2288 | // so X's must be what is used. |
2289 | if (C.sle(RHS: 0) && Shl->hasNoUnsignedWrap() && Shl->hasNoSignedWrap()) |
2290 | return new ICmpInst(Pred, Shl->getOperand(i_nocapture: 0), Cmp.getOperand(i_nocapture: 1)); |
2291 | |
2292 | // (icmp eq/ne (shl nuw|nsw X, Y), 0) |
2293 | // -> (icmp eq/ne X, 0) |
2294 | if (ICmpInst::isEquality(P: Pred) && C.isZero() && |
2295 | (Shl->hasNoUnsignedWrap() || Shl->hasNoSignedWrap())) |
2296 | return new ICmpInst(Pred, Shl->getOperand(i_nocapture: 0), Cmp.getOperand(i_nocapture: 1)); |
2297 | |
2298 | // (icmp slt (shl nsw X, Y), 0/1) |
2299 | // -> (icmp slt X, 0/1) |
2300 | // (icmp sgt (shl nsw X, Y), 0/-1) |
2301 | // -> (icmp sgt X, 0/-1) |
2302 | // |
2303 | // NB: sge/sle with a constant will canonicalize to sgt/slt. |
2304 | if (Shl->hasNoSignedWrap() && |
2305 | (Pred == ICmpInst::ICMP_SGT || Pred == ICmpInst::ICMP_SLT)) |
2306 | if (C.isZero() || (Pred == ICmpInst::ICMP_SGT ? C.isAllOnes() : C.isOne())) |
2307 | return new ICmpInst(Pred, Shl->getOperand(i_nocapture: 0), Cmp.getOperand(i_nocapture: 1)); |
2308 | |
2309 | const APInt *ShiftAmt; |
2310 | if (!match(V: Shl->getOperand(i_nocapture: 1), P: m_APInt(Res&: ShiftAmt))) |
2311 | return foldICmpShlOne(Cmp, Shl, C); |
2312 | |
2313 | // Check that the shift amount is in range. If not, don't perform undefined |
2314 | // shifts. When the shift is visited, it will be simplified. |
2315 | unsigned TypeBits = C.getBitWidth(); |
2316 | if (ShiftAmt->uge(RHS: TypeBits)) |
2317 | return nullptr; |
2318 | |
2319 | Value *X = Shl->getOperand(i_nocapture: 0); |
2320 | Type *ShType = Shl->getType(); |
2321 | |
2322 | // NSW guarantees that we are only shifting out sign bits from the high bits, |
2323 | // so we can ASHR the compare constant without needing a mask and eliminate |
2324 | // the shift. |
2325 | if (Shl->hasNoSignedWrap()) { |
2326 | if (Pred == ICmpInst::ICMP_SGT) { |
2327 | // icmp Pred (shl nsw X, ShiftAmt), C --> icmp Pred X, (C >>s ShiftAmt) |
2328 | APInt ShiftedC = C.ashr(ShiftAmt: *ShiftAmt); |
2329 | return new ICmpInst(Pred, X, ConstantInt::get(Ty: ShType, V: ShiftedC)); |
2330 | } |
2331 | if ((Pred == ICmpInst::ICMP_EQ || Pred == ICmpInst::ICMP_NE) && |
2332 | C.ashr(ShiftAmt: *ShiftAmt).shl(ShiftAmt: *ShiftAmt) == C) { |
2333 | APInt ShiftedC = C.ashr(ShiftAmt: *ShiftAmt); |
2334 | return new ICmpInst(Pred, X, ConstantInt::get(Ty: ShType, V: ShiftedC)); |
2335 | } |
2336 | if (Pred == ICmpInst::ICMP_SLT) { |
2337 | // SLE is the same as above, but SLE is canonicalized to SLT, so convert: |
2338 | // (X << S) <=s C is equiv to X <=s (C >> S) for all C |
2339 | // (X << S) <s (C + 1) is equiv to X <s (C >> S) + 1 if C <s SMAX |
2340 | // (X << S) <s C is equiv to X <s ((C - 1) >> S) + 1 if C >s SMIN |
2341 | assert(!C.isMinSignedValue() && "Unexpected icmp slt" ); |
2342 | APInt ShiftedC = (C - 1).ashr(ShiftAmt: *ShiftAmt) + 1; |
2343 | return new ICmpInst(Pred, X, ConstantInt::get(Ty: ShType, V: ShiftedC)); |
2344 | } |
2345 | } |
2346 | |
2347 | // NUW guarantees that we are only shifting out zero bits from the high bits, |
2348 | // so we can LSHR the compare constant without needing a mask and eliminate |
2349 | // the shift. |
2350 | if (Shl->hasNoUnsignedWrap()) { |
2351 | if (Pred == ICmpInst::ICMP_UGT) { |
2352 | // icmp Pred (shl nuw X, ShiftAmt), C --> icmp Pred X, (C >>u ShiftAmt) |
2353 | APInt ShiftedC = C.lshr(ShiftAmt: *ShiftAmt); |
2354 | return new ICmpInst(Pred, X, ConstantInt::get(Ty: ShType, V: ShiftedC)); |
2355 | } |
2356 | if ((Pred == ICmpInst::ICMP_EQ || Pred == ICmpInst::ICMP_NE) && |
2357 | C.lshr(ShiftAmt: *ShiftAmt).shl(ShiftAmt: *ShiftAmt) == C) { |
2358 | APInt ShiftedC = C.lshr(ShiftAmt: *ShiftAmt); |
2359 | return new ICmpInst(Pred, X, ConstantInt::get(Ty: ShType, V: ShiftedC)); |
2360 | } |
2361 | if (Pred == ICmpInst::ICMP_ULT) { |
2362 | // ULE is the same as above, but ULE is canonicalized to ULT, so convert: |
2363 | // (X << S) <=u C is equiv to X <=u (C >> S) for all C |
2364 | // (X << S) <u (C + 1) is equiv to X <u (C >> S) + 1 if C <u ~0u |
2365 | // (X << S) <u C is equiv to X <u ((C - 1) >> S) + 1 if C >u 0 |
2366 | assert(C.ugt(0) && "ult 0 should have been eliminated" ); |
2367 | APInt ShiftedC = (C - 1).lshr(ShiftAmt: *ShiftAmt) + 1; |
2368 | return new ICmpInst(Pred, X, ConstantInt::get(Ty: ShType, V: ShiftedC)); |
2369 | } |
2370 | } |
2371 | |
2372 | if (Cmp.isEquality() && Shl->hasOneUse()) { |
2373 | // Strength-reduce the shift into an 'and'. |
2374 | Constant *Mask = ConstantInt::get( |
2375 | Ty: ShType, |
2376 | V: APInt::getLowBitsSet(numBits: TypeBits, loBitsSet: TypeBits - ShiftAmt->getZExtValue())); |
2377 | Value *And = Builder.CreateAnd(LHS: X, RHS: Mask, Name: Shl->getName() + ".mask" ); |
2378 | Constant *LShrC = ConstantInt::get(Ty: ShType, V: C.lshr(ShiftAmt: *ShiftAmt)); |
2379 | return new ICmpInst(Pred, And, LShrC); |
2380 | } |
2381 | |
2382 | // Otherwise, if this is a comparison of the sign bit, simplify to and/test. |
2383 | bool TrueIfSigned = false; |
2384 | if (Shl->hasOneUse() && isSignBitCheck(Pred, RHS: C, TrueIfSigned)) { |
2385 | // (X << 31) <s 0 --> (X & 1) != 0 |
2386 | Constant *Mask = ConstantInt::get( |
2387 | Ty: ShType, |
2388 | V: APInt::getOneBitSet(numBits: TypeBits, BitNo: TypeBits - ShiftAmt->getZExtValue() - 1)); |
2389 | Value *And = Builder.CreateAnd(LHS: X, RHS: Mask, Name: Shl->getName() + ".mask" ); |
2390 | return new ICmpInst(TrueIfSigned ? ICmpInst::ICMP_NE : ICmpInst::ICMP_EQ, |
2391 | And, Constant::getNullValue(Ty: ShType)); |
2392 | } |
2393 | |
2394 | // Simplify 'shl' inequality test into 'and' equality test. |
2395 | if (Cmp.isUnsigned() && Shl->hasOneUse()) { |
2396 | // (X l<< C2) u<=/u> C1 iff C1+1 is power of two -> X & (~C1 l>> C2) ==/!= 0 |
2397 | if ((C + 1).isPowerOf2() && |
2398 | (Pred == ICmpInst::ICMP_ULE || Pred == ICmpInst::ICMP_UGT)) { |
2399 | Value *And = Builder.CreateAnd(LHS: X, RHS: (~C).lshr(shiftAmt: ShiftAmt->getZExtValue())); |
2400 | return new ICmpInst(Pred == ICmpInst::ICMP_ULE ? ICmpInst::ICMP_EQ |
2401 | : ICmpInst::ICMP_NE, |
2402 | And, Constant::getNullValue(Ty: ShType)); |
2403 | } |
2404 | // (X l<< C2) u</u>= C1 iff C1 is power of two -> X & (-C1 l>> C2) ==/!= 0 |
2405 | if (C.isPowerOf2() && |
2406 | (Pred == ICmpInst::ICMP_ULT || Pred == ICmpInst::ICMP_UGE)) { |
2407 | Value *And = |
2408 | Builder.CreateAnd(LHS: X, RHS: (~(C - 1)).lshr(shiftAmt: ShiftAmt->getZExtValue())); |
2409 | return new ICmpInst(Pred == ICmpInst::ICMP_ULT ? ICmpInst::ICMP_EQ |
2410 | : ICmpInst::ICMP_NE, |
2411 | And, Constant::getNullValue(Ty: ShType)); |
2412 | } |
2413 | } |
2414 | |
2415 | // Transform (icmp pred iM (shl iM %v, N), C) |
2416 | // -> (icmp pred i(M-N) (trunc %v iM to i(M-N)), (trunc (C>>N)) |
2417 | // Transform the shl to a trunc if (trunc (C>>N)) has no loss and M-N. |
2418 | // This enables us to get rid of the shift in favor of a trunc that may be |
2419 | // free on the target. It has the additional benefit of comparing to a |
2420 | // smaller constant that may be more target-friendly. |
2421 | unsigned Amt = ShiftAmt->getLimitedValue(Limit: TypeBits - 1); |
2422 | if (Shl->hasOneUse() && Amt != 0 && C.countr_zero() >= Amt && |
2423 | DL.isLegalInteger(Width: TypeBits - Amt)) { |
2424 | Type *TruncTy = IntegerType::get(C&: Cmp.getContext(), NumBits: TypeBits - Amt); |
2425 | if (auto *ShVTy = dyn_cast<VectorType>(Val: ShType)) |
2426 | TruncTy = VectorType::get(ElementType: TruncTy, EC: ShVTy->getElementCount()); |
2427 | Constant *NewC = |
2428 | ConstantInt::get(Ty: TruncTy, V: C.ashr(ShiftAmt: *ShiftAmt).trunc(width: TypeBits - Amt)); |
2429 | return new ICmpInst(Pred, Builder.CreateTrunc(V: X, DestTy: TruncTy), NewC); |
2430 | } |
2431 | |
2432 | return nullptr; |
2433 | } |
2434 | |
2435 | /// Fold icmp ({al}shr X, Y), C. |
2436 | Instruction *InstCombinerImpl::foldICmpShrConstant(ICmpInst &Cmp, |
2437 | BinaryOperator *Shr, |
2438 | const APInt &C) { |
2439 | // An exact shr only shifts out zero bits, so: |
2440 | // icmp eq/ne (shr X, Y), 0 --> icmp eq/ne X, 0 |
2441 | Value *X = Shr->getOperand(i_nocapture: 0); |
2442 | CmpInst::Predicate Pred = Cmp.getPredicate(); |
2443 | if (Cmp.isEquality() && Shr->isExact() && C.isZero()) |
2444 | return new ICmpInst(Pred, X, Cmp.getOperand(i_nocapture: 1)); |
2445 | |
2446 | bool IsAShr = Shr->getOpcode() == Instruction::AShr; |
2447 | const APInt *ShiftValC; |
2448 | if (match(V: X, P: m_APInt(Res&: ShiftValC))) { |
2449 | if (Cmp.isEquality()) |
2450 | return foldICmpShrConstConst(I&: Cmp, A: Shr->getOperand(i_nocapture: 1), AP1: C, AP2: *ShiftValC); |
2451 | |
2452 | // (ShiftValC >> Y) >s -1 --> Y != 0 with ShiftValC < 0 |
2453 | // (ShiftValC >> Y) <s 0 --> Y == 0 with ShiftValC < 0 |
2454 | bool TrueIfSigned; |
2455 | if (!IsAShr && ShiftValC->isNegative() && |
2456 | isSignBitCheck(Pred, RHS: C, TrueIfSigned)) |
2457 | return new ICmpInst(TrueIfSigned ? CmpInst::ICMP_EQ : CmpInst::ICMP_NE, |
2458 | Shr->getOperand(i_nocapture: 1), |
2459 | ConstantInt::getNullValue(Ty: X->getType())); |
2460 | |
2461 | // If the shifted constant is a power-of-2, test the shift amount directly: |
2462 | // (ShiftValC >> Y) >u C --> X <u (LZ(C) - LZ(ShiftValC)) |
2463 | // (ShiftValC >> Y) <u C --> X >=u (LZ(C-1) - LZ(ShiftValC)) |
2464 | if (!IsAShr && ShiftValC->isPowerOf2() && |
2465 | (Pred == CmpInst::ICMP_UGT || Pred == CmpInst::ICMP_ULT)) { |
2466 | bool IsUGT = Pred == CmpInst::ICMP_UGT; |
2467 | assert(ShiftValC->uge(C) && "Expected simplify of compare" ); |
2468 | assert((IsUGT || !C.isZero()) && "Expected X u< 0 to simplify" ); |
2469 | |
2470 | unsigned CmpLZ = IsUGT ? C.countl_zero() : (C - 1).countl_zero(); |
2471 | unsigned ShiftLZ = ShiftValC->countl_zero(); |
2472 | Constant *NewC = ConstantInt::get(Ty: Shr->getType(), V: CmpLZ - ShiftLZ); |
2473 | auto NewPred = IsUGT ? CmpInst::ICMP_ULT : CmpInst::ICMP_UGE; |
2474 | return new ICmpInst(NewPred, Shr->getOperand(i_nocapture: 1), NewC); |
2475 | } |
2476 | } |
2477 | |
2478 | const APInt *ShiftAmtC; |
2479 | if (!match(V: Shr->getOperand(i_nocapture: 1), P: m_APInt(Res&: ShiftAmtC))) |
2480 | return nullptr; |
2481 | |
2482 | // Check that the shift amount is in range. If not, don't perform undefined |
2483 | // shifts. When the shift is visited it will be simplified. |
2484 | unsigned TypeBits = C.getBitWidth(); |
2485 | unsigned ShAmtVal = ShiftAmtC->getLimitedValue(Limit: TypeBits); |
2486 | if (ShAmtVal >= TypeBits || ShAmtVal == 0) |
2487 | return nullptr; |
2488 | |
2489 | bool IsExact = Shr->isExact(); |
2490 | Type *ShrTy = Shr->getType(); |
2491 | // TODO: If we could guarantee that InstSimplify would handle all of the |
2492 | // constant-value-based preconditions in the folds below, then we could assert |
2493 | // those conditions rather than checking them. This is difficult because of |
2494 | // undef/poison (PR34838). |
2495 | if (IsAShr && Shr->hasOneUse()) { |
2496 | if (IsExact || Pred == CmpInst::ICMP_SLT || Pred == CmpInst::ICMP_ULT) { |
2497 | // When ShAmtC can be shifted losslessly: |
2498 | // icmp PRED (ashr exact X, ShAmtC), C --> icmp PRED X, (C << ShAmtC) |
2499 | // icmp slt/ult (ashr X, ShAmtC), C --> icmp slt/ult X, (C << ShAmtC) |
2500 | APInt ShiftedC = C.shl(shiftAmt: ShAmtVal); |
2501 | if (ShiftedC.ashr(ShiftAmt: ShAmtVal) == C) |
2502 | return new ICmpInst(Pred, X, ConstantInt::get(Ty: ShrTy, V: ShiftedC)); |
2503 | } |
2504 | if (Pred == CmpInst::ICMP_SGT) { |
2505 | // icmp sgt (ashr X, ShAmtC), C --> icmp sgt X, ((C + 1) << ShAmtC) - 1 |
2506 | APInt ShiftedC = (C + 1).shl(shiftAmt: ShAmtVal) - 1; |
2507 | if (!C.isMaxSignedValue() && !(C + 1).shl(shiftAmt: ShAmtVal).isMinSignedValue() && |
2508 | (ShiftedC + 1).ashr(ShiftAmt: ShAmtVal) == (C + 1)) |
2509 | return new ICmpInst(Pred, X, ConstantInt::get(Ty: ShrTy, V: ShiftedC)); |
2510 | } |
2511 | if (Pred == CmpInst::ICMP_UGT) { |
2512 | // icmp ugt (ashr X, ShAmtC), C --> icmp ugt X, ((C + 1) << ShAmtC) - 1 |
2513 | // 'C + 1 << ShAmtC' can overflow as a signed number, so the 2nd |
2514 | // clause accounts for that pattern. |
2515 | APInt ShiftedC = (C + 1).shl(shiftAmt: ShAmtVal) - 1; |
2516 | if ((ShiftedC + 1).ashr(ShiftAmt: ShAmtVal) == (C + 1) || |
2517 | (C + 1).shl(shiftAmt: ShAmtVal).isMinSignedValue()) |
2518 | return new ICmpInst(Pred, X, ConstantInt::get(Ty: ShrTy, V: ShiftedC)); |
2519 | } |
2520 | |
2521 | // If the compare constant has significant bits above the lowest sign-bit, |
2522 | // then convert an unsigned cmp to a test of the sign-bit: |
2523 | // (ashr X, ShiftC) u> C --> X s< 0 |
2524 | // (ashr X, ShiftC) u< C --> X s> -1 |
2525 | if (C.getBitWidth() > 2 && C.getNumSignBits() <= ShAmtVal) { |
2526 | if (Pred == CmpInst::ICMP_UGT) { |
2527 | return new ICmpInst(CmpInst::ICMP_SLT, X, |
2528 | ConstantInt::getNullValue(Ty: ShrTy)); |
2529 | } |
2530 | if (Pred == CmpInst::ICMP_ULT) { |
2531 | return new ICmpInst(CmpInst::ICMP_SGT, X, |
2532 | ConstantInt::getAllOnesValue(Ty: ShrTy)); |
2533 | } |
2534 | } |
2535 | } else if (!IsAShr) { |
2536 | if (Pred == CmpInst::ICMP_ULT || (Pred == CmpInst::ICMP_UGT && IsExact)) { |
2537 | // icmp ult (lshr X, ShAmtC), C --> icmp ult X, (C << ShAmtC) |
2538 | // icmp ugt (lshr exact X, ShAmtC), C --> icmp ugt X, (C << ShAmtC) |
2539 | APInt ShiftedC = C.shl(shiftAmt: ShAmtVal); |
2540 | if (ShiftedC.lshr(shiftAmt: ShAmtVal) == C) |
2541 | return new ICmpInst(Pred, X, ConstantInt::get(Ty: ShrTy, V: ShiftedC)); |
2542 | } |
2543 | if (Pred == CmpInst::ICMP_UGT) { |
2544 | // icmp ugt (lshr X, ShAmtC), C --> icmp ugt X, ((C + 1) << ShAmtC) - 1 |
2545 | APInt ShiftedC = (C + 1).shl(shiftAmt: ShAmtVal) - 1; |
2546 | if ((ShiftedC + 1).lshr(shiftAmt: ShAmtVal) == (C + 1)) |
2547 | return new ICmpInst(Pred, X, ConstantInt::get(Ty: ShrTy, V: ShiftedC)); |
2548 | } |
2549 | } |
2550 | |
2551 | if (!Cmp.isEquality()) |
2552 | return nullptr; |
2553 | |
2554 | // Handle equality comparisons of shift-by-constant. |
2555 | |
2556 | // If the comparison constant changes with the shift, the comparison cannot |
2557 | // succeed (bits of the comparison constant cannot match the shifted value). |
2558 | // This should be known by InstSimplify and already be folded to true/false. |
2559 | assert(((IsAShr && C.shl(ShAmtVal).ashr(ShAmtVal) == C) || |
2560 | (!IsAShr && C.shl(ShAmtVal).lshr(ShAmtVal) == C)) && |
2561 | "Expected icmp+shr simplify did not occur." ); |
2562 | |
2563 | // If the bits shifted out are known zero, compare the unshifted value: |
2564 | // (X & 4) >> 1 == 2 --> (X & 4) == 4. |
2565 | if (Shr->isExact()) |
2566 | return new ICmpInst(Pred, X, ConstantInt::get(Ty: ShrTy, V: C << ShAmtVal)); |
2567 | |
2568 | if (C.isZero()) { |
2569 | // == 0 is u< 1. |
2570 | if (Pred == CmpInst::ICMP_EQ) |
2571 | return new ICmpInst(CmpInst::ICMP_ULT, X, |
2572 | ConstantInt::get(Ty: ShrTy, V: (C + 1).shl(shiftAmt: ShAmtVal))); |
2573 | else |
2574 | return new ICmpInst(CmpInst::ICMP_UGT, X, |
2575 | ConstantInt::get(Ty: ShrTy, V: (C + 1).shl(shiftAmt: ShAmtVal) - 1)); |
2576 | } |
2577 | |
2578 | if (Shr->hasOneUse()) { |
2579 | // Canonicalize the shift into an 'and': |
2580 | // icmp eq/ne (shr X, ShAmt), C --> icmp eq/ne (and X, HiMask), (C << ShAmt) |
2581 | APInt Val(APInt::getHighBitsSet(numBits: TypeBits, hiBitsSet: TypeBits - ShAmtVal)); |
2582 | Constant *Mask = ConstantInt::get(Ty: ShrTy, V: Val); |
2583 | Value *And = Builder.CreateAnd(LHS: X, RHS: Mask, Name: Shr->getName() + ".mask" ); |
2584 | return new ICmpInst(Pred, And, ConstantInt::get(Ty: ShrTy, V: C << ShAmtVal)); |
2585 | } |
2586 | |
2587 | return nullptr; |
2588 | } |
2589 | |
2590 | Instruction *InstCombinerImpl::foldICmpSRemConstant(ICmpInst &Cmp, |
2591 | BinaryOperator *SRem, |
2592 | const APInt &C) { |
2593 | // Match an 'is positive' or 'is negative' comparison of remainder by a |
2594 | // constant power-of-2 value: |
2595 | // (X % pow2C) sgt/slt 0 |
2596 | const ICmpInst::Predicate Pred = Cmp.getPredicate(); |
2597 | if (Pred != ICmpInst::ICMP_SGT && Pred != ICmpInst::ICMP_SLT && |
2598 | Pred != ICmpInst::ICMP_EQ && Pred != ICmpInst::ICMP_NE) |
2599 | return nullptr; |
2600 | |
2601 | // TODO: The one-use check is standard because we do not typically want to |
2602 | // create longer instruction sequences, but this might be a special-case |
2603 | // because srem is not good for analysis or codegen. |
2604 | if (!SRem->hasOneUse()) |
2605 | return nullptr; |
2606 | |
2607 | const APInt *DivisorC; |
2608 | if (!match(V: SRem->getOperand(i_nocapture: 1), P: m_Power2(V&: DivisorC))) |
2609 | return nullptr; |
2610 | |
2611 | // For cmp_sgt/cmp_slt only zero valued C is handled. |
2612 | // For cmp_eq/cmp_ne only positive valued C is handled. |
2613 | if (((Pred == ICmpInst::ICMP_SGT || Pred == ICmpInst::ICMP_SLT) && |
2614 | !C.isZero()) || |
2615 | ((Pred == ICmpInst::ICMP_EQ || Pred == ICmpInst::ICMP_NE) && |
2616 | !C.isStrictlyPositive())) |
2617 | return nullptr; |
2618 | |
2619 | // Mask off the sign bit and the modulo bits (low-bits). |
2620 | Type *Ty = SRem->getType(); |
2621 | APInt SignMask = APInt::getSignMask(BitWidth: Ty->getScalarSizeInBits()); |
2622 | Constant *MaskC = ConstantInt::get(Ty, V: SignMask | (*DivisorC - 1)); |
2623 | Value *And = Builder.CreateAnd(LHS: SRem->getOperand(i_nocapture: 0), RHS: MaskC); |
2624 | |
2625 | if (Pred == ICmpInst::ICMP_EQ || Pred == ICmpInst::ICMP_NE) |
2626 | return new ICmpInst(Pred, And, ConstantInt::get(Ty, V: C)); |
2627 | |
2628 | // For 'is positive?' check that the sign-bit is clear and at least 1 masked |
2629 | // bit is set. Example: |
2630 | // (i8 X % 32) s> 0 --> (X & 159) s> 0 |
2631 | if (Pred == ICmpInst::ICMP_SGT) |
2632 | return new ICmpInst(ICmpInst::ICMP_SGT, And, ConstantInt::getNullValue(Ty)); |
2633 | |
2634 | // For 'is negative?' check that the sign-bit is set and at least 1 masked |
2635 | // bit is set. Example: |
2636 | // (i16 X % 4) s< 0 --> (X & 32771) u> 32768 |
2637 | return new ICmpInst(ICmpInst::ICMP_UGT, And, ConstantInt::get(Ty, V: SignMask)); |
2638 | } |
2639 | |
2640 | /// Fold icmp (udiv X, Y), C. |
2641 | Instruction *InstCombinerImpl::foldICmpUDivConstant(ICmpInst &Cmp, |
2642 | BinaryOperator *UDiv, |
2643 | const APInt &C) { |
2644 | ICmpInst::Predicate Pred = Cmp.getPredicate(); |
2645 | Value *X = UDiv->getOperand(i_nocapture: 0); |
2646 | Value *Y = UDiv->getOperand(i_nocapture: 1); |
2647 | Type *Ty = UDiv->getType(); |
2648 | |
2649 | const APInt *C2; |
2650 | if (!match(V: X, P: m_APInt(Res&: C2))) |
2651 | return nullptr; |
2652 | |
2653 | assert(*C2 != 0 && "udiv 0, X should have been simplified already." ); |
2654 | |
2655 | // (icmp ugt (udiv C2, Y), C) -> (icmp ule Y, C2/(C+1)) |
2656 | if (Pred == ICmpInst::ICMP_UGT) { |
2657 | assert(!C.isMaxValue() && |
2658 | "icmp ugt X, UINT_MAX should have been simplified already." ); |
2659 | return new ICmpInst(ICmpInst::ICMP_ULE, Y, |
2660 | ConstantInt::get(Ty, V: C2->udiv(RHS: C + 1))); |
2661 | } |
2662 | |
2663 | // (icmp ult (udiv C2, Y), C) -> (icmp ugt Y, C2/C) |
2664 | if (Pred == ICmpInst::ICMP_ULT) { |
2665 | assert(C != 0 && "icmp ult X, 0 should have been simplified already." ); |
2666 | return new ICmpInst(ICmpInst::ICMP_UGT, Y, |
2667 | ConstantInt::get(Ty, V: C2->udiv(RHS: C))); |
2668 | } |
2669 | |
2670 | return nullptr; |
2671 | } |
2672 | |
2673 | /// Fold icmp ({su}div X, Y), C. |
2674 | Instruction *InstCombinerImpl::foldICmpDivConstant(ICmpInst &Cmp, |
2675 | BinaryOperator *Div, |
2676 | const APInt &C) { |
2677 | ICmpInst::Predicate Pred = Cmp.getPredicate(); |
2678 | Value *X = Div->getOperand(i_nocapture: 0); |
2679 | Value *Y = Div->getOperand(i_nocapture: 1); |
2680 | Type *Ty = Div->getType(); |
2681 | bool DivIsSigned = Div->getOpcode() == Instruction::SDiv; |
2682 | |
2683 | // If unsigned division and the compare constant is bigger than |
2684 | // UMAX/2 (negative), there's only one pair of values that satisfies an |
2685 | // equality check, so eliminate the division: |
2686 | // (X u/ Y) == C --> (X == C) && (Y == 1) |
2687 | // (X u/ Y) != C --> (X != C) || (Y != 1) |
2688 | // Similarly, if signed division and the compare constant is exactly SMIN: |
2689 | // (X s/ Y) == SMIN --> (X == SMIN) && (Y == 1) |
2690 | // (X s/ Y) != SMIN --> (X != SMIN) || (Y != 1) |
2691 | if (Cmp.isEquality() && Div->hasOneUse() && C.isSignBitSet() && |
2692 | (!DivIsSigned || C.isMinSignedValue())) { |
2693 | Value *XBig = Builder.CreateICmp(P: Pred, LHS: X, RHS: ConstantInt::get(Ty, V: C)); |
2694 | Value *YOne = Builder.CreateICmp(P: Pred, LHS: Y, RHS: ConstantInt::get(Ty, V: 1)); |
2695 | auto Logic = Pred == ICmpInst::ICMP_EQ ? Instruction::And : Instruction::Or; |
2696 | return BinaryOperator::Create(Op: Logic, S1: XBig, S2: YOne); |
2697 | } |
2698 | |
2699 | // Fold: icmp pred ([us]div X, C2), C -> range test |
2700 | // Fold this div into the comparison, producing a range check. |
2701 | // Determine, based on the divide type, what the range is being |
2702 | // checked. If there is an overflow on the low or high side, remember |
2703 | // it, otherwise compute the range [low, hi) bounding the new value. |
2704 | // See: InsertRangeTest above for the kinds of replacements possible. |
2705 | const APInt *C2; |
2706 | if (!match(V: Y, P: m_APInt(Res&: C2))) |
2707 | return nullptr; |
2708 | |
2709 | // FIXME: If the operand types don't match the type of the divide |
2710 | // then don't attempt this transform. The code below doesn't have the |
2711 | // logic to deal with a signed divide and an unsigned compare (and |
2712 | // vice versa). This is because (x /s C2) <s C produces different |
2713 | // results than (x /s C2) <u C or (x /u C2) <s C or even |
2714 | // (x /u C2) <u C. Simply casting the operands and result won't |
2715 | // work. :( The if statement below tests that condition and bails |
2716 | // if it finds it. |
2717 | if (!Cmp.isEquality() && DivIsSigned != Cmp.isSigned()) |
2718 | return nullptr; |
2719 | |
2720 | // The ProdOV computation fails on divide by 0 and divide by -1. Cases with |
2721 | // INT_MIN will also fail if the divisor is 1. Although folds of all these |
2722 | // division-by-constant cases should be present, we can not assert that they |
2723 | // have happened before we reach this icmp instruction. |
2724 | if (C2->isZero() || C2->isOne() || (DivIsSigned && C2->isAllOnes())) |
2725 | return nullptr; |
2726 | |
2727 | // Compute Prod = C * C2. We are essentially solving an equation of |
2728 | // form X / C2 = C. We solve for X by multiplying C2 and C. |
2729 | // By solving for X, we can turn this into a range check instead of computing |
2730 | // a divide. |
2731 | APInt Prod = C * *C2; |
2732 | |
2733 | // Determine if the product overflows by seeing if the product is not equal to |
2734 | // the divide. Make sure we do the same kind of divide as in the LHS |
2735 | // instruction that we're folding. |
2736 | bool ProdOV = (DivIsSigned ? Prod.sdiv(RHS: *C2) : Prod.udiv(RHS: *C2)) != C; |
2737 | |
2738 | // If the division is known to be exact, then there is no remainder from the |
2739 | // divide, so the covered range size is unit, otherwise it is the divisor. |
2740 | APInt RangeSize = Div->isExact() ? APInt(C2->getBitWidth(), 1) : *C2; |
2741 | |
2742 | // Figure out the interval that is being checked. For example, a comparison |
2743 | // like "X /u 5 == 0" is really checking that X is in the interval [0, 5). |
2744 | // Compute this interval based on the constants involved and the signedness of |
2745 | // the compare/divide. This computes a half-open interval, keeping track of |
2746 | // whether either value in the interval overflows. After analysis each |
2747 | // overflow variable is set to 0 if it's corresponding bound variable is valid |
2748 | // -1 if overflowed off the bottom end, or +1 if overflowed off the top end. |
2749 | int LoOverflow = 0, HiOverflow = 0; |
2750 | APInt LoBound, HiBound; |
2751 | |
2752 | if (!DivIsSigned) { // udiv |
2753 | // e.g. X/5 op 3 --> [15, 20) |
2754 | LoBound = Prod; |
2755 | HiOverflow = LoOverflow = ProdOV; |
2756 | if (!HiOverflow) { |
2757 | // If this is not an exact divide, then many values in the range collapse |
2758 | // to the same result value. |
2759 | HiOverflow = addWithOverflow(Result&: HiBound, In1: LoBound, In2: RangeSize, IsSigned: false); |
2760 | } |
2761 | } else if (C2->isStrictlyPositive()) { // Divisor is > 0. |
2762 | if (C.isZero()) { // (X / pos) op 0 |
2763 | // Can't overflow. e.g. X/2 op 0 --> [-1, 2) |
2764 | LoBound = -(RangeSize - 1); |
2765 | HiBound = RangeSize; |
2766 | } else if (C.isStrictlyPositive()) { // (X / pos) op pos |
2767 | LoBound = Prod; // e.g. X/5 op 3 --> [15, 20) |
2768 | HiOverflow = LoOverflow = ProdOV; |
2769 | if (!HiOverflow) |
2770 | HiOverflow = addWithOverflow(Result&: HiBound, In1: Prod, In2: RangeSize, IsSigned: true); |
2771 | } else { // (X / pos) op neg |
2772 | // e.g. X/5 op -3 --> [-15-4, -15+1) --> [-19, -14) |
2773 | HiBound = Prod + 1; |
2774 | LoOverflow = HiOverflow = ProdOV ? -1 : 0; |
2775 | if (!LoOverflow) { |
2776 | APInt DivNeg = -RangeSize; |
2777 | LoOverflow = addWithOverflow(Result&: LoBound, In1: HiBound, In2: DivNeg, IsSigned: true) ? -1 : 0; |
2778 | } |
2779 | } |
2780 | } else if (C2->isNegative()) { // Divisor is < 0. |
2781 | if (Div->isExact()) |
2782 | RangeSize.negate(); |
2783 | if (C.isZero()) { // (X / neg) op 0 |
2784 | // e.g. X/-5 op 0 --> [-4, 5) |
2785 | LoBound = RangeSize + 1; |
2786 | HiBound = -RangeSize; |
2787 | if (HiBound == *C2) { // -INTMIN = INTMIN |
2788 | HiOverflow = 1; // [INTMIN+1, overflow) |
2789 | HiBound = APInt(); // e.g. X/INTMIN = 0 --> X > INTMIN |
2790 | } |
2791 | } else if (C.isStrictlyPositive()) { // (X / neg) op pos |
2792 | // e.g. X/-5 op 3 --> [-19, -14) |
2793 | HiBound = Prod + 1; |
2794 | HiOverflow = LoOverflow = ProdOV ? -1 : 0; |
2795 | if (!LoOverflow) |
2796 | LoOverflow = |
2797 | addWithOverflow(Result&: LoBound, In1: HiBound, In2: RangeSize, IsSigned: true) ? -1 : 0; |
2798 | } else { // (X / neg) op neg |
2799 | LoBound = Prod; // e.g. X/-5 op -3 --> [15, 20) |
2800 | LoOverflow = HiOverflow = ProdOV; |
2801 | if (!HiOverflow) |
2802 | HiOverflow = subWithOverflow(Result&: HiBound, In1: Prod, In2: RangeSize, IsSigned: true); |
2803 | } |
2804 | |
2805 | // Dividing by a negative swaps the condition. LT <-> GT |
2806 | Pred = ICmpInst::getSwappedPredicate(pred: Pred); |
2807 | } |
2808 | |
2809 | switch (Pred) { |
2810 | default: |
2811 | llvm_unreachable("Unhandled icmp predicate!" ); |
2812 | case ICmpInst::ICMP_EQ: |
2813 | if (LoOverflow && HiOverflow) |
2814 | return replaceInstUsesWith(I&: Cmp, V: Builder.getFalse()); |
2815 | if (HiOverflow) |
2816 | return new ICmpInst(DivIsSigned ? ICmpInst::ICMP_SGE : ICmpInst::ICMP_UGE, |
2817 | X, ConstantInt::get(Ty, V: LoBound)); |
2818 | if (LoOverflow) |
2819 | return new ICmpInst(DivIsSigned ? ICmpInst::ICMP_SLT : ICmpInst::ICMP_ULT, |
2820 | X, ConstantInt::get(Ty, V: HiBound)); |
2821 | return replaceInstUsesWith( |
2822 | I&: Cmp, V: insertRangeTest(V: X, Lo: LoBound, Hi: HiBound, isSigned: DivIsSigned, Inside: true)); |
2823 | case ICmpInst::ICMP_NE: |
2824 | if (LoOverflow && HiOverflow) |
2825 | return replaceInstUsesWith(I&: Cmp, V: Builder.getTrue()); |
2826 | if (HiOverflow) |
2827 | return new ICmpInst(DivIsSigned ? ICmpInst::ICMP_SLT : ICmpInst::ICMP_ULT, |
2828 | X, ConstantInt::get(Ty, V: LoBound)); |
2829 | if (LoOverflow) |
2830 | return new ICmpInst(DivIsSigned ? ICmpInst::ICMP_SGE : ICmpInst::ICMP_UGE, |
2831 | X, ConstantInt::get(Ty, V: HiBound)); |
2832 | return replaceInstUsesWith( |
2833 | I&: Cmp, V: insertRangeTest(V: X, Lo: LoBound, Hi: HiBound, isSigned: DivIsSigned, Inside: false)); |
2834 | case ICmpInst::ICMP_ULT: |
2835 | case ICmpInst::ICMP_SLT: |
2836 | if (LoOverflow == +1) // Low bound is greater than input range. |
2837 | return replaceInstUsesWith(I&: Cmp, V: Builder.getTrue()); |
2838 | if (LoOverflow == -1) // Low bound is less than input range. |
2839 | return replaceInstUsesWith(I&: Cmp, V: Builder.getFalse()); |
2840 | return new ICmpInst(Pred, X, ConstantInt::get(Ty, V: LoBound)); |
2841 | case ICmpInst::ICMP_UGT: |
2842 | case ICmpInst::ICMP_SGT: |
2843 | if (HiOverflow == +1) // High bound greater than input range. |
2844 | return replaceInstUsesWith(I&: Cmp, V: Builder.getFalse()); |
2845 | if (HiOverflow == -1) // High bound less than input range. |
2846 | return replaceInstUsesWith(I&: Cmp, V: Builder.getTrue()); |
2847 | if (Pred == ICmpInst::ICMP_UGT) |
2848 | return new ICmpInst(ICmpInst::ICMP_UGE, X, ConstantInt::get(Ty, V: HiBound)); |
2849 | return new ICmpInst(ICmpInst::ICMP_SGE, X, ConstantInt::get(Ty, V: HiBound)); |
2850 | } |
2851 | |
2852 | return nullptr; |
2853 | } |
2854 | |
2855 | /// Fold icmp (sub X, Y), C. |
2856 | Instruction *InstCombinerImpl::foldICmpSubConstant(ICmpInst &Cmp, |
2857 | BinaryOperator *Sub, |
2858 | const APInt &C) { |
2859 | Value *X = Sub->getOperand(i_nocapture: 0), *Y = Sub->getOperand(i_nocapture: 1); |
2860 | ICmpInst::Predicate Pred = Cmp.getPredicate(); |
2861 | Type *Ty = Sub->getType(); |
2862 | |
2863 | // (SubC - Y) == C) --> Y == (SubC - C) |
2864 | // (SubC - Y) != C) --> Y != (SubC - C) |
2865 | Constant *SubC; |
2866 | if (Cmp.isEquality() && match(V: X, P: m_ImmConstant(C&: SubC))) { |
2867 | return new ICmpInst(Pred, Y, |
2868 | ConstantExpr::getSub(C1: SubC, C2: ConstantInt::get(Ty, V: C))); |
2869 | } |
2870 | |
2871 | // (icmp P (sub nuw|nsw C2, Y), C) -> (icmp swap(P) Y, C2-C) |
2872 | const APInt *C2; |
2873 | APInt SubResult; |
2874 | ICmpInst::Predicate SwappedPred = Cmp.getSwappedPredicate(); |
2875 | bool HasNSW = Sub->hasNoSignedWrap(); |
2876 | bool HasNUW = Sub->hasNoUnsignedWrap(); |
2877 | if (match(V: X, P: m_APInt(Res&: C2)) && |
2878 | ((Cmp.isUnsigned() && HasNUW) || (Cmp.isSigned() && HasNSW)) && |
2879 | !subWithOverflow(Result&: SubResult, In1: *C2, In2: C, IsSigned: Cmp.isSigned())) |
2880 | return new ICmpInst(SwappedPred, Y, ConstantInt::get(Ty, V: SubResult)); |
2881 | |
2882 | // X - Y == 0 --> X == Y. |
2883 | // X - Y != 0 --> X != Y. |
2884 | // TODO: We allow this with multiple uses as long as the other uses are not |
2885 | // in phis. The phi use check is guarding against a codegen regression |
2886 | // for a loop test. If the backend could undo this (and possibly |
2887 | // subsequent transforms), we would not need this hack. |
2888 | if (Cmp.isEquality() && C.isZero() && |
2889 | none_of(Range: (Sub->users()), P: [](const User *U) { return isa<PHINode>(Val: U); })) |
2890 | return new ICmpInst(Pred, X, Y); |
2891 | |
2892 | // The following transforms are only worth it if the only user of the subtract |
2893 | // is the icmp. |
2894 | // TODO: This is an artificial restriction for all of the transforms below |
2895 | // that only need a single replacement icmp. Can these use the phi test |
2896 | // like the transform above here? |
2897 | if (!Sub->hasOneUse()) |
2898 | return nullptr; |
2899 | |
2900 | if (Sub->hasNoSignedWrap()) { |
2901 | // (icmp sgt (sub nsw X, Y), -1) -> (icmp sge X, Y) |
2902 | if (Pred == ICmpInst::ICMP_SGT && C.isAllOnes()) |
2903 | return new ICmpInst(ICmpInst::ICMP_SGE, X, Y); |
2904 | |
2905 | // (icmp sgt (sub nsw X, Y), 0) -> (icmp sgt X, Y) |
2906 | if (Pred == ICmpInst::ICMP_SGT && C.isZero()) |
2907 | return new ICmpInst(ICmpInst::ICMP_SGT, X, Y); |
2908 | |
2909 | // (icmp slt (sub nsw X, Y), 0) -> (icmp slt X, Y) |
2910 | if (Pred == ICmpInst::ICMP_SLT && C.isZero()) |
2911 | return new ICmpInst(ICmpInst::ICMP_SLT, X, Y); |
2912 | |
2913 | // (icmp slt (sub nsw X, Y), 1) -> (icmp sle X, Y) |
2914 | if (Pred == ICmpInst::ICMP_SLT && C.isOne()) |
2915 | return new ICmpInst(ICmpInst::ICMP_SLE, X, Y); |
2916 | } |
2917 | |
2918 | if (!match(V: X, P: m_APInt(Res&: C2))) |
2919 | return nullptr; |
2920 | |
2921 | // C2 - Y <u C -> (Y | (C - 1)) == C2 |
2922 | // iff (C2 & (C - 1)) == C - 1 and C is a power of 2 |
2923 | if (Pred == ICmpInst::ICMP_ULT && C.isPowerOf2() && |
2924 | (*C2 & (C - 1)) == (C - 1)) |
2925 | return new ICmpInst(ICmpInst::ICMP_EQ, Builder.CreateOr(LHS: Y, RHS: C - 1), X); |
2926 | |
2927 | // C2 - Y >u C -> (Y | C) != C2 |
2928 | // iff C2 & C == C and C + 1 is a power of 2 |
2929 | if (Pred == ICmpInst::ICMP_UGT && (C + 1).isPowerOf2() && (*C2 & C) == C) |
2930 | return new ICmpInst(ICmpInst::ICMP_NE, Builder.CreateOr(LHS: Y, RHS: C), X); |
2931 | |
2932 | // We have handled special cases that reduce. |
2933 | // Canonicalize any remaining sub to add as: |
2934 | // (C2 - Y) > C --> (Y + ~C2) < ~C |
2935 | Value *Add = Builder.CreateAdd(LHS: Y, RHS: ConstantInt::get(Ty, V: ~(*C2)), Name: "notsub" , |
2936 | HasNUW, HasNSW); |
2937 | return new ICmpInst(SwappedPred, Add, ConstantInt::get(Ty, V: ~C)); |
2938 | } |
2939 | |
2940 | static Value *createLogicFromTable(const std::bitset<4> &Table, Value *Op0, |
2941 | Value *Op1, IRBuilderBase &Builder, |
2942 | bool HasOneUse) { |
2943 | auto FoldConstant = [&](bool Val) { |
2944 | Constant *Res = Val ? Builder.getTrue() : Builder.getFalse(); |
2945 | if (Op0->getType()->isVectorTy()) |
2946 | Res = ConstantVector::getSplat( |
2947 | EC: cast<VectorType>(Val: Op0->getType())->getElementCount(), Elt: Res); |
2948 | return Res; |
2949 | }; |
2950 | |
2951 | switch (Table.to_ulong()) { |
2952 | case 0: // 0 0 0 0 |
2953 | return FoldConstant(false); |
2954 | case 1: // 0 0 0 1 |
2955 | return HasOneUse ? Builder.CreateNot(V: Builder.CreateOr(LHS: Op0, RHS: Op1)) : nullptr; |
2956 | case 2: // 0 0 1 0 |
2957 | return HasOneUse ? Builder.CreateAnd(LHS: Builder.CreateNot(V: Op0), RHS: Op1) : nullptr; |
2958 | case 3: // 0 0 1 1 |
2959 | return Builder.CreateNot(V: Op0); |
2960 | case 4: // 0 1 0 0 |
2961 | return HasOneUse ? Builder.CreateAnd(LHS: Op0, RHS: Builder.CreateNot(V: Op1)) : nullptr; |
2962 | case 5: // 0 1 0 1 |
2963 | return Builder.CreateNot(V: Op1); |
2964 | case 6: // 0 1 1 0 |
2965 | return Builder.CreateXor(LHS: Op0, RHS: Op1); |
2966 | case 7: // 0 1 1 1 |
2967 | return HasOneUse ? Builder.CreateNot(V: Builder.CreateAnd(LHS: Op0, RHS: Op1)) : nullptr; |
2968 | case 8: // 1 0 0 0 |
2969 | return Builder.CreateAnd(LHS: Op0, RHS: Op1); |
2970 | case 9: // 1 0 0 1 |
2971 | return HasOneUse ? Builder.CreateNot(V: Builder.CreateXor(LHS: Op0, RHS: Op1)) : nullptr; |
2972 | case 10: // 1 0 1 0 |
2973 | return Op1; |
2974 | case 11: // 1 0 1 1 |
2975 | return HasOneUse ? Builder.CreateOr(LHS: Builder.CreateNot(V: Op0), RHS: Op1) : nullptr; |
2976 | case 12: // 1 1 0 0 |
2977 | return Op0; |
2978 | case 13: // 1 1 0 1 |
2979 | return HasOneUse ? Builder.CreateOr(LHS: Op0, RHS: Builder.CreateNot(V: Op1)) : nullptr; |
2980 | case 14: // 1 1 1 0 |
2981 | return Builder.CreateOr(LHS: Op0, RHS: Op1); |
2982 | case 15: // 1 1 1 1 |
2983 | return FoldConstant(true); |
2984 | default: |
2985 | llvm_unreachable("Invalid Operation" ); |
2986 | } |
2987 | return nullptr; |
2988 | } |
2989 | |
2990 | /// Fold icmp (add X, Y), C. |
2991 | Instruction *InstCombinerImpl::foldICmpAddConstant(ICmpInst &Cmp, |
2992 | BinaryOperator *Add, |
2993 | const APInt &C) { |
2994 | Value *Y = Add->getOperand(i_nocapture: 1); |
2995 | Value *X = Add->getOperand(i_nocapture: 0); |
2996 | |
2997 | Value *Op0, *Op1; |
2998 | Instruction *Ext0, *Ext1; |
2999 | const CmpInst::Predicate Pred = Cmp.getPredicate(); |
3000 | if (match(V: Add, |
3001 | P: m_Add(L: m_CombineAnd(L: m_Instruction(I&: Ext0), R: m_ZExtOrSExt(Op: m_Value(V&: Op0))), |
3002 | R: m_CombineAnd(L: m_Instruction(I&: Ext1), |
3003 | R: m_ZExtOrSExt(Op: m_Value(V&: Op1))))) && |
3004 | Op0->getType()->isIntOrIntVectorTy(BitWidth: 1) && |
3005 | Op1->getType()->isIntOrIntVectorTy(BitWidth: 1)) { |
3006 | unsigned BW = C.getBitWidth(); |
3007 | std::bitset<4> Table; |
3008 | auto ComputeTable = [&](bool Op0Val, bool Op1Val) { |
3009 | int Res = 0; |
3010 | if (Op0Val) |
3011 | Res += isa<ZExtInst>(Val: Ext0) ? 1 : -1; |
3012 | if (Op1Val) |
3013 | Res += isa<ZExtInst>(Val: Ext1) ? 1 : -1; |
3014 | return ICmpInst::compare(LHS: APInt(BW, Res, true), RHS: C, Pred); |
3015 | }; |
3016 | |
3017 | Table[0] = ComputeTable(false, false); |
3018 | Table[1] = ComputeTable(false, true); |
3019 | Table[2] = ComputeTable(true, false); |
3020 | Table[3] = ComputeTable(true, true); |
3021 | if (auto *Cond = |
3022 | createLogicFromTable(Table, Op0, Op1, Builder, HasOneUse: Add->hasOneUse())) |
3023 | return replaceInstUsesWith(I&: Cmp, V: Cond); |
3024 | } |
3025 | const APInt *C2; |
3026 | if (Cmp.isEquality() || !match(V: Y, P: m_APInt(Res&: C2))) |
3027 | return nullptr; |
3028 | |
3029 | // Fold icmp pred (add X, C2), C. |
3030 | Type *Ty = Add->getType(); |
3031 | |
3032 | // If the add does not wrap, we can always adjust the compare by subtracting |
3033 | // the constants. Equality comparisons are handled elsewhere. SGE/SLE/UGE/ULE |
3034 | // are canonicalized to SGT/SLT/UGT/ULT. |
3035 | if ((Add->hasNoSignedWrap() && |
3036 | (Pred == ICmpInst::ICMP_SGT || Pred == ICmpInst::ICMP_SLT)) || |
3037 | (Add->hasNoUnsignedWrap() && |
3038 | (Pred == ICmpInst::ICMP_UGT || Pred == ICmpInst::ICMP_ULT))) { |
3039 | bool Overflow; |
3040 | APInt NewC = |
3041 | Cmp.isSigned() ? C.ssub_ov(RHS: *C2, Overflow) : C.usub_ov(RHS: *C2, Overflow); |
3042 | // If there is overflow, the result must be true or false. |
3043 | // TODO: Can we assert there is no overflow because InstSimplify always |
3044 | // handles those cases? |
3045 | if (!Overflow) |
3046 | // icmp Pred (add nsw X, C2), C --> icmp Pred X, (C - C2) |
3047 | return new ICmpInst(Pred, X, ConstantInt::get(Ty, V: NewC)); |
3048 | } |
3049 | |
3050 | auto CR = ConstantRange::makeExactICmpRegion(Pred, Other: C).subtract(CI: *C2); |
3051 | const APInt &Upper = CR.getUpper(); |
3052 | const APInt &Lower = CR.getLower(); |
3053 | if (Cmp.isSigned()) { |
3054 | if (Lower.isSignMask()) |
3055 | return new ICmpInst(ICmpInst::ICMP_SLT, X, ConstantInt::get(Ty, V: Upper)); |
3056 | if (Upper.isSignMask()) |
3057 | return new ICmpInst(ICmpInst::ICMP_SGE, X, ConstantInt::get(Ty, V: Lower)); |
3058 | } else { |
3059 | if (Lower.isMinValue()) |
3060 | return new ICmpInst(ICmpInst::ICMP_ULT, X, ConstantInt::get(Ty, V: Upper)); |
3061 | if (Upper.isMinValue()) |
3062 | return new ICmpInst(ICmpInst::ICMP_UGE, X, ConstantInt::get(Ty, V: Lower)); |
3063 | } |
3064 | |
3065 | // This set of folds is intentionally placed after folds that use no-wrapping |
3066 | // flags because those folds are likely better for later analysis/codegen. |
3067 | const APInt SMax = APInt::getSignedMaxValue(numBits: Ty->getScalarSizeInBits()); |
3068 | const APInt SMin = APInt::getSignedMinValue(numBits: Ty->getScalarSizeInBits()); |
3069 | |
3070 | // Fold compare with offset to opposite sign compare if it eliminates offset: |
3071 | // (X + C2) >u C --> X <s -C2 (if C == C2 + SMAX) |
3072 | if (Pred == CmpInst::ICMP_UGT && C == *C2 + SMax) |
3073 | return new ICmpInst(ICmpInst::ICMP_SLT, X, ConstantInt::get(Ty, V: -(*C2))); |
3074 | |
3075 | // (X + C2) <u C --> X >s ~C2 (if C == C2 + SMIN) |
3076 | if (Pred == CmpInst::ICMP_ULT && C == *C2 + SMin) |
3077 | return new ICmpInst(ICmpInst::ICMP_SGT, X, ConstantInt::get(Ty, V: ~(*C2))); |
3078 | |
3079 | // (X + C2) >s C --> X <u (SMAX - C) (if C == C2 - 1) |
3080 | if (Pred == CmpInst::ICMP_SGT && C == *C2 - 1) |
3081 | return new ICmpInst(ICmpInst::ICMP_ULT, X, ConstantInt::get(Ty, V: SMax - C)); |
3082 | |
3083 | // (X + C2) <s C --> X >u (C ^ SMAX) (if C == C2) |
3084 | if (Pred == CmpInst::ICMP_SLT && C == *C2) |
3085 | return new ICmpInst(ICmpInst::ICMP_UGT, X, ConstantInt::get(Ty, V: C ^ SMax)); |
3086 | |
3087 | // (X + -1) <u C --> X <=u C (if X is never null) |
3088 | if (Pred == CmpInst::ICMP_ULT && C2->isAllOnes()) { |
3089 | const SimplifyQuery Q = SQ.getWithInstruction(I: &Cmp); |
3090 | if (llvm::isKnownNonZero(V: X, Q)) |
3091 | return new ICmpInst(ICmpInst::ICMP_ULE, X, ConstantInt::get(Ty, V: C)); |
3092 | } |
3093 | |
3094 | if (!Add->hasOneUse()) |
3095 | return nullptr; |
3096 | |
3097 | // X+C <u C2 -> (X & -C2) == C |
3098 | // iff C & (C2-1) == 0 |
3099 | // C2 is a power of 2 |
3100 | if (Pred == ICmpInst::ICMP_ULT && C.isPowerOf2() && (*C2 & (C - 1)) == 0) |
3101 | return new ICmpInst(ICmpInst::ICMP_EQ, Builder.CreateAnd(LHS: X, RHS: -C), |
3102 | ConstantExpr::getNeg(C: cast<Constant>(Val: Y))); |
3103 | |
3104 | // X+C >u C2 -> (X & ~C2) != C |
3105 | // iff C & C2 == 0 |
3106 | // C2+1 is a power of 2 |
3107 | if (Pred == ICmpInst::ICMP_UGT && (C + 1).isPowerOf2() && (*C2 & C) == 0) |
3108 | return new ICmpInst(ICmpInst::ICMP_NE, Builder.CreateAnd(LHS: X, RHS: ~C), |
3109 | ConstantExpr::getNeg(C: cast<Constant>(Val: Y))); |
3110 | |
3111 | // The range test idiom can use either ult or ugt. Arbitrarily canonicalize |
3112 | // to the ult form. |
3113 | // X+C2 >u C -> X+(C2-C-1) <u ~C |
3114 | if (Pred == ICmpInst::ICMP_UGT) |
3115 | return new ICmpInst(ICmpInst::ICMP_ULT, |
3116 | Builder.CreateAdd(LHS: X, RHS: ConstantInt::get(Ty, V: *C2 - C - 1)), |
3117 | ConstantInt::get(Ty, V: ~C)); |
3118 | |
3119 | return nullptr; |
3120 | } |
3121 | |
3122 | bool InstCombinerImpl::matchThreeWayIntCompare(SelectInst *SI, Value *&LHS, |
3123 | Value *&RHS, ConstantInt *&Less, |
3124 | ConstantInt *&Equal, |
3125 | ConstantInt *&Greater) { |
3126 | // TODO: Generalize this to work with other comparison idioms or ensure |
3127 | // they get canonicalized into this form. |
3128 | |
3129 | // select i1 (a == b), |
3130 | // i32 Equal, |
3131 | // i32 (select i1 (a < b), i32 Less, i32 Greater) |
3132 | // where Equal, Less and Greater are placeholders for any three constants. |
3133 | ICmpInst::Predicate PredA; |
3134 | if (!match(V: SI->getCondition(), P: m_ICmp(Pred&: PredA, L: m_Value(V&: LHS), R: m_Value(V&: RHS))) || |
3135 | !ICmpInst::isEquality(P: PredA)) |
3136 | return false; |
3137 | Value *EqualVal = SI->getTrueValue(); |
3138 | Value *UnequalVal = SI->getFalseValue(); |
3139 | // We still can get non-canonical predicate here, so canonicalize. |
3140 | if (PredA == ICmpInst::ICMP_NE) |
3141 | std::swap(a&: EqualVal, b&: UnequalVal); |
3142 | if (!match(V: EqualVal, P: m_ConstantInt(CI&: Equal))) |
3143 | return false; |
3144 | ICmpInst::Predicate PredB; |
3145 | Value *LHS2, *RHS2; |
3146 | if (!match(V: UnequalVal, P: m_Select(C: m_ICmp(Pred&: PredB, L: m_Value(V&: LHS2), R: m_Value(V&: RHS2)), |
3147 | L: m_ConstantInt(CI&: Less), R: m_ConstantInt(CI&: Greater)))) |
3148 | return false; |
3149 | // We can get predicate mismatch here, so canonicalize if possible: |
3150 | // First, ensure that 'LHS' match. |
3151 | if (LHS2 != LHS) { |
3152 | // x sgt y <--> y slt x |
3153 | std::swap(a&: LHS2, b&: RHS2); |
3154 | PredB = ICmpInst::getSwappedPredicate(pred: PredB); |
3155 | } |
3156 | if (LHS2 != LHS) |
3157 | return false; |
3158 | // We also need to canonicalize 'RHS'. |
3159 | if (PredB == ICmpInst::ICMP_SGT && isa<Constant>(Val: RHS2)) { |
3160 | // x sgt C-1 <--> x sge C <--> not(x slt C) |
3161 | auto FlippedStrictness = |
3162 | InstCombiner::getFlippedStrictnessPredicateAndConstant( |
3163 | Pred: PredB, C: cast<Constant>(Val: RHS2)); |
3164 | if (!FlippedStrictness) |
3165 | return false; |
3166 | assert(FlippedStrictness->first == ICmpInst::ICMP_SGE && |
3167 | "basic correctness failure" ); |
3168 | RHS2 = FlippedStrictness->second; |
3169 | // And kind-of perform the result swap. |
3170 | std::swap(a&: Less, b&: Greater); |
3171 | PredB = ICmpInst::ICMP_SLT; |
3172 | } |
3173 | return PredB == ICmpInst::ICMP_SLT && RHS == RHS2; |
3174 | } |
3175 | |
3176 | Instruction *InstCombinerImpl::foldICmpSelectConstant(ICmpInst &Cmp, |
3177 | SelectInst *Select, |
3178 | ConstantInt *C) { |
3179 | |
3180 | assert(C && "Cmp RHS should be a constant int!" ); |
3181 | // If we're testing a constant value against the result of a three way |
3182 | // comparison, the result can be expressed directly in terms of the |
3183 | // original values being compared. Note: We could possibly be more |
3184 | // aggressive here and remove the hasOneUse test. The original select is |
3185 | // really likely to simplify or sink when we remove a test of the result. |
3186 | Value *OrigLHS, *OrigRHS; |
3187 | ConstantInt *C1LessThan, *C2Equal, *C3GreaterThan; |
3188 | if (Cmp.hasOneUse() && |
3189 | matchThreeWayIntCompare(SI: Select, LHS&: OrigLHS, RHS&: OrigRHS, Less&: C1LessThan, Equal&: C2Equal, |
3190 | Greater&: C3GreaterThan)) { |
3191 | assert(C1LessThan && C2Equal && C3GreaterThan); |
3192 | |
3193 | bool TrueWhenLessThan = |
3194 | ConstantExpr::getCompare(pred: Cmp.getPredicate(), C1: C1LessThan, C2: C) |
3195 | ->isAllOnesValue(); |
3196 | bool TrueWhenEqual = |
3197 | ConstantExpr::getCompare(pred: Cmp.getPredicate(), C1: C2Equal, C2: C) |
3198 | ->isAllOnesValue(); |
3199 | bool TrueWhenGreaterThan = |
3200 | ConstantExpr::getCompare(pred: Cmp.getPredicate(), C1: C3GreaterThan, C2: C) |
3201 | ->isAllOnesValue(); |
3202 | |
3203 | // This generates the new instruction that will replace the original Cmp |
3204 | // Instruction. Instead of enumerating the various combinations when |
3205 | // TrueWhenLessThan, TrueWhenEqual and TrueWhenGreaterThan are true versus |
3206 | // false, we rely on chaining of ORs and future passes of InstCombine to |
3207 | // simplify the OR further (i.e. a s< b || a == b becomes a s<= b). |
3208 | |
3209 | // When none of the three constants satisfy the predicate for the RHS (C), |
3210 | // the entire original Cmp can be simplified to a false. |
3211 | Value *Cond = Builder.getFalse(); |
3212 | if (TrueWhenLessThan) |
3213 | Cond = Builder.CreateOr(LHS: Cond, RHS: Builder.CreateICmp(P: ICmpInst::ICMP_SLT, |
3214 | LHS: OrigLHS, RHS: OrigRHS)); |
3215 | if (TrueWhenEqual) |
3216 | Cond = Builder.CreateOr(LHS: Cond, RHS: Builder.CreateICmp(P: ICmpInst::ICMP_EQ, |
3217 | LHS: OrigLHS, RHS: OrigRHS)); |
3218 | if (TrueWhenGreaterThan) |
3219 | Cond = Builder.CreateOr(LHS: Cond, RHS: Builder.CreateICmp(P: ICmpInst::ICMP_SGT, |
3220 | LHS: OrigLHS, RHS: OrigRHS)); |
3221 | |
3222 | return replaceInstUsesWith(I&: Cmp, V: Cond); |
3223 | } |
3224 | return nullptr; |
3225 | } |
3226 | |
3227 | Instruction *InstCombinerImpl::foldICmpBitCast(ICmpInst &Cmp) { |
3228 | auto *Bitcast = dyn_cast<BitCastInst>(Val: Cmp.getOperand(i_nocapture: 0)); |
3229 | if (!Bitcast) |
3230 | return nullptr; |
3231 | |
3232 | ICmpInst::Predicate Pred = Cmp.getPredicate(); |
3233 | Value *Op1 = Cmp.getOperand(i_nocapture: 1); |
3234 | Value *BCSrcOp = Bitcast->getOperand(i_nocapture: 0); |
3235 | Type *SrcType = Bitcast->getSrcTy(); |
3236 | Type *DstType = Bitcast->getType(); |
3237 | |
3238 | // Make sure the bitcast doesn't change between scalar and vector and |
3239 | // doesn't change the number of vector elements. |
3240 | if (SrcType->isVectorTy() == DstType->isVectorTy() && |
3241 | SrcType->getScalarSizeInBits() == DstType->getScalarSizeInBits()) { |
3242 | // Zero-equality and sign-bit checks are preserved through sitofp + bitcast. |
3243 | Value *X; |
3244 | if (match(V: BCSrcOp, P: m_SIToFP(Op: m_Value(V&: X)))) { |
3245 | // icmp eq (bitcast (sitofp X)), 0 --> icmp eq X, 0 |
3246 | // icmp ne (bitcast (sitofp X)), 0 --> icmp ne X, 0 |
3247 | // icmp slt (bitcast (sitofp X)), 0 --> icmp slt X, 0 |
3248 | // icmp sgt (bitcast (sitofp X)), 0 --> icmp sgt X, 0 |
3249 | if ((Pred == ICmpInst::ICMP_EQ || Pred == ICmpInst::ICMP_SLT || |
3250 | Pred == ICmpInst::ICMP_NE || Pred == ICmpInst::ICMP_SGT) && |
3251 | match(V: Op1, P: m_Zero())) |
3252 | return new ICmpInst(Pred, X, ConstantInt::getNullValue(Ty: X->getType())); |
3253 | |
3254 | // icmp slt (bitcast (sitofp X)), 1 --> icmp slt X, 1 |
3255 | if (Pred == ICmpInst::ICMP_SLT && match(V: Op1, P: m_One())) |
3256 | return new ICmpInst(Pred, X, ConstantInt::get(Ty: X->getType(), V: 1)); |
3257 | |
3258 | // icmp sgt (bitcast (sitofp X)), -1 --> icmp sgt X, -1 |
3259 | if (Pred == ICmpInst::ICMP_SGT && match(V: Op1, P: m_AllOnes())) |
3260 | return new ICmpInst(Pred, X, |
3261 | ConstantInt::getAllOnesValue(Ty: X->getType())); |
3262 | } |
3263 | |
3264 | // Zero-equality checks are preserved through unsigned floating-point casts: |
3265 | // icmp eq (bitcast (uitofp X)), 0 --> icmp eq X, 0 |
3266 | // icmp ne (bitcast (uitofp X)), 0 --> icmp ne X, 0 |
3267 | if (match(V: BCSrcOp, P: m_UIToFP(Op: m_Value(V&: X)))) |
3268 | if (Cmp.isEquality() && match(V: Op1, P: m_Zero())) |
3269 | return new ICmpInst(Pred, X, ConstantInt::getNullValue(Ty: X->getType())); |
3270 | |
3271 | const APInt *C; |
3272 | bool TrueIfSigned; |
3273 | if (match(V: Op1, P: m_APInt(Res&: C)) && Bitcast->hasOneUse()) { |
3274 | // If this is a sign-bit test of a bitcast of a casted FP value, eliminate |
3275 | // the FP extend/truncate because that cast does not change the sign-bit. |
3276 | // This is true for all standard IEEE-754 types and the X86 80-bit type. |
3277 | // The sign-bit is always the most significant bit in those types. |
3278 | if (isSignBitCheck(Pred, RHS: *C, TrueIfSigned) && |
3279 | (match(V: BCSrcOp, P: m_FPExt(Op: m_Value(V&: X))) || |
3280 | match(V: BCSrcOp, P: m_FPTrunc(Op: m_Value(V&: X))))) { |
3281 | // (bitcast (fpext/fptrunc X)) to iX) < 0 --> (bitcast X to iY) < 0 |
3282 | // (bitcast (fpext/fptrunc X)) to iX) > -1 --> (bitcast X to iY) > -1 |
3283 | Type *XType = X->getType(); |
3284 | |
3285 | // We can't currently handle Power style floating point operations here. |
3286 | if (!(XType->isPPC_FP128Ty() || SrcType->isPPC_FP128Ty())) { |
3287 | Type *NewType = Builder.getIntNTy(N: XType->getScalarSizeInBits()); |
3288 | if (auto *XVTy = dyn_cast<VectorType>(Val: XType)) |
3289 | NewType = VectorType::get(ElementType: NewType, EC: XVTy->getElementCount()); |
3290 | Value *NewBitcast = Builder.CreateBitCast(V: X, DestTy: NewType); |
3291 | if (TrueIfSigned) |
3292 | return new ICmpInst(ICmpInst::ICMP_SLT, NewBitcast, |
3293 | ConstantInt::getNullValue(Ty: NewType)); |
3294 | else |
3295 | return new ICmpInst(ICmpInst::ICMP_SGT, NewBitcast, |
3296 | ConstantInt::getAllOnesValue(Ty: NewType)); |
3297 | } |
3298 | } |
3299 | |
3300 | // icmp eq/ne (bitcast X to int), special fp -> llvm.is.fpclass(X, class) |
3301 | Type *FPType = SrcType->getScalarType(); |
3302 | if (!Cmp.getParent()->getParent()->hasFnAttribute( |
3303 | Attribute::NoImplicitFloat) && |
3304 | Cmp.isEquality() && FPType->isIEEELikeFPTy()) { |
3305 | FPClassTest Mask = APFloat(FPType->getFltSemantics(), *C).classify(); |
3306 | if (Mask & (fcInf | fcZero)) { |
3307 | if (Pred == ICmpInst::ICMP_NE) |
3308 | Mask = ~Mask; |
3309 | return replaceInstUsesWith(I&: Cmp, |
3310 | V: Builder.createIsFPClass(FPNum: BCSrcOp, Test: Mask)); |
3311 | } |
3312 | } |
3313 | } |
3314 | } |
3315 | |
3316 | const APInt *C; |
3317 | if (!match(V: Cmp.getOperand(i_nocapture: 1), P: m_APInt(Res&: C)) || !DstType->isIntegerTy() || |
3318 | !SrcType->isIntOrIntVectorTy()) |
3319 | return nullptr; |
3320 | |
3321 | // If this is checking if all elements of a vector compare are set or not, |
3322 | // invert the casted vector equality compare and test if all compare |
3323 | // elements are clear or not. Compare against zero is generally easier for |
3324 | // analysis and codegen. |
3325 | // icmp eq/ne (bitcast (not X) to iN), -1 --> icmp eq/ne (bitcast X to iN), 0 |
3326 | // Example: are all elements equal? --> are zero elements not equal? |
3327 | // TODO: Try harder to reduce compare of 2 freely invertible operands? |
3328 | if (Cmp.isEquality() && C->isAllOnes() && Bitcast->hasOneUse()) { |
3329 | if (Value *NotBCSrcOp = |
3330 | getFreelyInverted(V: BCSrcOp, WillInvertAllUses: BCSrcOp->hasOneUse(), Builder: &Builder)) { |
3331 | Value *Cast = Builder.CreateBitCast(V: NotBCSrcOp, DestTy: DstType); |
3332 | return new ICmpInst(Pred, Cast, ConstantInt::getNullValue(Ty: DstType)); |
3333 | } |
3334 | } |
3335 | |
3336 | // If this is checking if all elements of an extended vector are clear or not, |
3337 | // compare in a narrow type to eliminate the extend: |
3338 | // icmp eq/ne (bitcast (ext X) to iN), 0 --> icmp eq/ne (bitcast X to iM), 0 |
3339 | Value *X; |
3340 | if (Cmp.isEquality() && C->isZero() && Bitcast->hasOneUse() && |
3341 | match(V: BCSrcOp, P: m_ZExtOrSExt(Op: m_Value(V&: X)))) { |
3342 | if (auto *VecTy = dyn_cast<FixedVectorType>(Val: X->getType())) { |
3343 | Type *NewType = Builder.getIntNTy(N: VecTy->getPrimitiveSizeInBits()); |
3344 | Value *NewCast = Builder.CreateBitCast(V: X, DestTy: NewType); |
3345 | return new ICmpInst(Pred, NewCast, ConstantInt::getNullValue(Ty: NewType)); |
3346 | } |
3347 | } |
3348 | |
3349 | // Folding: icmp <pred> iN X, C |
3350 | // where X = bitcast <M x iK> (shufflevector <M x iK> %vec, undef, SC)) to iN |
3351 | // and C is a splat of a K-bit pattern |
3352 | // and SC is a constant vector = <C', C', C', ..., C'> |
3353 | // Into: |
3354 | // %E = extractelement <M x iK> %vec, i32 C' |
3355 | // icmp <pred> iK %E, trunc(C) |
3356 | Value *Vec; |
3357 | ArrayRef<int> Mask; |
3358 | if (match(V: BCSrcOp, P: m_Shuffle(v1: m_Value(V&: Vec), v2: m_Undef(), mask: m_Mask(Mask)))) { |
3359 | // Check whether every element of Mask is the same constant |
3360 | if (all_equal(Range&: Mask)) { |
3361 | auto *VecTy = cast<VectorType>(Val: SrcType); |
3362 | auto *EltTy = cast<IntegerType>(Val: VecTy->getElementType()); |
3363 | if (C->isSplat(SplatSizeInBits: EltTy->getBitWidth())) { |
3364 | // Fold the icmp based on the value of C |
3365 | // If C is M copies of an iK sized bit pattern, |
3366 | // then: |
3367 | // => %E = extractelement <N x iK> %vec, i32 Elem |
3368 | // icmp <pred> iK %SplatVal, <pattern> |
3369 | Value *Elem = Builder.getInt32(C: Mask[0]); |
3370 | Value * = Builder.CreateExtractElement(Vec, Idx: Elem); |
3371 | Value *NewC = ConstantInt::get(Ty: EltTy, V: C->trunc(width: EltTy->getBitWidth())); |
3372 | return new ICmpInst(Pred, Extract, NewC); |
3373 | } |
3374 | } |
3375 | } |
3376 | return nullptr; |
3377 | } |
3378 | |
3379 | /// Try to fold integer comparisons with a constant operand: icmp Pred X, C |
3380 | /// where X is some kind of instruction. |
3381 | Instruction *InstCombinerImpl::foldICmpInstWithConstant(ICmpInst &Cmp) { |
3382 | const APInt *C; |
3383 | |
3384 | if (match(V: Cmp.getOperand(i_nocapture: 1), P: m_APInt(Res&: C))) { |
3385 | if (auto *BO = dyn_cast<BinaryOperator>(Val: Cmp.getOperand(i_nocapture: 0))) |
3386 | if (Instruction *I = foldICmpBinOpWithConstant(Cmp, BO, C: *C)) |
3387 | return I; |
3388 | |
3389 | if (auto *SI = dyn_cast<SelectInst>(Val: Cmp.getOperand(i_nocapture: 0))) |
3390 | // For now, we only support constant integers while folding the |
3391 | // ICMP(SELECT)) pattern. We can extend this to support vector of integers |
3392 | // similar to the cases handled by binary ops above. |
3393 | if (auto *ConstRHS = dyn_cast<ConstantInt>(Val: Cmp.getOperand(i_nocapture: 1))) |
3394 | if (Instruction *I = foldICmpSelectConstant(Cmp, Select: SI, C: ConstRHS)) |
3395 | return I; |
3396 | |
3397 | if (auto *TI = dyn_cast<TruncInst>(Val: Cmp.getOperand(i_nocapture: 0))) |
3398 | if (Instruction *I = foldICmpTruncConstant(Cmp, Trunc: TI, C: *C)) |
3399 | return I; |
3400 | |
3401 | if (auto *II = dyn_cast<IntrinsicInst>(Val: Cmp.getOperand(i_nocapture: 0))) |
3402 | if (Instruction *I = foldICmpIntrinsicWithConstant(ICI&: Cmp, II, C: *C)) |
3403 | return I; |
3404 | |
3405 | // (extractval ([s/u]subo X, Y), 0) == 0 --> X == Y |
3406 | // (extractval ([s/u]subo X, Y), 0) != 0 --> X != Y |
3407 | // TODO: This checks one-use, but that is not strictly necessary. |
3408 | Value *Cmp0 = Cmp.getOperand(i_nocapture: 0); |
3409 | Value *X, *Y; |
3410 | if (C->isZero() && Cmp.isEquality() && Cmp0->hasOneUse() && |
3411 | (match(Cmp0, |
3412 | m_ExtractValue<0>(m_Intrinsic<Intrinsic::ssub_with_overflow>( |
3413 | m_Value(X), m_Value(Y)))) || |
3414 | match(Cmp0, |
3415 | m_ExtractValue<0>(m_Intrinsic<Intrinsic::usub_with_overflow>( |
3416 | m_Value(X), m_Value(Y)))))) |
3417 | return new ICmpInst(Cmp.getPredicate(), X, Y); |
3418 | } |
3419 | |
3420 | if (match(V: Cmp.getOperand(i_nocapture: 1), P: m_APIntAllowPoison(Res&: C))) |
3421 | return foldICmpInstWithConstantAllowPoison(Cmp, C: *C); |
3422 | |
3423 | return nullptr; |
3424 | } |
3425 | |
3426 | /// Fold an icmp equality instruction with binary operator LHS and constant RHS: |
3427 | /// icmp eq/ne BO, C. |
3428 | Instruction *InstCombinerImpl::foldICmpBinOpEqualityWithConstant( |
3429 | ICmpInst &Cmp, BinaryOperator *BO, const APInt &C) { |
3430 | // TODO: Some of these folds could work with arbitrary constants, but this |
3431 | // function is limited to scalar and vector splat constants. |
3432 | if (!Cmp.isEquality()) |
3433 | return nullptr; |
3434 | |
3435 | ICmpInst::Predicate Pred = Cmp.getPredicate(); |
3436 | bool isICMP_NE = Pred == ICmpInst::ICMP_NE; |
3437 | Constant *RHS = cast<Constant>(Val: Cmp.getOperand(i_nocapture: 1)); |
3438 | Value *BOp0 = BO->getOperand(i_nocapture: 0), *BOp1 = BO->getOperand(i_nocapture: 1); |
3439 | |
3440 | switch (BO->getOpcode()) { |
3441 | case Instruction::SRem: |
3442 | // If we have a signed (X % (2^c)) == 0, turn it into an unsigned one. |
3443 | if (C.isZero() && BO->hasOneUse()) { |
3444 | const APInt *BOC; |
3445 | if (match(V: BOp1, P: m_APInt(Res&: BOC)) && BOC->sgt(RHS: 1) && BOC->isPowerOf2()) { |
3446 | Value *NewRem = Builder.CreateURem(LHS: BOp0, RHS: BOp1, Name: BO->getName()); |
3447 | return new ICmpInst(Pred, NewRem, |
3448 | Constant::getNullValue(Ty: BO->getType())); |
3449 | } |
3450 | } |
3451 | break; |
3452 | case Instruction::Add: { |
3453 | // (A + C2) == C --> A == (C - C2) |
3454 | // (A + C2) != C --> A != (C - C2) |
3455 | // TODO: Remove the one-use limitation? See discussion in D58633. |
3456 | if (Constant *C2 = dyn_cast<Constant>(Val: BOp1)) { |
3457 | if (BO->hasOneUse()) |
3458 | return new ICmpInst(Pred, BOp0, ConstantExpr::getSub(C1: RHS, C2)); |
3459 | } else if (C.isZero()) { |
3460 | // Replace ((add A, B) != 0) with (A != -B) if A or B is |
3461 | // efficiently invertible, or if the add has just this one use. |
3462 | if (Value *NegVal = dyn_castNegVal(V: BOp1)) |
3463 | return new ICmpInst(Pred, BOp0, NegVal); |
3464 | if (Value *NegVal = dyn_castNegVal(V: BOp0)) |
3465 | return new ICmpInst(Pred, NegVal, BOp1); |
3466 | if (BO->hasOneUse()) { |
3467 | // (add nuw A, B) != 0 -> (or A, B) != 0 |
3468 | if (match(V: BO, P: m_NUWAdd(L: m_Value(), R: m_Value()))) { |
3469 | Value *Or = Builder.CreateOr(LHS: BOp0, RHS: BOp1); |
3470 | return new ICmpInst(Pred, Or, Constant::getNullValue(Ty: BO->getType())); |
3471 | } |
3472 | Value *Neg = Builder.CreateNeg(V: BOp1); |
3473 | Neg->takeName(V: BO); |
3474 | return new ICmpInst(Pred, BOp0, Neg); |
3475 | } |
3476 | } |
3477 | break; |
3478 | } |
3479 | case Instruction::Xor: |
3480 | if (BO->hasOneUse()) { |
3481 | if (Constant *BOC = dyn_cast<Constant>(Val: BOp1)) { |
3482 | // For the xor case, we can xor two constants together, eliminating |
3483 | // the explicit xor. |
3484 | return new ICmpInst(Pred, BOp0, ConstantExpr::getXor(C1: RHS, C2: BOC)); |
3485 | } else if (C.isZero()) { |
3486 | // Replace ((xor A, B) != 0) with (A != B) |
3487 | return new ICmpInst(Pred, BOp0, BOp1); |
3488 | } |
3489 | } |
3490 | break; |
3491 | case Instruction::Or: { |
3492 | const APInt *BOC; |
3493 | if (match(V: BOp1, P: m_APInt(Res&: BOC)) && BO->hasOneUse() && RHS->isAllOnesValue()) { |
3494 | // Comparing if all bits outside of a constant mask are set? |
3495 | // Replace (X | C) == -1 with (X & ~C) == ~C. |
3496 | // This removes the -1 constant. |
3497 | Constant *NotBOC = ConstantExpr::getNot(C: cast<Constant>(Val: BOp1)); |
3498 | Value *And = Builder.CreateAnd(LHS: BOp0, RHS: NotBOC); |
3499 | return new ICmpInst(Pred, And, NotBOC); |
3500 | } |
3501 | break; |
3502 | } |
3503 | case Instruction::UDiv: |
3504 | case Instruction::SDiv: |
3505 | if (BO->isExact()) { |
3506 | // div exact X, Y eq/ne 0 -> X eq/ne 0 |
3507 | // div exact X, Y eq/ne 1 -> X eq/ne Y |
3508 | // div exact X, Y eq/ne C -> |
3509 | // if Y * C never-overflow && OneUse: |
3510 | // -> Y * C eq/ne X |
3511 | if (C.isZero()) |
3512 | return new ICmpInst(Pred, BOp0, Constant::getNullValue(Ty: BO->getType())); |
3513 | else if (C.isOne()) |
3514 | return new ICmpInst(Pred, BOp0, BOp1); |
3515 | else if (BO->hasOneUse()) { |
3516 | OverflowResult OR = computeOverflow( |
3517 | BinaryOp: Instruction::Mul, IsSigned: BO->getOpcode() == Instruction::SDiv, LHS: BOp1, |
3518 | RHS: Cmp.getOperand(i_nocapture: 1), CxtI: BO); |
3519 | if (OR == OverflowResult::NeverOverflows) { |
3520 | Value *YC = |
3521 | Builder.CreateMul(LHS: BOp1, RHS: ConstantInt::get(Ty: BO->getType(), V: C)); |
3522 | return new ICmpInst(Pred, YC, BOp0); |
3523 | } |
3524 | } |
3525 | } |
3526 | if (BO->getOpcode() == Instruction::UDiv && C.isZero()) { |
3527 | // (icmp eq/ne (udiv A, B), 0) -> (icmp ugt/ule i32 B, A) |
3528 | auto NewPred = isICMP_NE ? ICmpInst::ICMP_ULE : ICmpInst::ICMP_UGT; |
3529 | return new ICmpInst(NewPred, BOp1, BOp0); |
3530 | } |
3531 | break; |
3532 | default: |
3533 | break; |
3534 | } |
3535 | return nullptr; |
3536 | } |
3537 | |
3538 | static Instruction *foldCtpopPow2Test(ICmpInst &I, IntrinsicInst *CtpopLhs, |
3539 | const APInt &CRhs, |
3540 | InstCombiner::BuilderTy &Builder, |
3541 | const SimplifyQuery &Q) { |
3542 | assert(CtpopLhs->getIntrinsicID() == Intrinsic::ctpop && |
3543 | "Non-ctpop intrin in ctpop fold" ); |
3544 | if (!CtpopLhs->hasOneUse()) |
3545 | return nullptr; |
3546 | |
3547 | // Power of 2 test: |
3548 | // isPow2OrZero : ctpop(X) u< 2 |
3549 | // isPow2 : ctpop(X) == 1 |
3550 | // NotPow2OrZero: ctpop(X) u> 1 |
3551 | // NotPow2 : ctpop(X) != 1 |
3552 | // If we know any bit of X can be folded to: |
3553 | // IsPow2 : X & (~Bit) == 0 |
3554 | // NotPow2 : X & (~Bit) != 0 |
3555 | const ICmpInst::Predicate Pred = I.getPredicate(); |
3556 | if (((I.isEquality() || Pred == ICmpInst::ICMP_UGT) && CRhs == 1) || |
3557 | (Pred == ICmpInst::ICMP_ULT && CRhs == 2)) { |
3558 | Value *Op = CtpopLhs->getArgOperand(i: 0); |
3559 | KnownBits OpKnown = computeKnownBits(V: Op, DL: Q.DL, |
3560 | /*Depth*/ 0, AC: Q.AC, CxtI: Q.CxtI, DT: Q.DT); |
3561 | // No need to check for count > 1, that should be already constant folded. |
3562 | if (OpKnown.countMinPopulation() == 1) { |
3563 | Value *And = Builder.CreateAnd( |
3564 | LHS: Op, RHS: Constant::getIntegerValue(Ty: Op->getType(), V: ~(OpKnown.One))); |
3565 | return new ICmpInst( |
3566 | (Pred == ICmpInst::ICMP_EQ || Pred == ICmpInst::ICMP_ULT) |
3567 | ? ICmpInst::ICMP_EQ |
3568 | : ICmpInst::ICMP_NE, |
3569 | And, Constant::getNullValue(Ty: Op->getType())); |
3570 | } |
3571 | } |
3572 | |
3573 | return nullptr; |
3574 | } |
3575 | |
3576 | /// Fold an equality icmp with LLVM intrinsic and constant operand. |
3577 | Instruction *InstCombinerImpl::foldICmpEqIntrinsicWithConstant( |
3578 | ICmpInst &Cmp, IntrinsicInst *II, const APInt &C) { |
3579 | Type *Ty = II->getType(); |
3580 | unsigned BitWidth = C.getBitWidth(); |
3581 | const ICmpInst::Predicate Pred = Cmp.getPredicate(); |
3582 | |
3583 | switch (II->getIntrinsicID()) { |
3584 | case Intrinsic::abs: |
3585 | // abs(A) == 0 -> A == 0 |
3586 | // abs(A) == INT_MIN -> A == INT_MIN |
3587 | if (C.isZero() || C.isMinSignedValue()) |
3588 | return new ICmpInst(Pred, II->getArgOperand(i: 0), ConstantInt::get(Ty, V: C)); |
3589 | break; |
3590 | |
3591 | case Intrinsic::bswap: |
3592 | // bswap(A) == C -> A == bswap(C) |
3593 | return new ICmpInst(Pred, II->getArgOperand(i: 0), |
3594 | ConstantInt::get(Ty, V: C.byteSwap())); |
3595 | |
3596 | case Intrinsic::bitreverse: |
3597 | // bitreverse(A) == C -> A == bitreverse(C) |
3598 | return new ICmpInst(Pred, II->getArgOperand(i: 0), |
3599 | ConstantInt::get(Ty, V: C.reverseBits())); |
3600 | |
3601 | case Intrinsic::ctlz: |
3602 | case Intrinsic::cttz: { |
3603 | // ctz(A) == bitwidth(A) -> A == 0 and likewise for != |
3604 | if (C == BitWidth) |
3605 | return new ICmpInst(Pred, II->getArgOperand(i: 0), |
3606 | ConstantInt::getNullValue(Ty)); |
3607 | |
3608 | // ctz(A) == C -> A & Mask1 == Mask2, where Mask2 only has bit C set |
3609 | // and Mask1 has bits 0..C+1 set. Similar for ctl, but for high bits. |
3610 | // Limit to one use to ensure we don't increase instruction count. |
3611 | unsigned Num = C.getLimitedValue(Limit: BitWidth); |
3612 | if (Num != BitWidth && II->hasOneUse()) { |
3613 | bool IsTrailing = II->getIntrinsicID() == Intrinsic::cttz; |
3614 | APInt Mask1 = IsTrailing ? APInt::getLowBitsSet(numBits: BitWidth, loBitsSet: Num + 1) |
3615 | : APInt::getHighBitsSet(numBits: BitWidth, hiBitsSet: Num + 1); |
3616 | APInt Mask2 = IsTrailing |
3617 | ? APInt::getOneBitSet(numBits: BitWidth, BitNo: Num) |
3618 | : APInt::getOneBitSet(numBits: BitWidth, BitNo: BitWidth - Num - 1); |
3619 | return new ICmpInst(Pred, Builder.CreateAnd(LHS: II->getArgOperand(i: 0), RHS: Mask1), |
3620 | ConstantInt::get(Ty, V: Mask2)); |
3621 | } |
3622 | break; |
3623 | } |
3624 | |
3625 | case Intrinsic::ctpop: { |
3626 | // popcount(A) == 0 -> A == 0 and likewise for != |
3627 | // popcount(A) == bitwidth(A) -> A == -1 and likewise for != |
3628 | bool IsZero = C.isZero(); |
3629 | if (IsZero || C == BitWidth) |
3630 | return new ICmpInst(Pred, II->getArgOperand(i: 0), |
3631 | IsZero ? Constant::getNullValue(Ty) |
3632 | : Constant::getAllOnesValue(Ty)); |
3633 | |
3634 | break; |
3635 | } |
3636 | |
3637 | case Intrinsic::fshl: |
3638 | case Intrinsic::fshr: |
3639 | if (II->getArgOperand(i: 0) == II->getArgOperand(i: 1)) { |
3640 | const APInt *RotAmtC; |
3641 | // ror(X, RotAmtC) == C --> X == rol(C, RotAmtC) |
3642 | // rol(X, RotAmtC) == C --> X == ror(C, RotAmtC) |
3643 | if (match(V: II->getArgOperand(i: 2), P: m_APInt(Res&: RotAmtC))) |
3644 | return new ICmpInst(Pred, II->getArgOperand(i: 0), |
3645 | II->getIntrinsicID() == Intrinsic::fshl |
3646 | ? ConstantInt::get(Ty, V: C.rotr(rotateAmt: *RotAmtC)) |
3647 | : ConstantInt::get(Ty, V: C.rotl(rotateAmt: *RotAmtC))); |
3648 | } |
3649 | break; |
3650 | |
3651 | case Intrinsic::umax: |
3652 | case Intrinsic::uadd_sat: { |
3653 | // uadd.sat(a, b) == 0 -> (a | b) == 0 |
3654 | // umax(a, b) == 0 -> (a | b) == 0 |
3655 | if (C.isZero() && II->hasOneUse()) { |
3656 | Value *Or = Builder.CreateOr(LHS: II->getArgOperand(i: 0), RHS: II->getArgOperand(i: 1)); |
3657 | return new ICmpInst(Pred, Or, Constant::getNullValue(Ty)); |
3658 | } |
3659 | break; |
3660 | } |
3661 | |
3662 | case Intrinsic::ssub_sat: |
3663 | // ssub.sat(a, b) == 0 -> a == b |
3664 | if (C.isZero()) |
3665 | return new ICmpInst(Pred, II->getArgOperand(i: 0), II->getArgOperand(i: 1)); |
3666 | break; |
3667 | case Intrinsic::usub_sat: { |
3668 | // usub.sat(a, b) == 0 -> a <= b |
3669 | if (C.isZero()) { |
3670 | ICmpInst::Predicate NewPred = |
3671 | Pred == ICmpInst::ICMP_EQ ? ICmpInst::ICMP_ULE : ICmpInst::ICMP_UGT; |
3672 | return new ICmpInst(NewPred, II->getArgOperand(i: 0), II->getArgOperand(i: 1)); |
3673 | } |
3674 | break; |
3675 | } |
3676 | default: |
3677 | break; |
3678 | } |
3679 | |
3680 | return nullptr; |
3681 | } |
3682 | |
3683 | /// Fold an icmp with LLVM intrinsics |
3684 | static Instruction * |
3685 | foldICmpIntrinsicWithIntrinsic(ICmpInst &Cmp, |
3686 | InstCombiner::BuilderTy &Builder) { |
3687 | assert(Cmp.isEquality()); |
3688 | |
3689 | ICmpInst::Predicate Pred = Cmp.getPredicate(); |
3690 | Value *Op0 = Cmp.getOperand(i_nocapture: 0); |
3691 | Value *Op1 = Cmp.getOperand(i_nocapture: 1); |
3692 | const auto *IIOp0 = dyn_cast<IntrinsicInst>(Val: Op0); |
3693 | const auto *IIOp1 = dyn_cast<IntrinsicInst>(Val: Op1); |
3694 | if (!IIOp0 || !IIOp1 || IIOp0->getIntrinsicID() != IIOp1->getIntrinsicID()) |
3695 | return nullptr; |
3696 | |
3697 | switch (IIOp0->getIntrinsicID()) { |
3698 | case Intrinsic::bswap: |
3699 | case Intrinsic::bitreverse: |
3700 | // If both operands are byte-swapped or bit-reversed, just compare the |
3701 | // original values. |
3702 | return new ICmpInst(Pred, IIOp0->getOperand(i_nocapture: 0), IIOp1->getOperand(i_nocapture: 0)); |
3703 | case Intrinsic::fshl: |
3704 | case Intrinsic::fshr: { |
3705 | // If both operands are rotated by same amount, just compare the |
3706 | // original values. |
3707 | if (IIOp0->getOperand(i_nocapture: 0) != IIOp0->getOperand(i_nocapture: 1)) |
3708 | break; |
3709 | if (IIOp1->getOperand(i_nocapture: 0) != IIOp1->getOperand(i_nocapture: 1)) |
3710 | break; |
3711 | if (IIOp0->getOperand(i_nocapture: 2) == IIOp1->getOperand(i_nocapture: 2)) |
3712 | return new ICmpInst(Pred, IIOp0->getOperand(i_nocapture: 0), IIOp1->getOperand(i_nocapture: 0)); |
3713 | |
3714 | // rotate(X, AmtX) == rotate(Y, AmtY) |
3715 | // -> rotate(X, AmtX - AmtY) == Y |
3716 | // Do this if either both rotates have one use or if only one has one use |
3717 | // and AmtX/AmtY are constants. |
3718 | unsigned OneUses = IIOp0->hasOneUse() + IIOp1->hasOneUse(); |
3719 | if (OneUses == 2 || |
3720 | (OneUses == 1 && match(V: IIOp0->getOperand(i_nocapture: 2), P: m_ImmConstant()) && |
3721 | match(V: IIOp1->getOperand(i_nocapture: 2), P: m_ImmConstant()))) { |
3722 | Value *SubAmt = |
3723 | Builder.CreateSub(LHS: IIOp0->getOperand(i_nocapture: 2), RHS: IIOp1->getOperand(i_nocapture: 2)); |
3724 | Value *CombinedRotate = Builder.CreateIntrinsic( |
3725 | RetTy: Op0->getType(), ID: IIOp0->getIntrinsicID(), |
3726 | Args: {IIOp0->getOperand(i_nocapture: 0), IIOp0->getOperand(i_nocapture: 0), SubAmt}); |
3727 | return new ICmpInst(Pred, IIOp1->getOperand(i_nocapture: 0), CombinedRotate); |
3728 | } |
3729 | } break; |
3730 | default: |
3731 | break; |
3732 | } |
3733 | |
3734 | return nullptr; |
3735 | } |
3736 | |
3737 | /// Try to fold integer comparisons with a constant operand: icmp Pred X, C |
3738 | /// where X is some kind of instruction and C is AllowPoison. |
3739 | /// TODO: Move more folds which allow poison to this function. |
3740 | Instruction * |
3741 | InstCombinerImpl::foldICmpInstWithConstantAllowPoison(ICmpInst &Cmp, |
3742 | const APInt &C) { |
3743 | const ICmpInst::Predicate Pred = Cmp.getPredicate(); |
3744 | if (auto *II = dyn_cast<IntrinsicInst>(Val: Cmp.getOperand(i_nocapture: 0))) { |
3745 | switch (II->getIntrinsicID()) { |
3746 | default: |
3747 | break; |
3748 | case Intrinsic::fshl: |
3749 | case Intrinsic::fshr: |
3750 | if (Cmp.isEquality() && II->getArgOperand(i: 0) == II->getArgOperand(i: 1)) { |
3751 | // (rot X, ?) == 0/-1 --> X == 0/-1 |
3752 | if (C.isZero() || C.isAllOnes()) |
3753 | return new ICmpInst(Pred, II->getArgOperand(i: 0), Cmp.getOperand(i_nocapture: 1)); |
3754 | } |
3755 | break; |
3756 | } |
3757 | } |
3758 | |
3759 | return nullptr; |
3760 | } |
3761 | |
3762 | /// Fold an icmp with BinaryOp and constant operand: icmp Pred BO, C. |
3763 | Instruction *InstCombinerImpl::foldICmpBinOpWithConstant(ICmpInst &Cmp, |
3764 | BinaryOperator *BO, |
3765 | const APInt &C) { |
3766 | switch (BO->getOpcode()) { |
3767 | case Instruction::Xor: |
3768 | if (Instruction *I = foldICmpXorConstant(Cmp, Xor: BO, C)) |
3769 | return I; |
3770 | break; |
3771 | case Instruction::And: |
3772 | if (Instruction *I = foldICmpAndConstant(Cmp, And: BO, C)) |
3773 | return I; |
3774 | break; |
3775 | case Instruction::Or: |
3776 | if (Instruction *I = foldICmpOrConstant(Cmp, Or: BO, C)) |
3777 | return I; |
3778 | break; |
3779 | case Instruction::Mul: |
3780 | if (Instruction *I = foldICmpMulConstant(Cmp, Mul: BO, C)) |
3781 | return I; |
3782 | break; |
3783 | case Instruction::Shl: |
3784 | if (Instruction *I = foldICmpShlConstant(Cmp, Shl: BO, C)) |
3785 | return I; |
3786 | break; |
3787 | case Instruction::LShr: |
3788 | case Instruction::AShr: |
3789 | if (Instruction *I = foldICmpShrConstant(Cmp, Shr: BO, C)) |
3790 | return I; |
3791 | break; |
3792 | case Instruction::SRem: |
3793 | if (Instruction *I = foldICmpSRemConstant(Cmp, SRem: BO, C)) |
3794 | return I; |
3795 | break; |
3796 | case Instruction::UDiv: |
3797 | if (Instruction *I = foldICmpUDivConstant(Cmp, UDiv: BO, C)) |
3798 | return I; |
3799 | [[fallthrough]]; |
3800 | case Instruction::SDiv: |
3801 | if (Instruction *I = foldICmpDivConstant(Cmp, Div: BO, C)) |
3802 | return I; |
3803 | break; |
3804 | case Instruction::Sub: |
3805 | if (Instruction *I = foldICmpSubConstant(Cmp, Sub: BO, C)) |
3806 | return I; |
3807 | break; |
3808 | case Instruction::Add: |
3809 | if (Instruction *I = foldICmpAddConstant(Cmp, Add: BO, C)) |
3810 | return I; |
3811 | break; |
3812 | default: |
3813 | break; |
3814 | } |
3815 | |
3816 | // TODO: These folds could be refactored to be part of the above calls. |
3817 | return foldICmpBinOpEqualityWithConstant(Cmp, BO, C); |
3818 | } |
3819 | |
3820 | static Instruction * |
3821 | foldICmpUSubSatOrUAddSatWithConstant(ICmpInst::Predicate Pred, |
3822 | SaturatingInst *II, const APInt &C, |
3823 | InstCombiner::BuilderTy &Builder) { |
3824 | // This transform may end up producing more than one instruction for the |
3825 | // intrinsic, so limit it to one user of the intrinsic. |
3826 | if (!II->hasOneUse()) |
3827 | return nullptr; |
3828 | |
3829 | // Let Y = [add/sub]_sat(X, C) pred C2 |
3830 | // SatVal = The saturating value for the operation |
3831 | // WillWrap = Whether or not the operation will underflow / overflow |
3832 | // => Y = (WillWrap ? SatVal : (X binop C)) pred C2 |
3833 | // => Y = WillWrap ? (SatVal pred C2) : ((X binop C) pred C2) |
3834 | // |
3835 | // When (SatVal pred C2) is true, then |
3836 | // Y = WillWrap ? true : ((X binop C) pred C2) |
3837 | // => Y = WillWrap || ((X binop C) pred C2) |
3838 | // else |
3839 | // Y = WillWrap ? false : ((X binop C) pred C2) |
3840 | // => Y = !WillWrap ? ((X binop C) pred C2) : false |
3841 | // => Y = !WillWrap && ((X binop C) pred C2) |
3842 | Value *Op0 = II->getOperand(i_nocapture: 0); |
3843 | Value *Op1 = II->getOperand(i_nocapture: 1); |
3844 | |
3845 | const APInt *COp1; |
3846 | // This transform only works when the intrinsic has an integral constant or |
3847 | // splat vector as the second operand. |
3848 | if (!match(V: Op1, P: m_APInt(Res&: COp1))) |
3849 | return nullptr; |
3850 | |
3851 | APInt SatVal; |
3852 | switch (II->getIntrinsicID()) { |
3853 | default: |
3854 | llvm_unreachable( |
3855 | "This function only works with usub_sat and uadd_sat for now!" ); |
3856 | case Intrinsic::uadd_sat: |
3857 | SatVal = APInt::getAllOnes(numBits: C.getBitWidth()); |
3858 | break; |
3859 | case Intrinsic::usub_sat: |
3860 | SatVal = APInt::getZero(numBits: C.getBitWidth()); |
3861 | break; |
3862 | } |
3863 | |
3864 | // Check (SatVal pred C2) |
3865 | bool SatValCheck = ICmpInst::compare(LHS: SatVal, RHS: C, Pred); |
3866 | |
3867 | // !WillWrap. |
3868 | ConstantRange C1 = ConstantRange::makeExactNoWrapRegion( |
3869 | BinOp: II->getBinaryOp(), Other: *COp1, NoWrapKind: II->getNoWrapKind()); |
3870 | |
3871 | // WillWrap. |
3872 | if (SatValCheck) |
3873 | C1 = C1.inverse(); |
3874 | |
3875 | ConstantRange C2 = ConstantRange::makeExactICmpRegion(Pred, Other: C); |
3876 | if (II->getBinaryOp() == Instruction::Add) |
3877 | C2 = C2.sub(Other: *COp1); |
3878 | else |
3879 | C2 = C2.add(Other: *COp1); |
3880 | |
3881 | Instruction::BinaryOps CombiningOp = |
3882 | SatValCheck ? Instruction::BinaryOps::Or : Instruction::BinaryOps::And; |
3883 | |
3884 | std::optional<ConstantRange> Combination; |
3885 | if (CombiningOp == Instruction::BinaryOps::Or) |
3886 | Combination = C1.exactUnionWith(CR: C2); |
3887 | else /* CombiningOp == Instruction::BinaryOps::And */ |
3888 | Combination = C1.exactIntersectWith(CR: C2); |
3889 | |
3890 | if (!Combination) |
3891 | return nullptr; |
3892 | |
3893 | CmpInst::Predicate EquivPred; |
3894 | APInt EquivInt; |
3895 | APInt EquivOffset; |
3896 | |
3897 | Combination->getEquivalentICmp(Pred&: EquivPred, RHS&: EquivInt, Offset&: EquivOffset); |
3898 | |
3899 | return new ICmpInst( |
3900 | EquivPred, |
3901 | Builder.CreateAdd(LHS: Op0, RHS: ConstantInt::get(Ty: Op1->getType(), V: EquivOffset)), |
3902 | ConstantInt::get(Ty: Op1->getType(), V: EquivInt)); |
3903 | } |
3904 | |
3905 | /// Fold an icmp with LLVM intrinsic and constant operand: icmp Pred II, C. |
3906 | Instruction *InstCombinerImpl::foldICmpIntrinsicWithConstant(ICmpInst &Cmp, |
3907 | IntrinsicInst *II, |
3908 | const APInt &C) { |
3909 | ICmpInst::Predicate Pred = Cmp.getPredicate(); |
3910 | |
3911 | // Handle folds that apply for any kind of icmp. |
3912 | switch (II->getIntrinsicID()) { |
3913 | default: |
3914 | break; |
3915 | case Intrinsic::uadd_sat: |
3916 | case Intrinsic::usub_sat: |
3917 | if (auto *Folded = foldICmpUSubSatOrUAddSatWithConstant( |
3918 | Pred, II: cast<SaturatingInst>(Val: II), C, Builder)) |
3919 | return Folded; |
3920 | break; |
3921 | case Intrinsic::ctpop: { |
3922 | const SimplifyQuery Q = SQ.getWithInstruction(I: &Cmp); |
3923 | if (Instruction *R = foldCtpopPow2Test(I&: Cmp, CtpopLhs: II, CRhs: C, Builder, Q)) |
3924 | return R; |
3925 | } break; |
3926 | } |
3927 | |
3928 | if (Cmp.isEquality()) |
3929 | return foldICmpEqIntrinsicWithConstant(Cmp, II, C); |
3930 | |
3931 | Type *Ty = II->getType(); |
3932 | unsigned BitWidth = C.getBitWidth(); |
3933 | switch (II->getIntrinsicID()) { |
3934 | case Intrinsic::ctpop: { |
3935 | // (ctpop X > BitWidth - 1) --> X == -1 |
3936 | Value *X = II->getArgOperand(i: 0); |
3937 | if (C == BitWidth - 1 && Pred == ICmpInst::ICMP_UGT) |
3938 | return CmpInst::Create(Op: Instruction::ICmp, Pred: ICmpInst::ICMP_EQ, S1: X, |
3939 | S2: ConstantInt::getAllOnesValue(Ty)); |
3940 | // (ctpop X < BitWidth) --> X != -1 |
3941 | if (C == BitWidth && Pred == ICmpInst::ICMP_ULT) |
3942 | return CmpInst::Create(Op: Instruction::ICmp, Pred: ICmpInst::ICMP_NE, S1: X, |
3943 | S2: ConstantInt::getAllOnesValue(Ty)); |
3944 | break; |
3945 | } |
3946 | case Intrinsic::ctlz: { |
3947 | // ctlz(0bXXXXXXXX) > 3 -> 0bXXXXXXXX < 0b00010000 |
3948 | if (Pred == ICmpInst::ICMP_UGT && C.ult(RHS: BitWidth)) { |
3949 | unsigned Num = C.getLimitedValue(); |
3950 | APInt Limit = APInt::getOneBitSet(numBits: BitWidth, BitNo: BitWidth - Num - 1); |
3951 | return CmpInst::Create(Op: Instruction::ICmp, Pred: ICmpInst::ICMP_ULT, |
3952 | S1: II->getArgOperand(i: 0), S2: ConstantInt::get(Ty, V: Limit)); |
3953 | } |
3954 | |
3955 | // ctlz(0bXXXXXXXX) < 3 -> 0bXXXXXXXX > 0b00011111 |
3956 | if (Pred == ICmpInst::ICMP_ULT && C.uge(RHS: 1) && C.ule(RHS: BitWidth)) { |
3957 | unsigned Num = C.getLimitedValue(); |
3958 | APInt Limit = APInt::getLowBitsSet(numBits: BitWidth, loBitsSet: BitWidth - Num); |
3959 | return CmpInst::Create(Op: Instruction::ICmp, Pred: ICmpInst::ICMP_UGT, |
3960 | S1: II->getArgOperand(i: 0), S2: ConstantInt::get(Ty, V: Limit)); |
3961 | } |
3962 | break; |
3963 | } |
3964 | case Intrinsic::cttz: { |
3965 | // Limit to one use to ensure we don't increase instruction count. |
3966 | if (!II->hasOneUse()) |
3967 | return nullptr; |
3968 | |
3969 | // cttz(0bXXXXXXXX) > 3 -> 0bXXXXXXXX & 0b00001111 == 0 |
3970 | if (Pred == ICmpInst::ICMP_UGT && C.ult(RHS: BitWidth)) { |
3971 | APInt Mask = APInt::getLowBitsSet(numBits: BitWidth, loBitsSet: C.getLimitedValue() + 1); |
3972 | return CmpInst::Create(Op: Instruction::ICmp, Pred: ICmpInst::ICMP_EQ, |
3973 | S1: Builder.CreateAnd(LHS: II->getArgOperand(i: 0), RHS: Mask), |
3974 | S2: ConstantInt::getNullValue(Ty)); |
3975 | } |
3976 | |
3977 | // cttz(0bXXXXXXXX) < 3 -> 0bXXXXXXXX & 0b00000111 != 0 |
3978 | if (Pred == ICmpInst::ICMP_ULT && C.uge(RHS: 1) && C.ule(RHS: BitWidth)) { |
3979 | APInt Mask = APInt::getLowBitsSet(numBits: BitWidth, loBitsSet: C.getLimitedValue()); |
3980 | return CmpInst::Create(Op: Instruction::ICmp, Pred: ICmpInst::ICMP_NE, |
3981 | S1: Builder.CreateAnd(LHS: II->getArgOperand(i: 0), RHS: Mask), |
3982 | S2: ConstantInt::getNullValue(Ty)); |
3983 | } |
3984 | break; |
3985 | } |
3986 | case Intrinsic::ssub_sat: |
3987 | // ssub.sat(a, b) spred 0 -> a spred b |
3988 | if (ICmpInst::isSigned(predicate: Pred)) { |
3989 | if (C.isZero()) |
3990 | return new ICmpInst(Pred, II->getArgOperand(i: 0), II->getArgOperand(i: 1)); |
3991 | // X s<= 0 is cannonicalized to X s< 1 |
3992 | if (Pred == ICmpInst::ICMP_SLT && C.isOne()) |
3993 | return new ICmpInst(ICmpInst::ICMP_SLE, II->getArgOperand(i: 0), |
3994 | II->getArgOperand(i: 1)); |
3995 | // X s>= 0 is cannonicalized to X s> -1 |
3996 | if (Pred == ICmpInst::ICMP_SGT && C.isAllOnes()) |
3997 | return new ICmpInst(ICmpInst::ICMP_SGE, II->getArgOperand(i: 0), |
3998 | II->getArgOperand(i: 1)); |
3999 | } |
4000 | break; |
4001 | default: |
4002 | break; |
4003 | } |
4004 | |
4005 | return nullptr; |
4006 | } |
4007 | |
4008 | /// Handle icmp with constant (but not simple integer constant) RHS. |
4009 | Instruction *InstCombinerImpl::foldICmpInstWithConstantNotInt(ICmpInst &I) { |
4010 | Value *Op0 = I.getOperand(i_nocapture: 0), *Op1 = I.getOperand(i_nocapture: 1); |
4011 | Constant *RHSC = dyn_cast<Constant>(Val: Op1); |
4012 | Instruction *LHSI = dyn_cast<Instruction>(Val: Op0); |
4013 | if (!RHSC || !LHSI) |
4014 | return nullptr; |
4015 | |
4016 | switch (LHSI->getOpcode()) { |
4017 | case Instruction::PHI: |
4018 | if (Instruction *NV = foldOpIntoPhi(I, PN: cast<PHINode>(Val: LHSI))) |
4019 | return NV; |
4020 | break; |
4021 | case Instruction::IntToPtr: |
4022 | // icmp pred inttoptr(X), null -> icmp pred X, 0 |
4023 | if (RHSC->isNullValue() && |
4024 | DL.getIntPtrType(RHSC->getType()) == LHSI->getOperand(i: 0)->getType()) |
4025 | return new ICmpInst( |
4026 | I.getPredicate(), LHSI->getOperand(i: 0), |
4027 | Constant::getNullValue(Ty: LHSI->getOperand(i: 0)->getType())); |
4028 | break; |
4029 | |
4030 | case Instruction::Load: |
4031 | // Try to optimize things like "A[i] > 4" to index computations. |
4032 | if (GetElementPtrInst *GEP = |
4033 | dyn_cast<GetElementPtrInst>(Val: LHSI->getOperand(i: 0))) |
4034 | if (GlobalVariable *GV = dyn_cast<GlobalVariable>(Val: GEP->getOperand(i_nocapture: 0))) |
4035 | if (Instruction *Res = |
4036 | foldCmpLoadFromIndexedGlobal(LI: cast<LoadInst>(Val: LHSI), GEP, GV, ICI&: I)) |
4037 | return Res; |
4038 | break; |
4039 | } |
4040 | |
4041 | return nullptr; |
4042 | } |
4043 | |
4044 | Instruction *InstCombinerImpl::foldSelectICmp(ICmpInst::Predicate Pred, |
4045 | SelectInst *SI, Value *RHS, |
4046 | const ICmpInst &I) { |
4047 | // Try to fold the comparison into the select arms, which will cause the |
4048 | // select to be converted into a logical and/or. |
4049 | auto SimplifyOp = [&](Value *Op, bool SelectCondIsTrue) -> Value * { |
4050 | if (Value *Res = simplifyICmpInst(Predicate: Pred, LHS: Op, RHS, Q: SQ)) |
4051 | return Res; |
4052 | if (std::optional<bool> Impl = isImpliedCondition( |
4053 | LHS: SI->getCondition(), RHSPred: Pred, RHSOp0: Op, RHSOp1: RHS, DL, LHSIsTrue: SelectCondIsTrue)) |
4054 | return ConstantInt::get(Ty: I.getType(), V: *Impl); |
4055 | return nullptr; |
4056 | }; |
4057 | |
4058 | ConstantInt *CI = nullptr; |
4059 | Value *Op1 = SimplifyOp(SI->getOperand(i_nocapture: 1), true); |
4060 | if (Op1) |
4061 | CI = dyn_cast<ConstantInt>(Val: Op1); |
4062 | |
4063 | Value *Op2 = SimplifyOp(SI->getOperand(i_nocapture: 2), false); |
4064 | if (Op2) |
4065 | CI = dyn_cast<ConstantInt>(Val: Op2); |
4066 | |
4067 | // We only want to perform this transformation if it will not lead to |
4068 | // additional code. This is true if either both sides of the select |
4069 | // fold to a constant (in which case the icmp is replaced with a select |
4070 | // which will usually simplify) or this is the only user of the |
4071 | // select (in which case we are trading a select+icmp for a simpler |
4072 | // select+icmp) or all uses of the select can be replaced based on |
4073 | // dominance information ("Global cases"). |
4074 | bool Transform = false; |
4075 | if (Op1 && Op2) |
4076 | Transform = true; |
4077 | else if (Op1 || Op2) { |
4078 | // Local case |
4079 | if (SI->hasOneUse()) |
4080 | Transform = true; |
4081 | // Global cases |
4082 | else if (CI && !CI->isZero()) |
4083 | // When Op1 is constant try replacing select with second operand. |
4084 | // Otherwise Op2 is constant and try replacing select with first |
4085 | // operand. |
4086 | Transform = replacedSelectWithOperand(SI, Icmp: &I, SIOpd: Op1 ? 2 : 1); |
4087 | } |
4088 | if (Transform) { |
4089 | if (!Op1) |
4090 | Op1 = Builder.CreateICmp(P: Pred, LHS: SI->getOperand(i_nocapture: 1), RHS, Name: I.getName()); |
4091 | if (!Op2) |
4092 | Op2 = Builder.CreateICmp(P: Pred, LHS: SI->getOperand(i_nocapture: 2), RHS, Name: I.getName()); |
4093 | return SelectInst::Create(C: SI->getOperand(i_nocapture: 0), S1: Op1, S2: Op2); |
4094 | } |
4095 | |
4096 | return nullptr; |
4097 | } |
4098 | |
4099 | // Returns whether V is a Mask ((X + 1) & X == 0) or ~Mask (-Pow2OrZero) |
4100 | static bool isMaskOrZero(const Value *V, bool Not, const SimplifyQuery &Q, |
4101 | unsigned Depth = 0) { |
4102 | if (Not ? match(V, P: m_NegatedPower2OrZero()) : match(V, P: m_LowBitMaskOrZero())) |
4103 | return true; |
4104 | if (V->getType()->getScalarSizeInBits() == 1) |
4105 | return true; |
4106 | if (Depth++ >= MaxAnalysisRecursionDepth) |
4107 | return false; |
4108 | Value *X; |
4109 | const Instruction *I = dyn_cast<Instruction>(Val: V); |
4110 | if (!I) |
4111 | return false; |
4112 | switch (I->getOpcode()) { |
4113 | case Instruction::ZExt: |
4114 | // ZExt(Mask) is a Mask. |
4115 | return !Not && isMaskOrZero(V: I->getOperand(i: 0), Not, Q, Depth); |
4116 | case Instruction::SExt: |
4117 | // SExt(Mask) is a Mask. |
4118 | // SExt(~Mask) is a ~Mask. |
4119 | return isMaskOrZero(V: I->getOperand(i: 0), Not, Q, Depth); |
4120 | case Instruction::And: |
4121 | case Instruction::Or: |
4122 | // Mask0 | Mask1 is a Mask. |
4123 | // Mask0 & Mask1 is a Mask. |
4124 | // ~Mask0 | ~Mask1 is a ~Mask. |
4125 | // ~Mask0 & ~Mask1 is a ~Mask. |
4126 | return isMaskOrZero(V: I->getOperand(i: 1), Not, Q, Depth) && |
4127 | isMaskOrZero(V: I->getOperand(i: 0), Not, Q, Depth); |
4128 | case Instruction::Xor: |
4129 | if (match(V, P: m_Not(V: m_Value(V&: X)))) |
4130 | return isMaskOrZero(V: X, Not: !Not, Q, Depth); |
4131 | |
4132 | // (X ^ -X) is a ~Mask |
4133 | if (Not) |
4134 | return match(V, P: m_c_Xor(L: m_Value(V&: X), R: m_Neg(V: m_Deferred(V: X)))); |
4135 | // (X ^ (X - 1)) is a Mask |
4136 | else |
4137 | return match(V, P: m_c_Xor(L: m_Value(V&: X), R: m_Add(L: m_Deferred(V: X), R: m_AllOnes()))); |
4138 | case Instruction::Select: |
4139 | // c ? Mask0 : Mask1 is a Mask. |
4140 | return isMaskOrZero(V: I->getOperand(i: 1), Not, Q, Depth) && |
4141 | isMaskOrZero(V: I->getOperand(i: 2), Not, Q, Depth); |
4142 | case Instruction::Shl: |
4143 | // (~Mask) << X is a ~Mask. |
4144 | return Not && isMaskOrZero(V: I->getOperand(i: 0), Not, Q, Depth); |
4145 | case Instruction::LShr: |
4146 | // Mask >> X is a Mask. |
4147 | return !Not && isMaskOrZero(V: I->getOperand(i: 0), Not, Q, Depth); |
4148 | case Instruction::AShr: |
4149 | // Mask s>> X is a Mask. |
4150 | // ~Mask s>> X is a ~Mask. |
4151 | return isMaskOrZero(V: I->getOperand(i: 0), Not, Q, Depth); |
4152 | case Instruction::Add: |
4153 | // Pow2 - 1 is a Mask. |
4154 | if (!Not && match(V: I->getOperand(i: 1), P: m_AllOnes())) |
4155 | return isKnownToBeAPowerOfTwo(V: I->getOperand(i: 0), DL: Q.DL, /*OrZero*/ true, |
4156 | Depth, AC: Q.AC, CxtI: Q.CxtI, DT: Q.DT); |
4157 | break; |
4158 | case Instruction::Sub: |
4159 | // -Pow2 is a ~Mask. |
4160 | if (Not && match(V: I->getOperand(i: 0), P: m_Zero())) |
4161 | return isKnownToBeAPowerOfTwo(V: I->getOperand(i: 1), DL: Q.DL, /*OrZero*/ true, |
4162 | Depth, AC: Q.AC, CxtI: Q.CxtI, DT: Q.DT); |
4163 | break; |
4164 | case Instruction::Call: { |
4165 | if (auto *II = dyn_cast<IntrinsicInst>(Val: I)) { |
4166 | switch (II->getIntrinsicID()) { |
4167 | // min/max(Mask0, Mask1) is a Mask. |
4168 | // min/max(~Mask0, ~Mask1) is a ~Mask. |
4169 | case Intrinsic::umax: |
4170 | case Intrinsic::smax: |
4171 | case Intrinsic::umin: |
4172 | case Intrinsic::smin: |
4173 | return isMaskOrZero(V: II->getArgOperand(i: 1), Not, Q, Depth) && |
4174 | isMaskOrZero(V: II->getArgOperand(i: 0), Not, Q, Depth); |
4175 | |
4176 | // In the context of masks, bitreverse(Mask) == ~Mask |
4177 | case Intrinsic::bitreverse: |
4178 | return isMaskOrZero(V: II->getArgOperand(i: 0), Not: !Not, Q, Depth); |
4179 | default: |
4180 | break; |
4181 | } |
4182 | } |
4183 | break; |
4184 | } |
4185 | default: |
4186 | break; |
4187 | } |
4188 | return false; |
4189 | } |
4190 | |
4191 | /// Some comparisons can be simplified. |
4192 | /// In this case, we are looking for comparisons that look like |
4193 | /// a check for a lossy truncation. |
4194 | /// Folds: |
4195 | /// icmp SrcPred (x & Mask), x to icmp DstPred x, Mask |
4196 | /// icmp SrcPred (x & ~Mask), ~Mask to icmp DstPred x, ~Mask |
4197 | /// icmp eq/ne (x & ~Mask), 0 to icmp DstPred x, Mask |
4198 | /// icmp eq/ne (~x | Mask), -1 to icmp DstPred x, Mask |
4199 | /// Where Mask is some pattern that produces all-ones in low bits: |
4200 | /// (-1 >> y) |
4201 | /// ((-1 << y) >> y) <- non-canonical, has extra uses |
4202 | /// ~(-1 << y) |
4203 | /// ((1 << y) + (-1)) <- non-canonical, has extra uses |
4204 | /// The Mask can be a constant, too. |
4205 | /// For some predicates, the operands are commutative. |
4206 | /// For others, x can only be on a specific side. |
4207 | static Value *foldICmpWithLowBitMaskedVal(ICmpInst::Predicate Pred, Value *Op0, |
4208 | Value *Op1, const SimplifyQuery &Q, |
4209 | InstCombiner &IC) { |
4210 | |
4211 | ICmpInst::Predicate DstPred; |
4212 | switch (Pred) { |
4213 | case ICmpInst::Predicate::ICMP_EQ: |
4214 | // x & Mask == x |
4215 | // x & ~Mask == 0 |
4216 | // ~x | Mask == -1 |
4217 | // -> x u<= Mask |
4218 | // x & ~Mask == ~Mask |
4219 | // -> ~Mask u<= x |
4220 | DstPred = ICmpInst::Predicate::ICMP_ULE; |
4221 | break; |
4222 | case ICmpInst::Predicate::ICMP_NE: |
4223 | // x & Mask != x |
4224 | // x & ~Mask != 0 |
4225 | // ~x | Mask != -1 |
4226 | // -> x u> Mask |
4227 | // x & ~Mask != ~Mask |
4228 | // -> ~Mask u> x |
4229 | DstPred = ICmpInst::Predicate::ICMP_UGT; |
4230 | break; |
4231 | case ICmpInst::Predicate::ICMP_ULT: |
4232 | // x & Mask u< x |
4233 | // -> x u> Mask |
4234 | // x & ~Mask u< ~Mask |
4235 | // -> ~Mask u> x |
4236 | DstPred = ICmpInst::Predicate::ICMP_UGT; |
4237 | break; |
4238 | case ICmpInst::Predicate::ICMP_UGE: |
4239 | // x & Mask u>= x |
4240 | // -> x u<= Mask |
4241 | // x & ~Mask u>= ~Mask |
4242 | // -> ~Mask u<= x |
4243 | DstPred = ICmpInst::Predicate::ICMP_ULE; |
4244 | break; |
4245 | case ICmpInst::Predicate::ICMP_SLT: |
4246 | // x & Mask s< x [iff Mask s>= 0] |
4247 | // -> x s> Mask |
4248 | // x & ~Mask s< ~Mask [iff ~Mask != 0] |
4249 | // -> ~Mask s> x |
4250 | DstPred = ICmpInst::Predicate::ICMP_SGT; |
4251 | break; |
4252 | case ICmpInst::Predicate::ICMP_SGE: |
4253 | // x & Mask s>= x [iff Mask s>= 0] |
4254 | // -> x s<= Mask |
4255 | // x & ~Mask s>= ~Mask [iff ~Mask != 0] |
4256 | // -> ~Mask s<= x |
4257 | DstPred = ICmpInst::Predicate::ICMP_SLE; |
4258 | break; |
4259 | default: |
4260 | // We don't support sgt,sle |
4261 | // ult/ugt are simplified to true/false respectively. |
4262 | return nullptr; |
4263 | } |
4264 | |
4265 | Value *X, *M; |
4266 | // Put search code in lambda for early positive returns. |
4267 | auto IsLowBitMask = [&]() { |
4268 | if (match(V: Op0, P: m_c_And(L: m_Specific(V: Op1), R: m_Value(V&: M)))) { |
4269 | X = Op1; |
4270 | // Look for: x & Mask pred x |
4271 | if (isMaskOrZero(V: M, /*Not=*/false, Q)) { |
4272 | return !ICmpInst::isSigned(predicate: Pred) || |
4273 | (match(V: M, P: m_NonNegative()) || isKnownNonNegative(V: M, SQ: Q)); |
4274 | } |
4275 | |
4276 | // Look for: x & ~Mask pred ~Mask |
4277 | if (isMaskOrZero(V: X, /*Not=*/true, Q)) { |
4278 | return !ICmpInst::isSigned(predicate: Pred) || isKnownNonZero(V: X, Q); |
4279 | } |
4280 | return false; |
4281 | } |
4282 | if (ICmpInst::isEquality(P: Pred) && match(V: Op1, P: m_AllOnes()) && |
4283 | match(V: Op0, P: m_OneUse(SubPattern: m_Or(L: m_Value(V&: X), R: m_Value(V&: M))))) { |
4284 | |
4285 | auto Check = [&]() { |
4286 | // Look for: ~x | Mask == -1 |
4287 | if (isMaskOrZero(V: M, /*Not=*/false, Q)) { |
4288 | if (Value *NotX = |
4289 | IC.getFreelyInverted(V: X, WillInvertAllUses: X->hasOneUse(), Builder: &IC.Builder)) { |
4290 | X = NotX; |
4291 | return true; |
4292 | } |
4293 | } |
4294 | return false; |
4295 | }; |
4296 | if (Check()) |
4297 | return true; |
4298 | std::swap(a&: X, b&: M); |
4299 | return Check(); |
4300 | } |
4301 | if (ICmpInst::isEquality(P: Pred) && match(V: Op1, P: m_Zero()) && |
4302 | match(V: Op0, P: m_OneUse(SubPattern: m_And(L: m_Value(V&: X), R: m_Value(V&: M))))) { |
4303 | auto Check = [&]() { |
4304 | // Look for: x & ~Mask == 0 |
4305 | if (isMaskOrZero(V: M, /*Not=*/true, Q)) { |
4306 | if (Value *NotM = |
4307 | IC.getFreelyInverted(V: M, WillInvertAllUses: M->hasOneUse(), Builder: &IC.Builder)) { |
4308 | M = NotM; |
4309 | return true; |
4310 | } |
4311 | } |
4312 | return false; |
4313 | }; |
4314 | if (Check()) |
4315 | return true; |
4316 | std::swap(a&: X, b&: M); |
4317 | return Check(); |
4318 | } |
4319 | return false; |
4320 | }; |
4321 | |
4322 | if (!IsLowBitMask()) |
4323 | return nullptr; |
4324 | |
4325 | return IC.Builder.CreateICmp(P: DstPred, LHS: X, RHS: M); |
4326 | } |
4327 | |
4328 | /// Some comparisons can be simplified. |
4329 | /// In this case, we are looking for comparisons that look like |
4330 | /// a check for a lossy signed truncation. |
4331 | /// Folds: (MaskedBits is a constant.) |
4332 | /// ((%x << MaskedBits) a>> MaskedBits) SrcPred %x |
4333 | /// Into: |
4334 | /// (add %x, (1 << (KeptBits-1))) DstPred (1 << KeptBits) |
4335 | /// Where KeptBits = bitwidth(%x) - MaskedBits |
4336 | static Value * |
4337 | foldICmpWithTruncSignExtendedVal(ICmpInst &I, |
4338 | InstCombiner::BuilderTy &Builder) { |
4339 | ICmpInst::Predicate SrcPred; |
4340 | Value *X; |
4341 | const APInt *C0, *C1; // FIXME: non-splats, potentially with undef. |
4342 | // We are ok with 'shl' having multiple uses, but 'ashr' must be one-use. |
4343 | if (!match(V: &I, P: m_c_ICmp(Pred&: SrcPred, |
4344 | L: m_OneUse(SubPattern: m_AShr(L: m_Shl(L: m_Value(V&: X), R: m_APInt(Res&: C0)), |
4345 | R: m_APInt(Res&: C1))), |
4346 | R: m_Deferred(V: X)))) |
4347 | return nullptr; |
4348 | |
4349 | // Potential handling of non-splats: for each element: |
4350 | // * if both are undef, replace with constant 0. |
4351 | // Because (1<<0) is OK and is 1, and ((1<<0)>>1) is also OK and is 0. |
4352 | // * if both are not undef, and are different, bailout. |
4353 | // * else, only one is undef, then pick the non-undef one. |
4354 | |
4355 | // The shift amount must be equal. |
4356 | if (*C0 != *C1) |
4357 | return nullptr; |
4358 | const APInt &MaskedBits = *C0; |
4359 | assert(MaskedBits != 0 && "shift by zero should be folded away already." ); |
4360 | |
4361 | ICmpInst::Predicate DstPred; |
4362 | switch (SrcPred) { |
4363 | case ICmpInst::Predicate::ICMP_EQ: |
4364 | // ((%x << MaskedBits) a>> MaskedBits) == %x |
4365 | // => |
4366 | // (add %x, (1 << (KeptBits-1))) u< (1 << KeptBits) |
4367 | DstPred = ICmpInst::Predicate::ICMP_ULT; |
4368 | break; |
4369 | case ICmpInst::Predicate::ICMP_NE: |
4370 | // ((%x << MaskedBits) a>> MaskedBits) != %x |
4371 | // => |
4372 | // (add %x, (1 << (KeptBits-1))) u>= (1 << KeptBits) |
4373 | DstPred = ICmpInst::Predicate::ICMP_UGE; |
4374 | break; |
4375 | // FIXME: are more folds possible? |
4376 | default: |
4377 | return nullptr; |
4378 | } |
4379 | |
4380 | auto *XType = X->getType(); |
4381 | const unsigned XBitWidth = XType->getScalarSizeInBits(); |
4382 | const APInt BitWidth = APInt(XBitWidth, XBitWidth); |
4383 | assert(BitWidth.ugt(MaskedBits) && "shifts should leave some bits untouched" ); |
4384 | |
4385 | // KeptBits = bitwidth(%x) - MaskedBits |
4386 | const APInt KeptBits = BitWidth - MaskedBits; |
4387 | assert(KeptBits.ugt(0) && KeptBits.ult(BitWidth) && "unreachable" ); |
4388 | // ICmpCst = (1 << KeptBits) |
4389 | const APInt ICmpCst = APInt(XBitWidth, 1).shl(ShiftAmt: KeptBits); |
4390 | assert(ICmpCst.isPowerOf2()); |
4391 | // AddCst = (1 << (KeptBits-1)) |
4392 | const APInt AddCst = ICmpCst.lshr(shiftAmt: 1); |
4393 | assert(AddCst.ult(ICmpCst) && AddCst.isPowerOf2()); |
4394 | |
4395 | // T0 = add %x, AddCst |
4396 | Value *T0 = Builder.CreateAdd(LHS: X, RHS: ConstantInt::get(Ty: XType, V: AddCst)); |
4397 | // T1 = T0 DstPred ICmpCst |
4398 | Value *T1 = Builder.CreateICmp(P: DstPred, LHS: T0, RHS: ConstantInt::get(Ty: XType, V: ICmpCst)); |
4399 | |
4400 | return T1; |
4401 | } |
4402 | |
4403 | // Given pattern: |
4404 | // icmp eq/ne (and ((x shift Q), (y oppositeshift K))), 0 |
4405 | // we should move shifts to the same hand of 'and', i.e. rewrite as |
4406 | // icmp eq/ne (and (x shift (Q+K)), y), 0 iff (Q+K) u< bitwidth(x) |
4407 | // We are only interested in opposite logical shifts here. |
4408 | // One of the shifts can be truncated. |
4409 | // If we can, we want to end up creating 'lshr' shift. |
4410 | static Value * |
4411 | foldShiftIntoShiftInAnotherHandOfAndInICmp(ICmpInst &I, const SimplifyQuery SQ, |
4412 | InstCombiner::BuilderTy &Builder) { |
4413 | if (!I.isEquality() || !match(V: I.getOperand(i_nocapture: 1), P: m_Zero()) || |
4414 | !I.getOperand(i_nocapture: 0)->hasOneUse()) |
4415 | return nullptr; |
4416 | |
4417 | auto m_AnyLogicalShift = m_LogicalShift(L: m_Value(), R: m_Value()); |
4418 | |
4419 | // Look for an 'and' of two logical shifts, one of which may be truncated. |
4420 | // We use m_TruncOrSelf() on the RHS to correctly handle commutative case. |
4421 | Instruction *XShift, *MaybeTruncation, *YShift; |
4422 | if (!match( |
4423 | V: I.getOperand(i_nocapture: 0), |
4424 | P: m_c_And(L: m_CombineAnd(L: m_AnyLogicalShift, R: m_Instruction(I&: XShift)), |
4425 | R: m_CombineAnd(L: m_TruncOrSelf(Op: m_CombineAnd( |
4426 | L: m_AnyLogicalShift, R: m_Instruction(I&: YShift))), |
4427 | R: m_Instruction(I&: MaybeTruncation))))) |
4428 | return nullptr; |
4429 | |
4430 | // We potentially looked past 'trunc', but only when matching YShift, |
4431 | // therefore YShift must have the widest type. |
4432 | Instruction *WidestShift = YShift; |
4433 | // Therefore XShift must have the shallowest type. |
4434 | // Or they both have identical types if there was no truncation. |
4435 | Instruction *NarrowestShift = XShift; |
4436 | |
4437 | Type *WidestTy = WidestShift->getType(); |
4438 | Type *NarrowestTy = NarrowestShift->getType(); |
4439 | assert(NarrowestTy == I.getOperand(0)->getType() && |
4440 | "We did not look past any shifts while matching XShift though." ); |
4441 | bool HadTrunc = WidestTy != I.getOperand(i_nocapture: 0)->getType(); |
4442 | |
4443 | // If YShift is a 'lshr', swap the shifts around. |
4444 | if (match(V: YShift, P: m_LShr(L: m_Value(), R: m_Value()))) |
4445 | std::swap(a&: XShift, b&: YShift); |
4446 | |
4447 | // The shifts must be in opposite directions. |
4448 | auto XShiftOpcode = XShift->getOpcode(); |
4449 | if (XShiftOpcode == YShift->getOpcode()) |
4450 | return nullptr; // Do not care about same-direction shifts here. |
4451 | |
4452 | Value *X, *XShAmt, *Y, *YShAmt; |
4453 | match(V: XShift, P: m_BinOp(L: m_Value(V&: X), R: m_ZExtOrSelf(Op: m_Value(V&: XShAmt)))); |
4454 | match(V: YShift, P: m_BinOp(L: m_Value(V&: Y), R: m_ZExtOrSelf(Op: m_Value(V&: YShAmt)))); |
4455 | |
4456 | // If one of the values being shifted is a constant, then we will end with |
4457 | // and+icmp, and [zext+]shift instrs will be constant-folded. If they are not, |
4458 | // however, we will need to ensure that we won't increase instruction count. |
4459 | if (!isa<Constant>(Val: X) && !isa<Constant>(Val: Y)) { |
4460 | // At least one of the hands of the 'and' should be one-use shift. |
4461 | if (!match(V: I.getOperand(i_nocapture: 0), |
4462 | P: m_c_And(L: m_OneUse(SubPattern: m_AnyLogicalShift), R: m_Value()))) |
4463 | return nullptr; |
4464 | if (HadTrunc) { |
4465 | // Due to the 'trunc', we will need to widen X. For that either the old |
4466 | // 'trunc' or the shift amt in the non-truncated shift should be one-use. |
4467 | if (!MaybeTruncation->hasOneUse() && |
4468 | !NarrowestShift->getOperand(i: 1)->hasOneUse()) |
4469 | return nullptr; |
4470 | } |
4471 | } |
4472 | |
4473 | // We have two shift amounts from two different shifts. The types of those |
4474 | // shift amounts may not match. If that's the case let's bailout now. |
4475 | if (XShAmt->getType() != YShAmt->getType()) |
4476 | return nullptr; |
4477 | |
4478 | // As input, we have the following pattern: |
4479 | // icmp eq/ne (and ((x shift Q), (y oppositeshift K))), 0 |
4480 | // We want to rewrite that as: |
4481 | // icmp eq/ne (and (x shift (Q+K)), y), 0 iff (Q+K) u< bitwidth(x) |
4482 | // While we know that originally (Q+K) would not overflow |
4483 | // (because 2 * (N-1) u<= iN -1), we have looked past extensions of |
4484 | // shift amounts. so it may now overflow in smaller bitwidth. |
4485 | // To ensure that does not happen, we need to ensure that the total maximal |
4486 | // shift amount is still representable in that smaller bit width. |
4487 | unsigned MaximalPossibleTotalShiftAmount = |
4488 | (WidestTy->getScalarSizeInBits() - 1) + |
4489 | (NarrowestTy->getScalarSizeInBits() - 1); |
4490 | APInt MaximalRepresentableShiftAmount = |
4491 | APInt::getAllOnes(numBits: XShAmt->getType()->getScalarSizeInBits()); |
4492 | if (MaximalRepresentableShiftAmount.ult(RHS: MaximalPossibleTotalShiftAmount)) |
4493 | return nullptr; |
4494 | |
4495 | // Can we fold (XShAmt+YShAmt) ? |
4496 | auto *NewShAmt = dyn_cast_or_null<Constant>( |
4497 | Val: simplifyAddInst(LHS: XShAmt, RHS: YShAmt, /*isNSW=*/IsNSW: false, |
4498 | /*isNUW=*/IsNUW: false, Q: SQ.getWithInstruction(I: &I))); |
4499 | if (!NewShAmt) |
4500 | return nullptr; |
4501 | if (NewShAmt->getType() != WidestTy) { |
4502 | NewShAmt = |
4503 | ConstantFoldCastOperand(Opcode: Instruction::ZExt, C: NewShAmt, DestTy: WidestTy, DL: SQ.DL); |
4504 | if (!NewShAmt) |
4505 | return nullptr; |
4506 | } |
4507 | unsigned WidestBitWidth = WidestTy->getScalarSizeInBits(); |
4508 | |
4509 | // Is the new shift amount smaller than the bit width? |
4510 | // FIXME: could also rely on ConstantRange. |
4511 | if (!match(V: NewShAmt, |
4512 | P: m_SpecificInt_ICMP(Predicate: ICmpInst::Predicate::ICMP_ULT, |
4513 | Threshold: APInt(WidestBitWidth, WidestBitWidth)))) |
4514 | return nullptr; |
4515 | |
4516 | // An extra legality check is needed if we had trunc-of-lshr. |
4517 | if (HadTrunc && match(V: WidestShift, P: m_LShr(L: m_Value(), R: m_Value()))) { |
4518 | auto CanFold = [NewShAmt, WidestBitWidth, NarrowestShift, SQ, |
4519 | WidestShift]() { |
4520 | // It isn't obvious whether it's worth it to analyze non-constants here. |
4521 | // Also, let's basically give up on non-splat cases, pessimizing vectors. |
4522 | // If *any* of these preconditions matches we can perform the fold. |
4523 | Constant *NewShAmtSplat = NewShAmt->getType()->isVectorTy() |
4524 | ? NewShAmt->getSplatValue() |
4525 | : NewShAmt; |
4526 | // If it's edge-case shift (by 0 or by WidestBitWidth-1) we can fold. |
4527 | if (NewShAmtSplat && |
4528 | (NewShAmtSplat->isNullValue() || |
4529 | NewShAmtSplat->getUniqueInteger() == WidestBitWidth - 1)) |
4530 | return true; |
4531 | // We consider *min* leading zeros so a single outlier |
4532 | // blocks the transform as opposed to allowing it. |
4533 | if (auto *C = dyn_cast<Constant>(Val: NarrowestShift->getOperand(i: 0))) { |
4534 | KnownBits Known = computeKnownBits(V: C, DL: SQ.DL); |
4535 | unsigned MinLeadZero = Known.countMinLeadingZeros(); |
4536 | // If the value being shifted has at most lowest bit set we can fold. |
4537 | unsigned MaxActiveBits = Known.getBitWidth() - MinLeadZero; |
4538 | if (MaxActiveBits <= 1) |
4539 | return true; |
4540 | // Precondition: NewShAmt u<= countLeadingZeros(C) |
4541 | if (NewShAmtSplat && NewShAmtSplat->getUniqueInteger().ule(RHS: MinLeadZero)) |
4542 | return true; |
4543 | } |
4544 | if (auto *C = dyn_cast<Constant>(Val: WidestShift->getOperand(i: 0))) { |
4545 | KnownBits Known = computeKnownBits(V: C, DL: SQ.DL); |
4546 | unsigned MinLeadZero = Known.countMinLeadingZeros(); |
4547 | // If the value being shifted has at most lowest bit set we can fold. |
4548 | unsigned MaxActiveBits = Known.getBitWidth() - MinLeadZero; |
4549 | if (MaxActiveBits <= 1) |
4550 | return true; |
4551 | // Precondition: ((WidestBitWidth-1)-NewShAmt) u<= countLeadingZeros(C) |
4552 | if (NewShAmtSplat) { |
4553 | APInt AdjNewShAmt = |
4554 | (WidestBitWidth - 1) - NewShAmtSplat->getUniqueInteger(); |
4555 | if (AdjNewShAmt.ule(RHS: MinLeadZero)) |
4556 | return true; |
4557 | } |
4558 | } |
4559 | return false; // Can't tell if it's ok. |
4560 | }; |
4561 | if (!CanFold()) |
4562 | return nullptr; |
4563 | } |
4564 | |
4565 | // All good, we can do this fold. |
4566 | X = Builder.CreateZExt(V: X, DestTy: WidestTy); |
4567 | Y = Builder.CreateZExt(V: Y, DestTy: WidestTy); |
4568 | // The shift is the same that was for X. |
4569 | Value *T0 = XShiftOpcode == Instruction::BinaryOps::LShr |
4570 | ? Builder.CreateLShr(LHS: X, RHS: NewShAmt) |
4571 | : Builder.CreateShl(LHS: X, RHS: NewShAmt); |
4572 | Value *T1 = Builder.CreateAnd(LHS: T0, RHS: Y); |
4573 | return Builder.CreateICmp(P: I.getPredicate(), LHS: T1, |
4574 | RHS: Constant::getNullValue(Ty: WidestTy)); |
4575 | } |
4576 | |
4577 | /// Fold |
4578 | /// (-1 u/ x) u< y |
4579 | /// ((x * y) ?/ x) != y |
4580 | /// to |
4581 | /// @llvm.?mul.with.overflow(x, y) plus extraction of overflow bit |
4582 | /// Note that the comparison is commutative, while inverted (u>=, ==) predicate |
4583 | /// will mean that we are looking for the opposite answer. |
4584 | Value *InstCombinerImpl::foldMultiplicationOverflowCheck(ICmpInst &I) { |
4585 | ICmpInst::Predicate Pred; |
4586 | Value *X, *Y; |
4587 | Instruction *Mul; |
4588 | Instruction *Div; |
4589 | bool NeedNegation; |
4590 | // Look for: (-1 u/ x) u</u>= y |
4591 | if (!I.isEquality() && |
4592 | match(V: &I, P: m_c_ICmp(Pred, |
4593 | L: m_CombineAnd(L: m_OneUse(SubPattern: m_UDiv(L: m_AllOnes(), R: m_Value(V&: X))), |
4594 | R: m_Instruction(I&: Div)), |
4595 | R: m_Value(V&: Y)))) { |
4596 | Mul = nullptr; |
4597 | |
4598 | // Are we checking that overflow does not happen, or does happen? |
4599 | switch (Pred) { |
4600 | case ICmpInst::Predicate::ICMP_ULT: |
4601 | NeedNegation = false; |
4602 | break; // OK |
4603 | case ICmpInst::Predicate::ICMP_UGE: |
4604 | NeedNegation = true; |
4605 | break; // OK |
4606 | default: |
4607 | return nullptr; // Wrong predicate. |
4608 | } |
4609 | } else // Look for: ((x * y) / x) !=/== y |
4610 | if (I.isEquality() && |
4611 | match(V: &I, |
4612 | P: m_c_ICmp(Pred, L: m_Value(V&: Y), |
4613 | R: m_CombineAnd( |
4614 | L: m_OneUse(SubPattern: m_IDiv(L: m_CombineAnd(L: m_c_Mul(L: m_Deferred(V: Y), |
4615 | R: m_Value(V&: X)), |
4616 | R: m_Instruction(I&: Mul)), |
4617 | R: m_Deferred(V: X))), |
4618 | R: m_Instruction(I&: Div))))) { |
4619 | NeedNegation = Pred == ICmpInst::Predicate::ICMP_EQ; |
4620 | } else |
4621 | return nullptr; |
4622 | |
4623 | BuilderTy::InsertPointGuard Guard(Builder); |
4624 | // If the pattern included (x * y), we'll want to insert new instructions |
4625 | // right before that original multiplication so that we can replace it. |
4626 | bool MulHadOtherUses = Mul && !Mul->hasOneUse(); |
4627 | if (MulHadOtherUses) |
4628 | Builder.SetInsertPoint(Mul); |
4629 | |
4630 | Function *F = Intrinsic::getDeclaration(M: I.getModule(), |
4631 | id: Div->getOpcode() == Instruction::UDiv |
4632 | ? Intrinsic::umul_with_overflow |
4633 | : Intrinsic::smul_with_overflow, |
4634 | Tys: X->getType()); |
4635 | CallInst *Call = Builder.CreateCall(Callee: F, Args: {X, Y}, Name: "mul" ); |
4636 | |
4637 | // If the multiplication was used elsewhere, to ensure that we don't leave |
4638 | // "duplicate" instructions, replace uses of that original multiplication |
4639 | // with the multiplication result from the with.overflow intrinsic. |
4640 | if (MulHadOtherUses) |
4641 | replaceInstUsesWith(I&: *Mul, V: Builder.CreateExtractValue(Agg: Call, Idxs: 0, Name: "mul.val" )); |
4642 | |
4643 | Value *Res = Builder.CreateExtractValue(Agg: Call, Idxs: 1, Name: "mul.ov" ); |
4644 | if (NeedNegation) // This technically increases instruction count. |
4645 | Res = Builder.CreateNot(V: Res, Name: "mul.not.ov" ); |
4646 | |
4647 | // If we replaced the mul, erase it. Do this after all uses of Builder, |
4648 | // as the mul is used as insertion point. |
4649 | if (MulHadOtherUses) |
4650 | eraseInstFromFunction(I&: *Mul); |
4651 | |
4652 | return Res; |
4653 | } |
4654 | |
4655 | static Instruction *foldICmpXNegX(ICmpInst &I, |
4656 | InstCombiner::BuilderTy &Builder) { |
4657 | CmpInst::Predicate Pred; |
4658 | Value *X; |
4659 | if (match(V: &I, P: m_c_ICmp(Pred, L: m_NSWNeg(V: m_Value(V&: X)), R: m_Deferred(V: X)))) { |
4660 | |
4661 | if (ICmpInst::isSigned(predicate: Pred)) |
4662 | Pred = ICmpInst::getSwappedPredicate(pred: Pred); |
4663 | else if (ICmpInst::isUnsigned(predicate: Pred)) |
4664 | Pred = ICmpInst::getSignedPredicate(pred: Pred); |
4665 | // else for equality-comparisons just keep the predicate. |
4666 | |
4667 | return ICmpInst::Create(Op: Instruction::ICmp, Pred, S1: X, |
4668 | S2: Constant::getNullValue(Ty: X->getType()), Name: I.getName()); |
4669 | } |
4670 | |
4671 | // A value is not equal to its negation unless that value is 0 or |
4672 | // MinSignedValue, ie: a != -a --> (a & MaxSignedVal) != 0 |
4673 | if (match(V: &I, P: m_c_ICmp(Pred, L: m_OneUse(SubPattern: m_Neg(V: m_Value(V&: X))), R: m_Deferred(V: X))) && |
4674 | ICmpInst::isEquality(P: Pred)) { |
4675 | Type *Ty = X->getType(); |
4676 | uint32_t BitWidth = Ty->getScalarSizeInBits(); |
4677 | Constant *MaxSignedVal = |
4678 | ConstantInt::get(Ty, V: APInt::getSignedMaxValue(numBits: BitWidth)); |
4679 | Value *And = Builder.CreateAnd(LHS: X, RHS: MaxSignedVal); |
4680 | Constant *Zero = Constant::getNullValue(Ty); |
4681 | return CmpInst::Create(Op: Instruction::ICmp, Pred, S1: And, S2: Zero); |
4682 | } |
4683 | |
4684 | return nullptr; |
4685 | } |
4686 | |
4687 | static Instruction *foldICmpAndXX(ICmpInst &I, const SimplifyQuery &Q, |
4688 | InstCombinerImpl &IC) { |
4689 | Value *Op0 = I.getOperand(i_nocapture: 0), *Op1 = I.getOperand(i_nocapture: 1), *A; |
4690 | // Normalize and operand as operand 0. |
4691 | CmpInst::Predicate Pred = I.getPredicate(); |
4692 | if (match(V: Op1, P: m_c_And(L: m_Specific(V: Op0), R: m_Value()))) { |
4693 | std::swap(a&: Op0, b&: Op1); |
4694 | Pred = ICmpInst::getSwappedPredicate(pred: Pred); |
4695 | } |
4696 | |
4697 | if (!match(V: Op0, P: m_c_And(L: m_Specific(V: Op1), R: m_Value(V&: A)))) |
4698 | return nullptr; |
4699 | |
4700 | // (icmp (X & Y) u< X --> (X & Y) != X |
4701 | if (Pred == ICmpInst::ICMP_ULT) |
4702 | return new ICmpInst(ICmpInst::ICMP_NE, Op0, Op1); |
4703 | |
4704 | // (icmp (X & Y) u>= X --> (X & Y) == X |
4705 | if (Pred == ICmpInst::ICMP_UGE) |
4706 | return new ICmpInst(ICmpInst::ICMP_EQ, Op0, Op1); |
4707 | |
4708 | return nullptr; |
4709 | } |
4710 | |
4711 | static Instruction *foldICmpOrXX(ICmpInst &I, const SimplifyQuery &Q, |
4712 | InstCombinerImpl &IC) { |
4713 | Value *Op0 = I.getOperand(i_nocapture: 0), *Op1 = I.getOperand(i_nocapture: 1), *A; |
4714 | |
4715 | // Normalize or operand as operand 0. |
4716 | CmpInst::Predicate Pred = I.getPredicate(); |
4717 | if (match(V: Op1, P: m_c_Or(L: m_Specific(V: Op0), R: m_Value(V&: A)))) { |
4718 | std::swap(a&: Op0, b&: Op1); |
4719 | Pred = ICmpInst::getSwappedPredicate(pred: Pred); |
4720 | } else if (!match(V: Op0, P: m_c_Or(L: m_Specific(V: Op1), R: m_Value(V&: A)))) { |
4721 | return nullptr; |
4722 | } |
4723 | |
4724 | // icmp (X | Y) u<= X --> (X | Y) == X |
4725 | if (Pred == ICmpInst::ICMP_ULE) |
4726 | return new ICmpInst(ICmpInst::ICMP_EQ, Op0, Op1); |
4727 | |
4728 | // icmp (X | Y) u> X --> (X | Y) != X |
4729 | if (Pred == ICmpInst::ICMP_UGT) |
4730 | return new ICmpInst(ICmpInst::ICMP_NE, Op0, Op1); |
4731 | |
4732 | if (ICmpInst::isEquality(P: Pred) && Op0->hasOneUse()) { |
4733 | // icmp (X | Y) eq/ne Y --> (X & ~Y) eq/ne 0 if Y is freely invertible |
4734 | if (Value *NotOp1 = |
4735 | IC.getFreelyInverted(V: Op1, WillInvertAllUses: Op1->hasOneUse(), Builder: &IC.Builder)) |
4736 | return new ICmpInst(Pred, IC.Builder.CreateAnd(LHS: A, RHS: NotOp1), |
4737 | Constant::getNullValue(Ty: Op1->getType())); |
4738 | // icmp (X | Y) eq/ne Y --> (~X | Y) eq/ne -1 if X is freely invertible. |
4739 | if (Value *NotA = IC.getFreelyInverted(V: A, WillInvertAllUses: A->hasOneUse(), Builder: &IC.Builder)) |
4740 | return new ICmpInst(Pred, IC.Builder.CreateOr(LHS: Op1, RHS: NotA), |
4741 | Constant::getAllOnesValue(Ty: Op1->getType())); |
4742 | } |
4743 | return nullptr; |
4744 | } |
4745 | |
4746 | static Instruction *foldICmpXorXX(ICmpInst &I, const SimplifyQuery &Q, |
4747 | InstCombinerImpl &IC) { |
4748 | Value *Op0 = I.getOperand(i_nocapture: 0), *Op1 = I.getOperand(i_nocapture: 1), *A; |
4749 | // Normalize xor operand as operand 0. |
4750 | CmpInst::Predicate Pred = I.getPredicate(); |
4751 | if (match(V: Op1, P: m_c_Xor(L: m_Specific(V: Op0), R: m_Value()))) { |
4752 | std::swap(a&: Op0, b&: Op1); |
4753 | Pred = ICmpInst::getSwappedPredicate(pred: Pred); |
4754 | } |
4755 | if (!match(V: Op0, P: m_c_Xor(L: m_Specific(V: Op1), R: m_Value(V&: A)))) |
4756 | return nullptr; |
4757 | |
4758 | // icmp (X ^ Y_NonZero) u>= X --> icmp (X ^ Y_NonZero) u> X |
4759 | // icmp (X ^ Y_NonZero) u<= X --> icmp (X ^ Y_NonZero) u< X |
4760 | // icmp (X ^ Y_NonZero) s>= X --> icmp (X ^ Y_NonZero) s> X |
4761 | // icmp (X ^ Y_NonZero) s<= X --> icmp (X ^ Y_NonZero) s< X |
4762 | CmpInst::Predicate PredOut = CmpInst::getStrictPredicate(pred: Pred); |
4763 | if (PredOut != Pred && isKnownNonZero(V: A, Q)) |
4764 | return new ICmpInst(PredOut, Op0, Op1); |
4765 | |
4766 | return nullptr; |
4767 | } |
4768 | |
4769 | /// Try to fold icmp (binop), X or icmp X, (binop). |
4770 | /// TODO: A large part of this logic is duplicated in InstSimplify's |
4771 | /// simplifyICmpWithBinOp(). We should be able to share that and avoid the code |
4772 | /// duplication. |
4773 | Instruction *InstCombinerImpl::foldICmpBinOp(ICmpInst &I, |
4774 | const SimplifyQuery &SQ) { |
4775 | const SimplifyQuery Q = SQ.getWithInstruction(I: &I); |
4776 | Value *Op0 = I.getOperand(i_nocapture: 0), *Op1 = I.getOperand(i_nocapture: 1); |
4777 | |
4778 | // Special logic for binary operators. |
4779 | BinaryOperator *BO0 = dyn_cast<BinaryOperator>(Val: Op0); |
4780 | BinaryOperator *BO1 = dyn_cast<BinaryOperator>(Val: Op1); |
4781 | if (!BO0 && !BO1) |
4782 | return nullptr; |
4783 | |
4784 | if (Instruction *NewICmp = foldICmpXNegX(I, Builder)) |
4785 | return NewICmp; |
4786 | |
4787 | const CmpInst::Predicate Pred = I.getPredicate(); |
4788 | Value *X; |
4789 | |
4790 | // Convert add-with-unsigned-overflow comparisons into a 'not' with compare. |
4791 | // (Op1 + X) u</u>= Op1 --> ~Op1 u</u>= X |
4792 | if (match(V: Op0, P: m_OneUse(SubPattern: m_c_Add(L: m_Specific(V: Op1), R: m_Value(V&: X)))) && |
4793 | (Pred == ICmpInst::ICMP_ULT || Pred == ICmpInst::ICMP_UGE)) |
4794 | return new ICmpInst(Pred, Builder.CreateNot(V: Op1), X); |
4795 | // Op0 u>/u<= (Op0 + X) --> X u>/u<= ~Op0 |
4796 | if (match(V: Op1, P: m_OneUse(SubPattern: m_c_Add(L: m_Specific(V: Op0), R: m_Value(V&: X)))) && |
4797 | (Pred == ICmpInst::ICMP_UGT || Pred == ICmpInst::ICMP_ULE)) |
4798 | return new ICmpInst(Pred, X, Builder.CreateNot(V: Op0)); |
4799 | |
4800 | { |
4801 | // (Op1 + X) + C u</u>= Op1 --> ~C - X u</u>= Op1 |
4802 | Constant *C; |
4803 | if (match(V: Op0, P: m_OneUse(SubPattern: m_Add(L: m_c_Add(L: m_Specific(V: Op1), R: m_Value(V&: X)), |
4804 | R: m_ImmConstant(C)))) && |
4805 | (Pred == ICmpInst::ICMP_ULT || Pred == ICmpInst::ICMP_UGE)) { |
4806 | Constant *C2 = ConstantExpr::getNot(C); |
4807 | return new ICmpInst(Pred, Builder.CreateSub(LHS: C2, RHS: X), Op1); |
4808 | } |
4809 | // Op0 u>/u<= (Op0 + X) + C --> Op0 u>/u<= ~C - X |
4810 | if (match(V: Op1, P: m_OneUse(SubPattern: m_Add(L: m_c_Add(L: m_Specific(V: Op0), R: m_Value(V&: X)), |
4811 | R: m_ImmConstant(C)))) && |
4812 | (Pred == ICmpInst::ICMP_UGT || Pred == ICmpInst::ICMP_ULE)) { |
4813 | Constant *C2 = ConstantExpr::getNot(C); |
4814 | return new ICmpInst(Pred, Op0, Builder.CreateSub(LHS: C2, RHS: X)); |
4815 | } |
4816 | } |
4817 | |
4818 | { |
4819 | // Similar to above: an unsigned overflow comparison may use offset + mask: |
4820 | // ((Op1 + C) & C) u< Op1 --> Op1 != 0 |
4821 | // ((Op1 + C) & C) u>= Op1 --> Op1 == 0 |
4822 | // Op0 u> ((Op0 + C) & C) --> Op0 != 0 |
4823 | // Op0 u<= ((Op0 + C) & C) --> Op0 == 0 |
4824 | BinaryOperator *BO; |
4825 | const APInt *C; |
4826 | if ((Pred == ICmpInst::ICMP_ULT || Pred == ICmpInst::ICMP_UGE) && |
4827 | match(V: Op0, P: m_And(L: m_BinOp(I&: BO), R: m_LowBitMask(V&: C))) && |
4828 | match(V: BO, P: m_Add(L: m_Specific(V: Op1), R: m_SpecificIntAllowPoison(V: *C)))) { |
4829 | CmpInst::Predicate NewPred = |
4830 | Pred == ICmpInst::ICMP_ULT ? ICmpInst::ICMP_NE : ICmpInst::ICMP_EQ; |
4831 | Constant *Zero = ConstantInt::getNullValue(Ty: Op1->getType()); |
4832 | return new ICmpInst(NewPred, Op1, Zero); |
4833 | } |
4834 | |
4835 | if ((Pred == ICmpInst::ICMP_UGT || Pred == ICmpInst::ICMP_ULE) && |
4836 | match(V: Op1, P: m_And(L: m_BinOp(I&: BO), R: m_LowBitMask(V&: C))) && |
4837 | match(V: BO, P: m_Add(L: m_Specific(V: Op0), R: m_SpecificIntAllowPoison(V: *C)))) { |
4838 | CmpInst::Predicate NewPred = |
4839 | Pred == ICmpInst::ICMP_UGT ? ICmpInst::ICMP_NE : ICmpInst::ICMP_EQ; |
4840 | Constant *Zero = ConstantInt::getNullValue(Ty: Op1->getType()); |
4841 | return new ICmpInst(NewPred, Op0, Zero); |
4842 | } |
4843 | } |
4844 | |
4845 | bool NoOp0WrapProblem = false, NoOp1WrapProblem = false; |
4846 | bool Op0HasNUW = false, Op1HasNUW = false; |
4847 | bool Op0HasNSW = false, Op1HasNSW = false; |
4848 | // Analyze the case when either Op0 or Op1 is an add instruction. |
4849 | // Op0 = A + B (or A and B are null); Op1 = C + D (or C and D are null). |
4850 | auto hasNoWrapProblem = [](const BinaryOperator &BO, CmpInst::Predicate Pred, |
4851 | bool &HasNSW, bool &HasNUW) -> bool { |
4852 | if (isa<OverflowingBinaryOperator>(Val: BO)) { |
4853 | HasNUW = BO.hasNoUnsignedWrap(); |
4854 | HasNSW = BO.hasNoSignedWrap(); |
4855 | return ICmpInst::isEquality(P: Pred) || |
4856 | (CmpInst::isUnsigned(predicate: Pred) && HasNUW) || |
4857 | (CmpInst::isSigned(predicate: Pred) && HasNSW); |
4858 | } else if (BO.getOpcode() == Instruction::Or) { |
4859 | HasNUW = true; |
4860 | HasNSW = true; |
4861 | return true; |
4862 | } else { |
4863 | return false; |
4864 | } |
4865 | }; |
4866 | Value *A = nullptr, *B = nullptr, *C = nullptr, *D = nullptr; |
4867 | |
4868 | if (BO0) { |
4869 | match(V: BO0, P: m_AddLike(L: m_Value(V&: A), R: m_Value(V&: B))); |
4870 | NoOp0WrapProblem = hasNoWrapProblem(*BO0, Pred, Op0HasNSW, Op0HasNUW); |
4871 | } |
4872 | if (BO1) { |
4873 | match(V: BO1, P: m_AddLike(L: m_Value(V&: C), R: m_Value(V&: D))); |
4874 | NoOp1WrapProblem = hasNoWrapProblem(*BO1, Pred, Op1HasNSW, Op1HasNUW); |
4875 | } |
4876 | |
4877 | // icmp (A+B), A -> icmp B, 0 for equalities or if there is no overflow. |
4878 | // icmp (A+B), B -> icmp A, 0 for equalities or if there is no overflow. |
4879 | if ((A == Op1 || B == Op1) && NoOp0WrapProblem) |
4880 | return new ICmpInst(Pred, A == Op1 ? B : A, |
4881 | Constant::getNullValue(Ty: Op1->getType())); |
4882 | |
4883 | // icmp C, (C+D) -> icmp 0, D for equalities or if there is no overflow. |
4884 | // icmp D, (C+D) -> icmp 0, C for equalities or if there is no overflow. |
4885 | if ((C == Op0 || D == Op0) && NoOp1WrapProblem) |
4886 | return new ICmpInst(Pred, Constant::getNullValue(Ty: Op0->getType()), |
4887 | C == Op0 ? D : C); |
4888 | |
4889 | // icmp (A+B), (A+D) -> icmp B, D for equalities or if there is no overflow. |
4890 | if (A && C && (A == C || A == D || B == C || B == D) && NoOp0WrapProblem && |
4891 | NoOp1WrapProblem) { |
4892 | // Determine Y and Z in the form icmp (X+Y), (X+Z). |
4893 | Value *Y, *Z; |
4894 | if (A == C) { |
4895 | // C + B == C + D -> B == D |
4896 | Y = B; |
4897 | Z = D; |
4898 | } else if (A == D) { |
4899 | // D + B == C + D -> B == C |
4900 | Y = B; |
4901 | Z = C; |
4902 | } else if (B == C) { |
4903 | // A + C == C + D -> A == D |
4904 | Y = A; |
4905 | Z = D; |
4906 | } else { |
4907 | assert(B == D); |
4908 | // A + D == C + D -> A == C |
4909 | Y = A; |
4910 | Z = C; |
4911 | } |
4912 | return new ICmpInst(Pred, Y, Z); |
4913 | } |
4914 | |
4915 | // icmp slt (A + -1), Op1 -> icmp sle A, Op1 |
4916 | if (A && NoOp0WrapProblem && Pred == CmpInst::ICMP_SLT && |
4917 | match(V: B, P: m_AllOnes())) |
4918 | return new ICmpInst(CmpInst::ICMP_SLE, A, Op1); |
4919 | |
4920 | // icmp sge (A + -1), Op1 -> icmp sgt A, Op1 |
4921 | if (A && NoOp0WrapProblem && Pred == CmpInst::ICMP_SGE && |
4922 | match(V: B, P: m_AllOnes())) |
4923 | return new ICmpInst(CmpInst::ICMP_SGT, A, Op1); |
4924 | |
4925 | // icmp sle (A + 1), Op1 -> icmp slt A, Op1 |
4926 | if (A && NoOp0WrapProblem && Pred == CmpInst::ICMP_SLE && match(V: B, P: m_One())) |
4927 | return new ICmpInst(CmpInst::ICMP_SLT, A, Op1); |
4928 | |
4929 | // icmp sgt (A + 1), Op1 -> icmp sge A, Op1 |
4930 | if (A && NoOp0WrapProblem && Pred == CmpInst::ICMP_SGT && match(V: B, P: m_One())) |
4931 | return new ICmpInst(CmpInst::ICMP_SGE, A, Op1); |
4932 | |
4933 | // icmp sgt Op0, (C + -1) -> icmp sge Op0, C |
4934 | if (C && NoOp1WrapProblem && Pred == CmpInst::ICMP_SGT && |
4935 | match(V: D, P: m_AllOnes())) |
4936 | return new ICmpInst(CmpInst::ICMP_SGE, Op0, C); |
4937 | |
4938 | // icmp sle Op0, (C + -1) -> icmp slt Op0, C |
4939 | if (C && NoOp1WrapProblem && Pred == CmpInst::ICMP_SLE && |
4940 | match(V: D, P: m_AllOnes())) |
4941 | return new ICmpInst(CmpInst::ICMP_SLT, Op0, C); |
4942 | |
4943 | // icmp sge Op0, (C + 1) -> icmp sgt Op0, C |
4944 | if (C && NoOp1WrapProblem && Pred == CmpInst::ICMP_SGE && match(V: D, P: m_One())) |
4945 | return new ICmpInst(CmpInst::ICMP_SGT, Op0, C); |
4946 | |
4947 | // icmp slt Op0, (C + 1) -> icmp sle Op0, C |
4948 | if (C && NoOp1WrapProblem && Pred == CmpInst::ICMP_SLT && match(V: D, P: m_One())) |
4949 | return new ICmpInst(CmpInst::ICMP_SLE, Op0, C); |
4950 | |
4951 | // TODO: The subtraction-related identities shown below also hold, but |
4952 | // canonicalization from (X -nuw 1) to (X + -1) means that the combinations |
4953 | // wouldn't happen even if they were implemented. |
4954 | // |
4955 | // icmp ult (A - 1), Op1 -> icmp ule A, Op1 |
4956 | // icmp uge (A - 1), Op1 -> icmp ugt A, Op1 |
4957 | // icmp ugt Op0, (C - 1) -> icmp uge Op0, C |
4958 | // icmp ule Op0, (C - 1) -> icmp ult Op0, C |
4959 | |
4960 | // icmp ule (A + 1), Op0 -> icmp ult A, Op1 |
4961 | if (A && NoOp0WrapProblem && Pred == CmpInst::ICMP_ULE && match(V: B, P: m_One())) |
4962 | return new ICmpInst(CmpInst::ICMP_ULT, A, Op1); |
4963 | |
4964 | // icmp ugt (A + 1), Op0 -> icmp uge A, Op1 |
4965 | if (A && NoOp0WrapProblem && Pred == CmpInst::ICMP_UGT && match(V: B, P: m_One())) |
4966 | return new ICmpInst(CmpInst::ICMP_UGE, A, Op1); |
4967 | |
4968 | // icmp uge Op0, (C + 1) -> icmp ugt Op0, C |
4969 | if (C && NoOp1WrapProblem && Pred == CmpInst::ICMP_UGE && match(V: D, P: m_One())) |
4970 | return new ICmpInst(CmpInst::ICMP_UGT, Op0, C); |
4971 | |
4972 | // icmp ult Op0, (C + 1) -> icmp ule Op0, C |
4973 | if (C && NoOp1WrapProblem && Pred == CmpInst::ICMP_ULT && match(V: D, P: m_One())) |
4974 | return new ICmpInst(CmpInst::ICMP_ULE, Op0, C); |
4975 | |
4976 | // if C1 has greater magnitude than C2: |
4977 | // icmp (A + C1), (C + C2) -> icmp (A + C3), C |
4978 | // s.t. C3 = C1 - C2 |
4979 | // |
4980 | // if C2 has greater magnitude than C1: |
4981 | // icmp (A + C1), (C + C2) -> icmp A, (C + C3) |
4982 | // s.t. C3 = C2 - C1 |
4983 | if (A && C && NoOp0WrapProblem && NoOp1WrapProblem && |
4984 | (BO0->hasOneUse() || BO1->hasOneUse()) && !I.isUnsigned()) { |
4985 | const APInt *AP1, *AP2; |
4986 | // TODO: Support non-uniform vectors. |
4987 | // TODO: Allow poison passthrough if B or D's element is poison. |
4988 | if (match(V: B, P: m_APIntAllowPoison(Res&: AP1)) && |
4989 | match(V: D, P: m_APIntAllowPoison(Res&: AP2)) && |
4990 | AP1->isNegative() == AP2->isNegative()) { |
4991 | APInt AP1Abs = AP1->abs(); |
4992 | APInt AP2Abs = AP2->abs(); |
4993 | if (AP1Abs.uge(RHS: AP2Abs)) { |
4994 | APInt Diff = *AP1 - *AP2; |
4995 | Constant *C3 = Constant::getIntegerValue(Ty: BO0->getType(), V: Diff); |
4996 | Value *NewAdd = Builder.CreateAdd( |
4997 | LHS: A, RHS: C3, Name: "" , HasNUW: Op0HasNUW && Diff.ule(RHS: *AP1), HasNSW: Op0HasNSW); |
4998 | return new ICmpInst(Pred, NewAdd, C); |
4999 | } else { |
5000 | APInt Diff = *AP2 - *AP1; |
5001 | Constant *C3 = Constant::getIntegerValue(Ty: BO0->getType(), V: Diff); |
5002 | Value *NewAdd = Builder.CreateAdd( |
5003 | LHS: C, RHS: C3, Name: "" , HasNUW: Op1HasNUW && Diff.ule(RHS: *AP2), HasNSW: Op1HasNSW); |
5004 | return new ICmpInst(Pred, A, NewAdd); |
5005 | } |
5006 | } |
5007 | Constant *Cst1, *Cst2; |
5008 | if (match(V: B, P: m_ImmConstant(C&: Cst1)) && match(V: D, P: m_ImmConstant(C&: Cst2)) && |
5009 | ICmpInst::isEquality(P: Pred)) { |
5010 | Constant *Diff = ConstantExpr::getSub(C1: Cst2, C2: Cst1); |
5011 | Value *NewAdd = Builder.CreateAdd(LHS: C, RHS: Diff); |
5012 | return new ICmpInst(Pred, A, NewAdd); |
5013 | } |
5014 | } |
5015 | |
5016 | // Analyze the case when either Op0 or Op1 is a sub instruction. |
5017 | // Op0 = A - B (or A and B are null); Op1 = C - D (or C and D are null). |
5018 | A = nullptr; |
5019 | B = nullptr; |
5020 | C = nullptr; |
5021 | D = nullptr; |
5022 | if (BO0 && BO0->getOpcode() == Instruction::Sub) { |
5023 | A = BO0->getOperand(i_nocapture: 0); |
5024 | B = BO0->getOperand(i_nocapture: 1); |
5025 | } |
5026 | if (BO1 && BO1->getOpcode() == Instruction::Sub) { |
5027 | C = BO1->getOperand(i_nocapture: 0); |
5028 | D = BO1->getOperand(i_nocapture: 1); |
5029 | } |
5030 | |
5031 | // icmp (A-B), A -> icmp 0, B for equalities or if there is no overflow. |
5032 | if (A == Op1 && NoOp0WrapProblem) |
5033 | return new ICmpInst(Pred, Constant::getNullValue(Ty: Op1->getType()), B); |
5034 | // icmp C, (C-D) -> icmp D, 0 for equalities or if there is no overflow. |
5035 | if (C == Op0 && NoOp1WrapProblem) |
5036 | return new ICmpInst(Pred, D, Constant::getNullValue(Ty: Op0->getType())); |
5037 | |
5038 | // Convert sub-with-unsigned-overflow comparisons into a comparison of args. |
5039 | // (A - B) u>/u<= A --> B u>/u<= A |
5040 | if (A == Op1 && (Pred == ICmpInst::ICMP_UGT || Pred == ICmpInst::ICMP_ULE)) |
5041 | return new ICmpInst(Pred, B, A); |
5042 | // C u</u>= (C - D) --> C u</u>= D |
5043 | if (C == Op0 && (Pred == ICmpInst::ICMP_ULT || Pred == ICmpInst::ICMP_UGE)) |
5044 | return new ICmpInst(Pred, C, D); |
5045 | // (A - B) u>=/u< A --> B u>/u<= A iff B != 0 |
5046 | if (A == Op1 && (Pred == ICmpInst::ICMP_UGE || Pred == ICmpInst::ICMP_ULT) && |
5047 | isKnownNonZero(V: B, Q)) |
5048 | return new ICmpInst(CmpInst::getFlippedStrictnessPredicate(pred: Pred), B, A); |
5049 | // C u<=/u> (C - D) --> C u</u>= D iff B != 0 |
5050 | if (C == Op0 && (Pred == ICmpInst::ICMP_ULE || Pred == ICmpInst::ICMP_UGT) && |
5051 | isKnownNonZero(V: D, Q)) |
5052 | return new ICmpInst(CmpInst::getFlippedStrictnessPredicate(pred: Pred), C, D); |
5053 | |
5054 | // icmp (A-B), (C-B) -> icmp A, C for equalities or if there is no overflow. |
5055 | if (B && D && B == D && NoOp0WrapProblem && NoOp1WrapProblem) |
5056 | return new ICmpInst(Pred, A, C); |
5057 | |
5058 | // icmp (A-B), (A-D) -> icmp D, B for equalities or if there is no overflow. |
5059 | if (A && C && A == C && NoOp0WrapProblem && NoOp1WrapProblem) |
5060 | return new ICmpInst(Pred, D, B); |
5061 | |
5062 | // icmp (0-X) < cst --> x > -cst |
5063 | if (NoOp0WrapProblem && ICmpInst::isSigned(predicate: Pred)) { |
5064 | Value *X; |
5065 | if (match(V: BO0, P: m_Neg(V: m_Value(V&: X)))) |
5066 | if (Constant *RHSC = dyn_cast<Constant>(Val: Op1)) |
5067 | if (RHSC->isNotMinSignedValue()) |
5068 | return new ICmpInst(I.getSwappedPredicate(), X, |
5069 | ConstantExpr::getNeg(C: RHSC)); |
5070 | } |
5071 | |
5072 | if (Instruction * R = foldICmpXorXX(I, Q, IC&: *this)) |
5073 | return R; |
5074 | if (Instruction *R = foldICmpOrXX(I, Q, IC&: *this)) |
5075 | return R; |
5076 | |
5077 | { |
5078 | // Try to remove shared multiplier from comparison: |
5079 | // X * Z u{lt/le/gt/ge}/eq/ne Y * Z |
5080 | Value *X, *Y, *Z; |
5081 | if (Pred == ICmpInst::getUnsignedPredicate(pred: Pred) && |
5082 | ((match(V: Op0, P: m_Mul(L: m_Value(V&: X), R: m_Value(V&: Z))) && |
5083 | match(V: Op1, P: m_c_Mul(L: m_Specific(V: Z), R: m_Value(V&: Y)))) || |
5084 | (match(V: Op0, P: m_Mul(L: m_Value(V&: Z), R: m_Value(V&: X))) && |
5085 | match(V: Op1, P: m_c_Mul(L: m_Specific(V: Z), R: m_Value(V&: Y)))))) { |
5086 | bool NonZero; |
5087 | if (ICmpInst::isEquality(P: Pred)) { |
5088 | KnownBits ZKnown = computeKnownBits(V: Z, Depth: 0, CxtI: &I); |
5089 | // if Z % 2 != 0 |
5090 | // X * Z eq/ne Y * Z -> X eq/ne Y |
5091 | if (ZKnown.countMaxTrailingZeros() == 0) |
5092 | return new ICmpInst(Pred, X, Y); |
5093 | NonZero = !ZKnown.One.isZero() || isKnownNonZero(V: Z, Q); |
5094 | // if Z != 0 and nsw(X * Z) and nsw(Y * Z) |
5095 | // X * Z eq/ne Y * Z -> X eq/ne Y |
5096 | if (NonZero && BO0 && BO1 && Op0HasNSW && Op1HasNSW) |
5097 | return new ICmpInst(Pred, X, Y); |
5098 | } else |
5099 | NonZero = isKnownNonZero(V: Z, Q); |
5100 | |
5101 | // If Z != 0 and nuw(X * Z) and nuw(Y * Z) |
5102 | // X * Z u{lt/le/gt/ge}/eq/ne Y * Z -> X u{lt/le/gt/ge}/eq/ne Y |
5103 | if (NonZero && BO0 && BO1 && Op0HasNUW && Op1HasNUW) |
5104 | return new ICmpInst(Pred, X, Y); |
5105 | } |
5106 | } |
5107 | |
5108 | BinaryOperator *SRem = nullptr; |
5109 | // icmp (srem X, Y), Y |
5110 | if (BO0 && BO0->getOpcode() == Instruction::SRem && Op1 == BO0->getOperand(i_nocapture: 1)) |
5111 | SRem = BO0; |
5112 | // icmp Y, (srem X, Y) |
5113 | else if (BO1 && BO1->getOpcode() == Instruction::SRem && |
5114 | Op0 == BO1->getOperand(i_nocapture: 1)) |
5115 | SRem = BO1; |
5116 | if (SRem) { |
5117 | // We don't check hasOneUse to avoid increasing register pressure because |
5118 | // the value we use is the same value this instruction was already using. |
5119 | switch (SRem == BO0 ? ICmpInst::getSwappedPredicate(pred: Pred) : Pred) { |
5120 | default: |
5121 | break; |
5122 | case ICmpInst::ICMP_EQ: |
5123 | return replaceInstUsesWith(I, V: ConstantInt::getFalse(Ty: I.getType())); |
5124 | case ICmpInst::ICMP_NE: |
5125 | return replaceInstUsesWith(I, V: ConstantInt::getTrue(Ty: I.getType())); |
5126 | case ICmpInst::ICMP_SGT: |
5127 | case ICmpInst::ICMP_SGE: |
5128 | return new ICmpInst(ICmpInst::ICMP_SGT, SRem->getOperand(i_nocapture: 1), |
5129 | Constant::getAllOnesValue(Ty: SRem->getType())); |
5130 | case ICmpInst::ICMP_SLT: |
5131 | case ICmpInst::ICMP_SLE: |
5132 | return new ICmpInst(ICmpInst::ICMP_SLT, SRem->getOperand(i_nocapture: 1), |
5133 | Constant::getNullValue(Ty: SRem->getType())); |
5134 | } |
5135 | } |
5136 | |
5137 | if (BO0 && BO1 && BO0->getOpcode() == BO1->getOpcode() && |
5138 | (BO0->hasOneUse() || BO1->hasOneUse()) && |
5139 | BO0->getOperand(i_nocapture: 1) == BO1->getOperand(i_nocapture: 1)) { |
5140 | switch (BO0->getOpcode()) { |
5141 | default: |
5142 | break; |
5143 | case Instruction::Add: |
5144 | case Instruction::Sub: |
5145 | case Instruction::Xor: { |
5146 | if (I.isEquality()) // a+x icmp eq/ne b+x --> a icmp b |
5147 | return new ICmpInst(Pred, BO0->getOperand(i_nocapture: 0), BO1->getOperand(i_nocapture: 0)); |
5148 | |
5149 | const APInt *C; |
5150 | if (match(V: BO0->getOperand(i_nocapture: 1), P: m_APInt(Res&: C))) { |
5151 | // icmp u/s (a ^ signmask), (b ^ signmask) --> icmp s/u a, b |
5152 | if (C->isSignMask()) { |
5153 | ICmpInst::Predicate NewPred = I.getFlippedSignednessPredicate(); |
5154 | return new ICmpInst(NewPred, BO0->getOperand(i_nocapture: 0), BO1->getOperand(i_nocapture: 0)); |
5155 | } |
5156 | |
5157 | // icmp u/s (a ^ maxsignval), (b ^ maxsignval) --> icmp s/u' a, b |
5158 | if (BO0->getOpcode() == Instruction::Xor && C->isMaxSignedValue()) { |
5159 | ICmpInst::Predicate NewPred = I.getFlippedSignednessPredicate(); |
5160 | NewPred = I.getSwappedPredicate(pred: NewPred); |
5161 | return new ICmpInst(NewPred, BO0->getOperand(i_nocapture: 0), BO1->getOperand(i_nocapture: 0)); |
5162 | } |
5163 | } |
5164 | break; |
5165 | } |
5166 | case Instruction::Mul: { |
5167 | if (!I.isEquality()) |
5168 | break; |
5169 | |
5170 | const APInt *C; |
5171 | if (match(V: BO0->getOperand(i_nocapture: 1), P: m_APInt(Res&: C)) && !C->isZero() && |
5172 | !C->isOne()) { |
5173 | // icmp eq/ne (X * C), (Y * C) --> icmp (X & Mask), (Y & Mask) |
5174 | // Mask = -1 >> count-trailing-zeros(C). |
5175 | if (unsigned TZs = C->countr_zero()) { |
5176 | Constant *Mask = ConstantInt::get( |
5177 | Ty: BO0->getType(), |
5178 | V: APInt::getLowBitsSet(numBits: C->getBitWidth(), loBitsSet: C->getBitWidth() - TZs)); |
5179 | Value *And1 = Builder.CreateAnd(LHS: BO0->getOperand(i_nocapture: 0), RHS: Mask); |
5180 | Value *And2 = Builder.CreateAnd(LHS: BO1->getOperand(i_nocapture: 0), RHS: Mask); |
5181 | return new ICmpInst(Pred, And1, And2); |
5182 | } |
5183 | } |
5184 | break; |
5185 | } |
5186 | case Instruction::UDiv: |
5187 | case Instruction::LShr: |
5188 | if (I.isSigned() || !BO0->isExact() || !BO1->isExact()) |
5189 | break; |
5190 | return new ICmpInst(Pred, BO0->getOperand(i_nocapture: 0), BO1->getOperand(i_nocapture: 0)); |
5191 | |
5192 | case Instruction::SDiv: |
5193 | if (!(I.isEquality() || match(V: BO0->getOperand(i_nocapture: 1), P: m_NonNegative())) || |
5194 | !BO0->isExact() || !BO1->isExact()) |
5195 | break; |
5196 | return new ICmpInst(Pred, BO0->getOperand(i_nocapture: 0), BO1->getOperand(i_nocapture: 0)); |
5197 | |
5198 | case Instruction::AShr: |
5199 | if (!BO0->isExact() || !BO1->isExact()) |
5200 | break; |
5201 | return new ICmpInst(Pred, BO0->getOperand(i_nocapture: 0), BO1->getOperand(i_nocapture: 0)); |
5202 | |
5203 | case Instruction::Shl: { |
5204 | bool NUW = Op0HasNUW && Op1HasNUW; |
5205 | bool NSW = Op0HasNSW && Op1HasNSW; |
5206 | if (!NUW && !NSW) |
5207 | break; |
5208 | if (!NSW && I.isSigned()) |
5209 | break; |
5210 | return new ICmpInst(Pred, BO0->getOperand(i_nocapture: 0), BO1->getOperand(i_nocapture: 0)); |
5211 | } |
5212 | } |
5213 | } |
5214 | |
5215 | if (BO0) { |
5216 | // Transform A & (L - 1) `ult` L --> L != 0 |
5217 | auto LSubOne = m_Add(L: m_Specific(V: Op1), R: m_AllOnes()); |
5218 | auto BitwiseAnd = m_c_And(L: m_Value(), R: LSubOne); |
5219 | |
5220 | if (match(V: BO0, P: BitwiseAnd) && Pred == ICmpInst::ICMP_ULT) { |
5221 | auto *Zero = Constant::getNullValue(Ty: BO0->getType()); |
5222 | return new ICmpInst(ICmpInst::ICMP_NE, Op1, Zero); |
5223 | } |
5224 | } |
5225 | |
5226 | // For unsigned predicates / eq / ne: |
5227 | // icmp pred (x << 1), x --> icmp getSignedPredicate(pred) x, 0 |
5228 | // icmp pred x, (x << 1) --> icmp getSignedPredicate(pred) 0, x |
5229 | if (!ICmpInst::isSigned(predicate: Pred)) { |
5230 | if (match(V: Op0, P: m_Shl(L: m_Specific(V: Op1), R: m_One()))) |
5231 | return new ICmpInst(ICmpInst::getSignedPredicate(pred: Pred), Op1, |
5232 | Constant::getNullValue(Ty: Op1->getType())); |
5233 | else if (match(V: Op1, P: m_Shl(L: m_Specific(V: Op0), R: m_One()))) |
5234 | return new ICmpInst(ICmpInst::getSignedPredicate(pred: Pred), |
5235 | Constant::getNullValue(Ty: Op0->getType()), Op0); |
5236 | } |
5237 | |
5238 | if (Value *V = foldMultiplicationOverflowCheck(I)) |
5239 | return replaceInstUsesWith(I, V); |
5240 | |
5241 | if (Instruction *R = foldICmpAndXX(I, Q, IC&: *this)) |
5242 | return R; |
5243 | |
5244 | if (Value *V = foldICmpWithTruncSignExtendedVal(I, Builder)) |
5245 | return replaceInstUsesWith(I, V); |
5246 | |
5247 | if (Value *V = foldShiftIntoShiftInAnotherHandOfAndInICmp(I, SQ, Builder)) |
5248 | return replaceInstUsesWith(I, V); |
5249 | |
5250 | return nullptr; |
5251 | } |
5252 | |
5253 | /// Fold icmp Pred min|max(X, Y), Z. |
5254 | Instruction *InstCombinerImpl::foldICmpWithMinMax(Instruction &I, |
5255 | MinMaxIntrinsic *MinMax, |
5256 | Value *Z, |
5257 | ICmpInst::Predicate Pred) { |
5258 | Value *X = MinMax->getLHS(); |
5259 | Value *Y = MinMax->getRHS(); |
5260 | if (ICmpInst::isSigned(predicate: Pred) && !MinMax->isSigned()) |
5261 | return nullptr; |
5262 | if (ICmpInst::isUnsigned(predicate: Pred) && MinMax->isSigned()) { |
5263 | // Revert the transform signed pred -> unsigned pred |
5264 | // TODO: We can flip the signedness of predicate if both operands of icmp |
5265 | // are negative. |
5266 | if (isKnownNonNegative(V: Z, SQ: SQ.getWithInstruction(I: &I)) && |
5267 | isKnownNonNegative(V: MinMax, SQ: SQ.getWithInstruction(I: &I))) { |
5268 | Pred = ICmpInst::getFlippedSignednessPredicate(pred: Pred); |
5269 | } else |
5270 | return nullptr; |
5271 | } |
5272 | SimplifyQuery Q = SQ.getWithInstruction(I: &I); |
5273 | auto IsCondKnownTrue = [](Value *Val) -> std::optional<bool> { |
5274 | if (!Val) |
5275 | return std::nullopt; |
5276 | if (match(V: Val, P: m_One())) |
5277 | return true; |
5278 | if (match(V: Val, P: m_Zero())) |
5279 | return false; |
5280 | return std::nullopt; |
5281 | }; |
5282 | auto CmpXZ = IsCondKnownTrue(simplifyICmpInst(Predicate: Pred, LHS: X, RHS: Z, Q)); |
5283 | auto CmpYZ = IsCondKnownTrue(simplifyICmpInst(Predicate: Pred, LHS: Y, RHS: Z, Q)); |
5284 | if (!CmpXZ.has_value() && !CmpYZ.has_value()) |
5285 | return nullptr; |
5286 | if (!CmpXZ.has_value()) { |
5287 | std::swap(a&: X, b&: Y); |
5288 | std::swap(lhs&: CmpXZ, rhs&: CmpYZ); |
5289 | } |
5290 | |
5291 | auto FoldIntoCmpYZ = [&]() -> Instruction * { |
5292 | if (CmpYZ.has_value()) |
5293 | return replaceInstUsesWith(I, V: ConstantInt::getBool(Ty: I.getType(), V: *CmpYZ)); |
5294 | return ICmpInst::Create(Op: Instruction::ICmp, Pred, S1: Y, S2: Z); |
5295 | }; |
5296 | |
5297 | switch (Pred) { |
5298 | case ICmpInst::ICMP_EQ: |
5299 | case ICmpInst::ICMP_NE: { |
5300 | // If X == Z: |
5301 | // Expr Result |
5302 | // min(X, Y) == Z X <= Y |
5303 | // max(X, Y) == Z X >= Y |
5304 | // min(X, Y) != Z X > Y |
5305 | // max(X, Y) != Z X < Y |
5306 | if ((Pred == ICmpInst::ICMP_EQ) == *CmpXZ) { |
5307 | ICmpInst::Predicate NewPred = |
5308 | ICmpInst::getNonStrictPredicate(pred: MinMax->getPredicate()); |
5309 | if (Pred == ICmpInst::ICMP_NE) |
5310 | NewPred = ICmpInst::getInversePredicate(pred: NewPred); |
5311 | return ICmpInst::Create(Op: Instruction::ICmp, Pred: NewPred, S1: X, S2: Y); |
5312 | } |
5313 | // Otherwise (X != Z): |
5314 | ICmpInst::Predicate NewPred = MinMax->getPredicate(); |
5315 | auto MinMaxCmpXZ = IsCondKnownTrue(simplifyICmpInst(Predicate: NewPred, LHS: X, RHS: Z, Q)); |
5316 | if (!MinMaxCmpXZ.has_value()) { |
5317 | std::swap(a&: X, b&: Y); |
5318 | std::swap(lhs&: CmpXZ, rhs&: CmpYZ); |
5319 | // Re-check pre-condition X != Z |
5320 | if (!CmpXZ.has_value() || (Pred == ICmpInst::ICMP_EQ) == *CmpXZ) |
5321 | break; |
5322 | MinMaxCmpXZ = IsCondKnownTrue(simplifyICmpInst(Predicate: NewPred, LHS: X, RHS: Z, Q)); |
5323 | } |
5324 | if (!MinMaxCmpXZ.has_value()) |
5325 | break; |
5326 | if (*MinMaxCmpXZ) { |
5327 | // Expr Fact Result |
5328 | // min(X, Y) == Z X < Z false |
5329 | // max(X, Y) == Z X > Z false |
5330 | // min(X, Y) != Z X < Z true |
5331 | // max(X, Y) != Z X > Z true |
5332 | return replaceInstUsesWith( |
5333 | I, V: ConstantInt::getBool(Ty: I.getType(), V: Pred == ICmpInst::ICMP_NE)); |
5334 | } else { |
5335 | // Expr Fact Result |
5336 | // min(X, Y) == Z X > Z Y == Z |
5337 | // max(X, Y) == Z X < Z Y == Z |
5338 | // min(X, Y) != Z X > Z Y != Z |
5339 | // max(X, Y) != Z X < Z Y != Z |
5340 | return FoldIntoCmpYZ(); |
5341 | } |
5342 | break; |
5343 | } |
5344 | case ICmpInst::ICMP_SLT: |
5345 | case ICmpInst::ICMP_ULT: |
5346 | case ICmpInst::ICMP_SLE: |
5347 | case ICmpInst::ICMP_ULE: |
5348 | case ICmpInst::ICMP_SGT: |
5349 | case ICmpInst::ICMP_UGT: |
5350 | case ICmpInst::ICMP_SGE: |
5351 | case ICmpInst::ICMP_UGE: { |
5352 | bool IsSame = MinMax->getPredicate() == ICmpInst::getStrictPredicate(pred: Pred); |
5353 | if (*CmpXZ) { |
5354 | if (IsSame) { |
5355 | // Expr Fact Result |
5356 | // min(X, Y) < Z X < Z true |
5357 | // min(X, Y) <= Z X <= Z true |
5358 | // max(X, Y) > Z X > Z true |
5359 | // max(X, Y) >= Z X >= Z true |
5360 | return replaceInstUsesWith(I, V: ConstantInt::getTrue(Ty: I.getType())); |
5361 | } else { |
5362 | // Expr Fact Result |
5363 | // max(X, Y) < Z X < Z Y < Z |
5364 | // max(X, Y) <= Z X <= Z Y <= Z |
5365 | // min(X, Y) > Z X > Z Y > Z |
5366 | // min(X, Y) >= Z X >= Z Y >= Z |
5367 | return FoldIntoCmpYZ(); |
5368 | } |
5369 | } else { |
5370 | if (IsSame) { |
5371 | // Expr Fact Result |
5372 | // min(X, Y) < Z X >= Z Y < Z |
5373 | // min(X, Y) <= Z X > Z Y <= Z |
5374 | // max(X, Y) > Z X <= Z Y > Z |
5375 | // max(X, Y) >= Z X < Z Y >= Z |
5376 | return FoldIntoCmpYZ(); |
5377 | } else { |
5378 | // Expr Fact Result |
5379 | // max(X, Y) < Z X >= Z false |
5380 | // max(X, Y) <= Z X > Z false |
5381 | // min(X, Y) > Z X <= Z false |
5382 | // min(X, Y) >= Z X < Z false |
5383 | return replaceInstUsesWith(I, V: ConstantInt::getFalse(Ty: I.getType())); |
5384 | } |
5385 | } |
5386 | break; |
5387 | } |
5388 | default: |
5389 | break; |
5390 | } |
5391 | |
5392 | return nullptr; |
5393 | } |
5394 | |
5395 | // Canonicalize checking for a power-of-2-or-zero value: |
5396 | static Instruction *foldICmpPow2Test(ICmpInst &I, |
5397 | InstCombiner::BuilderTy &Builder) { |
5398 | Value *Op0 = I.getOperand(i_nocapture: 0), *Op1 = I.getOperand(i_nocapture: 1); |
5399 | const CmpInst::Predicate Pred = I.getPredicate(); |
5400 | Value *A = nullptr; |
5401 | bool CheckIs; |
5402 | if (I.isEquality()) { |
5403 | // (A & (A-1)) == 0 --> ctpop(A) < 2 (two commuted variants) |
5404 | // ((A-1) & A) != 0 --> ctpop(A) > 1 (two commuted variants) |
5405 | if (!match(V: Op0, P: m_OneUse(SubPattern: m_c_And(L: m_Add(L: m_Value(V&: A), R: m_AllOnes()), |
5406 | R: m_Deferred(V: A)))) || |
5407 | !match(V: Op1, P: m_ZeroInt())) |
5408 | A = nullptr; |
5409 | |
5410 | // (A & -A) == A --> ctpop(A) < 2 (four commuted variants) |
5411 | // (-A & A) != A --> ctpop(A) > 1 (four commuted variants) |
5412 | if (match(V: Op0, P: m_OneUse(SubPattern: m_c_And(L: m_Neg(V: m_Specific(V: Op1)), R: m_Specific(V: Op1))))) |
5413 | A = Op1; |
5414 | else if (match(V: Op1, |
5415 | P: m_OneUse(SubPattern: m_c_And(L: m_Neg(V: m_Specific(V: Op0)), R: m_Specific(V: Op0))))) |
5416 | A = Op0; |
5417 | |
5418 | CheckIs = Pred == ICmpInst::ICMP_EQ; |
5419 | } else if (ICmpInst::isUnsigned(predicate: Pred)) { |
5420 | // (A ^ (A-1)) u>= A --> ctpop(A) < 2 (two commuted variants) |
5421 | // ((A-1) ^ A) u< A --> ctpop(A) > 1 (two commuted variants) |
5422 | |
5423 | if ((Pred == ICmpInst::ICMP_UGE || Pred == ICmpInst::ICMP_ULT) && |
5424 | match(V: Op0, P: m_OneUse(SubPattern: m_c_Xor(L: m_Add(L: m_Specific(V: Op1), R: m_AllOnes()), |
5425 | R: m_Specific(V: Op1))))) { |
5426 | A = Op1; |
5427 | CheckIs = Pred == ICmpInst::ICMP_UGE; |
5428 | } else if ((Pred == ICmpInst::ICMP_UGT || Pred == ICmpInst::ICMP_ULE) && |
5429 | match(V: Op1, P: m_OneUse(SubPattern: m_c_Xor(L: m_Add(L: m_Specific(V: Op0), R: m_AllOnes()), |
5430 | R: m_Specific(V: Op0))))) { |
5431 | A = Op0; |
5432 | CheckIs = Pred == ICmpInst::ICMP_ULE; |
5433 | } |
5434 | } |
5435 | |
5436 | if (A) { |
5437 | Type *Ty = A->getType(); |
5438 | CallInst *CtPop = Builder.CreateUnaryIntrinsic(Intrinsic::ID: ctpop, V: A); |
5439 | return CheckIs ? new ICmpInst(ICmpInst::ICMP_ULT, CtPop, |
5440 | ConstantInt::get(Ty, V: 2)) |
5441 | : new ICmpInst(ICmpInst::ICMP_UGT, CtPop, |
5442 | ConstantInt::get(Ty, V: 1)); |
5443 | } |
5444 | |
5445 | return nullptr; |
5446 | } |
5447 | |
5448 | Instruction *InstCombinerImpl::foldICmpEquality(ICmpInst &I) { |
5449 | if (!I.isEquality()) |
5450 | return nullptr; |
5451 | |
5452 | Value *Op0 = I.getOperand(i_nocapture: 0), *Op1 = I.getOperand(i_nocapture: 1); |
5453 | const CmpInst::Predicate Pred = I.getPredicate(); |
5454 | Value *A, *B, *C, *D; |
5455 | if (match(V: Op0, P: m_Xor(L: m_Value(V&: A), R: m_Value(V&: B)))) { |
5456 | if (A == Op1 || B == Op1) { // (A^B) == A -> B == 0 |
5457 | Value *OtherVal = A == Op1 ? B : A; |
5458 | return new ICmpInst(Pred, OtherVal, Constant::getNullValue(Ty: A->getType())); |
5459 | } |
5460 | |
5461 | if (match(V: Op1, P: m_Xor(L: m_Value(V&: C), R: m_Value(V&: D)))) { |
5462 | // A^c1 == C^c2 --> A == C^(c1^c2) |
5463 | ConstantInt *C1, *C2; |
5464 | if (match(V: B, P: m_ConstantInt(CI&: C1)) && match(V: D, P: m_ConstantInt(CI&: C2)) && |
5465 | Op1->hasOneUse()) { |
5466 | Constant *NC = Builder.getInt(AI: C1->getValue() ^ C2->getValue()); |
5467 | Value *Xor = Builder.CreateXor(LHS: C, RHS: NC); |
5468 | return new ICmpInst(Pred, A, Xor); |
5469 | } |
5470 | |
5471 | // A^B == A^D -> B == D |
5472 | if (A == C) |
5473 | return new ICmpInst(Pred, B, D); |
5474 | if (A == D) |
5475 | return new ICmpInst(Pred, B, C); |
5476 | if (B == C) |
5477 | return new ICmpInst(Pred, A, D); |
5478 | if (B == D) |
5479 | return new ICmpInst(Pred, A, C); |
5480 | } |
5481 | } |
5482 | |
5483 | // canoncalize: |
5484 | // (icmp eq/ne (and X, C), X) |
5485 | // -> (icmp eq/ne (and X, ~C), 0) |
5486 | { |
5487 | Constant *CMask; |
5488 | A = nullptr; |
5489 | if (match(V: Op0, P: m_OneUse(SubPattern: m_And(L: m_Specific(V: Op1), R: m_ImmConstant(C&: CMask))))) |
5490 | A = Op1; |
5491 | else if (match(V: Op1, P: m_OneUse(SubPattern: m_And(L: m_Specific(V: Op0), R: m_ImmConstant(C&: CMask))))) |
5492 | A = Op0; |
5493 | if (A) |
5494 | return new ICmpInst(Pred, Builder.CreateAnd(LHS: A, RHS: Builder.CreateNot(V: CMask)), |
5495 | Constant::getNullValue(Ty: A->getType())); |
5496 | } |
5497 | |
5498 | if (match(V: Op1, P: m_Xor(L: m_Value(V&: A), R: m_Value(V&: B))) && (A == Op0 || B == Op0)) { |
5499 | // A == (A^B) -> B == 0 |
5500 | Value *OtherVal = A == Op0 ? B : A; |
5501 | return new ICmpInst(Pred, OtherVal, Constant::getNullValue(Ty: A->getType())); |
5502 | } |
5503 | |
5504 | // (X&Z) == (Y&Z) -> (X^Y) & Z == 0 |
5505 | if (match(V: Op0, P: m_OneUse(SubPattern: m_And(L: m_Value(V&: A), R: m_Value(V&: B)))) && |
5506 | match(V: Op1, P: m_OneUse(SubPattern: m_And(L: m_Value(V&: C), R: m_Value(V&: D))))) { |
5507 | Value *X = nullptr, *Y = nullptr, *Z = nullptr; |
5508 | |
5509 | if (A == C) { |
5510 | X = B; |
5511 | Y = D; |
5512 | Z = A; |
5513 | } else if (A == D) { |
5514 | X = B; |
5515 | Y = C; |
5516 | Z = A; |
5517 | } else if (B == C) { |
5518 | X = A; |
5519 | Y = D; |
5520 | Z = B; |
5521 | } else if (B == D) { |
5522 | X = A; |
5523 | Y = C; |
5524 | Z = B; |
5525 | } |
5526 | |
5527 | if (X) { // Build (X^Y) & Z |
5528 | Op1 = Builder.CreateXor(LHS: X, RHS: Y); |
5529 | Op1 = Builder.CreateAnd(LHS: Op1, RHS: Z); |
5530 | return new ICmpInst(Pred, Op1, Constant::getNullValue(Ty: Op1->getType())); |
5531 | } |
5532 | } |
5533 | |
5534 | { |
5535 | // Similar to above, but specialized for constant because invert is needed: |
5536 | // (X | C) == (Y | C) --> (X ^ Y) & ~C == 0 |
5537 | Value *X, *Y; |
5538 | Constant *C; |
5539 | if (match(V: Op0, P: m_OneUse(SubPattern: m_Or(L: m_Value(V&: X), R: m_Constant(C)))) && |
5540 | match(V: Op1, P: m_OneUse(SubPattern: m_Or(L: m_Value(V&: Y), R: m_Specific(V: C))))) { |
5541 | Value *Xor = Builder.CreateXor(LHS: X, RHS: Y); |
5542 | Value *And = Builder.CreateAnd(LHS: Xor, RHS: ConstantExpr::getNot(C)); |
5543 | return new ICmpInst(Pred, And, Constant::getNullValue(Ty: And->getType())); |
5544 | } |
5545 | } |
5546 | |
5547 | if (match(V: Op1, P: m_ZExt(Op: m_Value(V&: A))) && |
5548 | (Op0->hasOneUse() || Op1->hasOneUse())) { |
5549 | // (B & (Pow2C-1)) == zext A --> A == trunc B |
5550 | // (B & (Pow2C-1)) != zext A --> A != trunc B |
5551 | const APInt *MaskC; |
5552 | if (match(V: Op0, P: m_And(L: m_Value(V&: B), R: m_LowBitMask(V&: MaskC))) && |
5553 | MaskC->countr_one() == A->getType()->getScalarSizeInBits()) |
5554 | return new ICmpInst(Pred, A, Builder.CreateTrunc(V: B, DestTy: A->getType())); |
5555 | } |
5556 | |
5557 | // (A >> C) == (B >> C) --> (A^B) u< (1 << C) |
5558 | // For lshr and ashr pairs. |
5559 | const APInt *AP1, *AP2; |
5560 | if ((match(V: Op0, P: m_OneUse(SubPattern: m_LShr(L: m_Value(V&: A), R: m_APIntAllowPoison(Res&: AP1)))) && |
5561 | match(V: Op1, P: m_OneUse(SubPattern: m_LShr(L: m_Value(V&: B), R: m_APIntAllowPoison(Res&: AP2))))) || |
5562 | (match(V: Op0, P: m_OneUse(SubPattern: m_AShr(L: m_Value(V&: A), R: m_APIntAllowPoison(Res&: AP1)))) && |
5563 | match(V: Op1, P: m_OneUse(SubPattern: m_AShr(L: m_Value(V&: B), R: m_APIntAllowPoison(Res&: AP2)))))) { |
5564 | if (AP1 != AP2) |
5565 | return nullptr; |
5566 | unsigned TypeBits = AP1->getBitWidth(); |
5567 | unsigned ShAmt = AP1->getLimitedValue(Limit: TypeBits); |
5568 | if (ShAmt < TypeBits && ShAmt != 0) { |
5569 | ICmpInst::Predicate NewPred = |
5570 | Pred == ICmpInst::ICMP_NE ? ICmpInst::ICMP_UGE : ICmpInst::ICMP_ULT; |
5571 | Value *Xor = Builder.CreateXor(LHS: A, RHS: B, Name: I.getName() + ".unshifted" ); |
5572 | APInt CmpVal = APInt::getOneBitSet(numBits: TypeBits, BitNo: ShAmt); |
5573 | return new ICmpInst(NewPred, Xor, ConstantInt::get(Ty: A->getType(), V: CmpVal)); |
5574 | } |
5575 | } |
5576 | |
5577 | // (A << C) == (B << C) --> ((A^B) & (~0U >> C)) == 0 |
5578 | ConstantInt *Cst1; |
5579 | if (match(V: Op0, P: m_OneUse(SubPattern: m_Shl(L: m_Value(V&: A), R: m_ConstantInt(CI&: Cst1)))) && |
5580 | match(V: Op1, P: m_OneUse(SubPattern: m_Shl(L: m_Value(V&: B), R: m_Specific(V: Cst1))))) { |
5581 | unsigned TypeBits = Cst1->getBitWidth(); |
5582 | unsigned ShAmt = (unsigned)Cst1->getLimitedValue(Limit: TypeBits); |
5583 | if (ShAmt < TypeBits && ShAmt != 0) { |
5584 | Value *Xor = Builder.CreateXor(LHS: A, RHS: B, Name: I.getName() + ".unshifted" ); |
5585 | APInt AndVal = APInt::getLowBitsSet(numBits: TypeBits, loBitsSet: TypeBits - ShAmt); |
5586 | Value *And = Builder.CreateAnd(LHS: Xor, RHS: Builder.getInt(AI: AndVal), |
5587 | Name: I.getName() + ".mask" ); |
5588 | return new ICmpInst(Pred, And, Constant::getNullValue(Ty: Cst1->getType())); |
5589 | } |
5590 | } |
5591 | |
5592 | // Transform "icmp eq (trunc (lshr(X, cst1)), cst" to |
5593 | // "icmp (and X, mask), cst" |
5594 | uint64_t ShAmt = 0; |
5595 | if (Op0->hasOneUse() && |
5596 | match(V: Op0, P: m_Trunc(Op: m_OneUse(SubPattern: m_LShr(L: m_Value(V&: A), R: m_ConstantInt(V&: ShAmt))))) && |
5597 | match(V: Op1, P: m_ConstantInt(CI&: Cst1)) && |
5598 | // Only do this when A has multiple uses. This is most important to do |
5599 | // when it exposes other optimizations. |
5600 | !A->hasOneUse()) { |
5601 | unsigned ASize = cast<IntegerType>(Val: A->getType())->getPrimitiveSizeInBits(); |
5602 | |
5603 | if (ShAmt < ASize) { |
5604 | APInt MaskV = |
5605 | APInt::getLowBitsSet(numBits: ASize, loBitsSet: Op0->getType()->getPrimitiveSizeInBits()); |
5606 | MaskV <<= ShAmt; |
5607 | |
5608 | APInt CmpV = Cst1->getValue().zext(width: ASize); |
5609 | CmpV <<= ShAmt; |
5610 | |
5611 | Value *Mask = Builder.CreateAnd(LHS: A, RHS: Builder.getInt(AI: MaskV)); |
5612 | return new ICmpInst(Pred, Mask, Builder.getInt(AI: CmpV)); |
5613 | } |
5614 | } |
5615 | |
5616 | if (Instruction *ICmp = foldICmpIntrinsicWithIntrinsic(Cmp&: I, Builder)) |
5617 | return ICmp; |
5618 | |
5619 | // Match icmp eq (trunc (lshr A, BW), (ashr (trunc A), BW-1)), which checks the |
5620 | // top BW/2 + 1 bits are all the same. Create "A >=s INT_MIN && A <=s INT_MAX", |
5621 | // which we generate as "icmp ult (add A, 2^(BW-1)), 2^BW" to skip a few steps |
5622 | // of instcombine. |
5623 | unsigned BitWidth = Op0->getType()->getScalarSizeInBits(); |
5624 | if (match(V: Op0, P: m_AShr(L: m_Trunc(Op: m_Value(V&: A)), R: m_SpecificInt(V: BitWidth - 1))) && |
5625 | match(V: Op1, P: m_Trunc(Op: m_LShr(L: m_Specific(V: A), R: m_SpecificInt(V: BitWidth)))) && |
5626 | A->getType()->getScalarSizeInBits() == BitWidth * 2 && |
5627 | (I.getOperand(i_nocapture: 0)->hasOneUse() || I.getOperand(i_nocapture: 1)->hasOneUse())) { |
5628 | APInt C = APInt::getOneBitSet(numBits: BitWidth * 2, BitNo: BitWidth - 1); |
5629 | Value *Add = Builder.CreateAdd(LHS: A, RHS: ConstantInt::get(Ty: A->getType(), V: C)); |
5630 | return new ICmpInst(Pred == ICmpInst::ICMP_EQ ? ICmpInst::ICMP_ULT |
5631 | : ICmpInst::ICMP_UGE, |
5632 | Add, ConstantInt::get(Ty: A->getType(), V: C.shl(shiftAmt: 1))); |
5633 | } |
5634 | |
5635 | // Canonicalize: |
5636 | // Assume B_Pow2 != 0 |
5637 | // 1. A & B_Pow2 != B_Pow2 -> A & B_Pow2 == 0 |
5638 | // 2. A & B_Pow2 == B_Pow2 -> A & B_Pow2 != 0 |
5639 | if (match(V: Op0, P: m_c_And(L: m_Specific(V: Op1), R: m_Value())) && |
5640 | isKnownToBeAPowerOfTwo(V: Op1, /* OrZero */ false, Depth: 0, CxtI: &I)) |
5641 | return new ICmpInst(CmpInst::getInversePredicate(pred: Pred), Op0, |
5642 | ConstantInt::getNullValue(Ty: Op0->getType())); |
5643 | |
5644 | if (match(V: Op1, P: m_c_And(L: m_Specific(V: Op0), R: m_Value())) && |
5645 | isKnownToBeAPowerOfTwo(V: Op0, /* OrZero */ false, Depth: 0, CxtI: &I)) |
5646 | return new ICmpInst(CmpInst::getInversePredicate(pred: Pred), Op1, |
5647 | ConstantInt::getNullValue(Ty: Op1->getType())); |
5648 | |
5649 | // Canonicalize: |
5650 | // icmp eq/ne X, OneUse(rotate-right(X)) |
5651 | // -> icmp eq/ne X, rotate-left(X) |
5652 | // We generally try to convert rotate-right -> rotate-left, this just |
5653 | // canonicalizes another case. |
5654 | CmpInst::Predicate PredUnused = Pred; |
5655 | if (match(&I, m_c_ICmp(PredUnused, m_Value(A), |
5656 | m_OneUse(m_Intrinsic<Intrinsic::fshr>( |
5657 | m_Deferred(A), m_Deferred(A), m_Value(B)))))) |
5658 | return new ICmpInst( |
5659 | Pred, A, |
5660 | Builder.CreateIntrinsic(Op0->getType(), Intrinsic::fshl, {A, A, B})); |
5661 | |
5662 | // Canonicalize: |
5663 | // icmp eq/ne OneUse(A ^ Cst), B --> icmp eq/ne (A ^ B), Cst |
5664 | Constant *Cst; |
5665 | if (match(V: &I, P: m_c_ICmp(Pred&: PredUnused, |
5666 | L: m_OneUse(SubPattern: m_Xor(L: m_Value(V&: A), R: m_ImmConstant(C&: Cst))), |
5667 | R: m_CombineAnd(L: m_Value(V&: B), R: m_Unless(M: m_ImmConstant()))))) |
5668 | return new ICmpInst(Pred, Builder.CreateXor(LHS: A, RHS: B), Cst); |
5669 | |
5670 | { |
5671 | // (icmp eq/ne (and (add/sub/xor X, P2), P2), P2) |
5672 | auto m_Matcher = |
5673 | m_CombineOr(L: m_CombineOr(L: m_c_Add(L: m_Value(V&: B), R: m_Deferred(V: A)), |
5674 | R: m_c_Xor(L: m_Value(V&: B), R: m_Deferred(V: A))), |
5675 | R: m_Sub(L: m_Value(V&: B), R: m_Deferred(V: A))); |
5676 | std::optional<bool> IsZero = std::nullopt; |
5677 | if (match(V: &I, P: m_c_ICmp(Pred&: PredUnused, L: m_OneUse(SubPattern: m_c_And(L: m_Value(V&: A), R: m_Matcher)), |
5678 | R: m_Deferred(V: A)))) |
5679 | IsZero = false; |
5680 | // (icmp eq/ne (and (add/sub/xor X, P2), P2), 0) |
5681 | else if (match(V: &I, |
5682 | P: m_ICmp(Pred&: PredUnused, L: m_OneUse(SubPattern: m_c_And(L: m_Value(V&: A), R: m_Matcher)), |
5683 | R: m_Zero()))) |
5684 | IsZero = true; |
5685 | |
5686 | if (IsZero && isKnownToBeAPowerOfTwo(V: A, /* OrZero */ true, /*Depth*/ 0, CxtI: &I)) |
5687 | // (icmp eq/ne (and (add/sub/xor X, P2), P2), P2) |
5688 | // -> (icmp eq/ne (and X, P2), 0) |
5689 | // (icmp eq/ne (and (add/sub/xor X, P2), P2), 0) |
5690 | // -> (icmp eq/ne (and X, P2), P2) |
5691 | return new ICmpInst(Pred, Builder.CreateAnd(LHS: B, RHS: A), |
5692 | *IsZero ? A |
5693 | : ConstantInt::getNullValue(Ty: A->getType())); |
5694 | } |
5695 | |
5696 | return nullptr; |
5697 | } |
5698 | |
5699 | Instruction *InstCombinerImpl::foldICmpWithTrunc(ICmpInst &ICmp) { |
5700 | ICmpInst::Predicate Pred = ICmp.getPredicate(); |
5701 | Value *Op0 = ICmp.getOperand(i_nocapture: 0), *Op1 = ICmp.getOperand(i_nocapture: 1); |
5702 | |
5703 | // Try to canonicalize trunc + compare-to-constant into a mask + cmp. |
5704 | // The trunc masks high bits while the compare may effectively mask low bits. |
5705 | Value *X; |
5706 | const APInt *C; |
5707 | if (!match(V: Op0, P: m_OneUse(SubPattern: m_Trunc(Op: m_Value(V&: X)))) || !match(V: Op1, P: m_APInt(Res&: C))) |
5708 | return nullptr; |
5709 | |
5710 | // This matches patterns corresponding to tests of the signbit as well as: |
5711 | // (trunc X) u< C --> (X & -C) == 0 (are all masked-high-bits clear?) |
5712 | // (trunc X) u> C --> (X & ~C) != 0 (are any masked-high-bits set?) |
5713 | APInt Mask; |
5714 | if (decomposeBitTestICmp(LHS: Op0, RHS: Op1, Pred, X, Mask, LookThroughTrunc: true /* WithTrunc */)) { |
5715 | Value *And = Builder.CreateAnd(LHS: X, RHS: Mask); |
5716 | Constant *Zero = ConstantInt::getNullValue(Ty: X->getType()); |
5717 | return new ICmpInst(Pred, And, Zero); |
5718 | } |
5719 | |
5720 | unsigned SrcBits = X->getType()->getScalarSizeInBits(); |
5721 | if (Pred == ICmpInst::ICMP_ULT && C->isNegatedPowerOf2()) { |
5722 | // If C is a negative power-of-2 (high-bit mask): |
5723 | // (trunc X) u< C --> (X & C) != C (are any masked-high-bits clear?) |
5724 | Constant *MaskC = ConstantInt::get(Ty: X->getType(), V: C->zext(width: SrcBits)); |
5725 | Value *And = Builder.CreateAnd(LHS: X, RHS: MaskC); |
5726 | return new ICmpInst(ICmpInst::ICMP_NE, And, MaskC); |
5727 | } |
5728 | |
5729 | if (Pred == ICmpInst::ICMP_UGT && (~*C).isPowerOf2()) { |
5730 | // If C is not-of-power-of-2 (one clear bit): |
5731 | // (trunc X) u> C --> (X & (C+1)) == C+1 (are all masked-high-bits set?) |
5732 | Constant *MaskC = ConstantInt::get(Ty: X->getType(), V: (*C + 1).zext(width: SrcBits)); |
5733 | Value *And = Builder.CreateAnd(LHS: X, RHS: MaskC); |
5734 | return new ICmpInst(ICmpInst::ICMP_EQ, And, MaskC); |
5735 | } |
5736 | |
5737 | if (auto *II = dyn_cast<IntrinsicInst>(Val: X)) { |
5738 | if (II->getIntrinsicID() == Intrinsic::cttz || |
5739 | II->getIntrinsicID() == Intrinsic::ctlz) { |
5740 | unsigned MaxRet = SrcBits; |
5741 | // If the "is_zero_poison" argument is set, then we know at least |
5742 | // one bit is set in the input, so the result is always at least one |
5743 | // less than the full bitwidth of that input. |
5744 | if (match(V: II->getArgOperand(i: 1), P: m_One())) |
5745 | MaxRet--; |
5746 | |
5747 | // Make sure the destination is wide enough to hold the largest output of |
5748 | // the intrinsic. |
5749 | if (llvm::Log2_32(Value: MaxRet) + 1 <= Op0->getType()->getScalarSizeInBits()) |
5750 | if (Instruction *I = |
5751 | foldICmpIntrinsicWithConstant(Cmp&: ICmp, II, C: C->zext(width: SrcBits))) |
5752 | return I; |
5753 | } |
5754 | } |
5755 | |
5756 | return nullptr; |
5757 | } |
5758 | |
5759 | Instruction *InstCombinerImpl::foldICmpWithZextOrSext(ICmpInst &ICmp) { |
5760 | assert(isa<CastInst>(ICmp.getOperand(0)) && "Expected cast for operand 0" ); |
5761 | auto *CastOp0 = cast<CastInst>(Val: ICmp.getOperand(i_nocapture: 0)); |
5762 | Value *X; |
5763 | if (!match(V: CastOp0, P: m_ZExtOrSExt(Op: m_Value(V&: X)))) |
5764 | return nullptr; |
5765 | |
5766 | bool IsSignedExt = CastOp0->getOpcode() == Instruction::SExt; |
5767 | bool IsSignedCmp = ICmp.isSigned(); |
5768 | |
5769 | // icmp Pred (ext X), (ext Y) |
5770 | Value *Y; |
5771 | if (match(V: ICmp.getOperand(i_nocapture: 1), P: m_ZExtOrSExt(Op: m_Value(V&: Y)))) { |
5772 | bool IsZext0 = isa<ZExtInst>(Val: ICmp.getOperand(i_nocapture: 0)); |
5773 | bool IsZext1 = isa<ZExtInst>(Val: ICmp.getOperand(i_nocapture: 1)); |
5774 | |
5775 | if (IsZext0 != IsZext1) { |
5776 | // If X and Y and both i1 |
5777 | // (icmp eq/ne (zext X) (sext Y)) |
5778 | // eq -> (icmp eq (or X, Y), 0) |
5779 | // ne -> (icmp ne (or X, Y), 0) |
5780 | if (ICmp.isEquality() && X->getType()->isIntOrIntVectorTy(BitWidth: 1) && |
5781 | Y->getType()->isIntOrIntVectorTy(BitWidth: 1)) |
5782 | return new ICmpInst(ICmp.getPredicate(), Builder.CreateOr(LHS: X, RHS: Y), |
5783 | Constant::getNullValue(Ty: X->getType())); |
5784 | |
5785 | // If we have mismatched casts and zext has the nneg flag, we can |
5786 | // treat the "zext nneg" as "sext". Otherwise, we cannot fold and quit. |
5787 | |
5788 | auto *NonNegInst0 = dyn_cast<PossiblyNonNegInst>(Val: ICmp.getOperand(i_nocapture: 0)); |
5789 | auto *NonNegInst1 = dyn_cast<PossiblyNonNegInst>(Val: ICmp.getOperand(i_nocapture: 1)); |
5790 | |
5791 | bool IsNonNeg0 = NonNegInst0 && NonNegInst0->hasNonNeg(); |
5792 | bool IsNonNeg1 = NonNegInst1 && NonNegInst1->hasNonNeg(); |
5793 | |
5794 | if ((IsZext0 && IsNonNeg0) || (IsZext1 && IsNonNeg1)) |
5795 | IsSignedExt = true; |
5796 | else |
5797 | return nullptr; |
5798 | } |
5799 | |
5800 | // Not an extension from the same type? |
5801 | Type *XTy = X->getType(), *YTy = Y->getType(); |
5802 | if (XTy != YTy) { |
5803 | // One of the casts must have one use because we are creating a new cast. |
5804 | if (!ICmp.getOperand(i_nocapture: 0)->hasOneUse() && !ICmp.getOperand(i_nocapture: 1)->hasOneUse()) |
5805 | return nullptr; |
5806 | // Extend the narrower operand to the type of the wider operand. |
5807 | CastInst::CastOps CastOpcode = |
5808 | IsSignedExt ? Instruction::SExt : Instruction::ZExt; |
5809 | if (XTy->getScalarSizeInBits() < YTy->getScalarSizeInBits()) |
5810 | X = Builder.CreateCast(Op: CastOpcode, V: X, DestTy: YTy); |
5811 | else if (YTy->getScalarSizeInBits() < XTy->getScalarSizeInBits()) |
5812 | Y = Builder.CreateCast(Op: CastOpcode, V: Y, DestTy: XTy); |
5813 | else |
5814 | return nullptr; |
5815 | } |
5816 | |
5817 | // (zext X) == (zext Y) --> X == Y |
5818 | // (sext X) == (sext Y) --> X == Y |
5819 | if (ICmp.isEquality()) |
5820 | return new ICmpInst(ICmp.getPredicate(), X, Y); |
5821 | |
5822 | // A signed comparison of sign extended values simplifies into a |
5823 | // signed comparison. |
5824 | if (IsSignedCmp && IsSignedExt) |
5825 | return new ICmpInst(ICmp.getPredicate(), X, Y); |
5826 | |
5827 | // The other three cases all fold into an unsigned comparison. |
5828 | return new ICmpInst(ICmp.getUnsignedPredicate(), X, Y); |
5829 | } |
5830 | |
5831 | // Below here, we are only folding a compare with constant. |
5832 | auto *C = dyn_cast<Constant>(Val: ICmp.getOperand(i_nocapture: 1)); |
5833 | if (!C) |
5834 | return nullptr; |
5835 | |
5836 | // If a lossless truncate is possible... |
5837 | Type *SrcTy = CastOp0->getSrcTy(); |
5838 | Constant *Res = getLosslessTrunc(C, TruncTy: SrcTy, ExtOp: CastOp0->getOpcode()); |
5839 | if (Res) { |
5840 | if (ICmp.isEquality()) |
5841 | return new ICmpInst(ICmp.getPredicate(), X, Res); |
5842 | |
5843 | // A signed comparison of sign extended values simplifies into a |
5844 | // signed comparison. |
5845 | if (IsSignedExt && IsSignedCmp) |
5846 | return new ICmpInst(ICmp.getPredicate(), X, Res); |
5847 | |
5848 | // The other three cases all fold into an unsigned comparison. |
5849 | return new ICmpInst(ICmp.getUnsignedPredicate(), X, Res); |
5850 | } |
5851 | |
5852 | // The re-extended constant changed, partly changed (in the case of a vector), |
5853 | // or could not be determined to be equal (in the case of a constant |
5854 | // expression), so the constant cannot be represented in the shorter type. |
5855 | // All the cases that fold to true or false will have already been handled |
5856 | // by simplifyICmpInst, so only deal with the tricky case. |
5857 | if (IsSignedCmp || !IsSignedExt || !isa<ConstantInt>(Val: C)) |
5858 | return nullptr; |
5859 | |
5860 | // Is source op positive? |
5861 | // icmp ult (sext X), C --> icmp sgt X, -1 |
5862 | if (ICmp.getPredicate() == ICmpInst::ICMP_ULT) |
5863 | return new ICmpInst(CmpInst::ICMP_SGT, X, Constant::getAllOnesValue(Ty: SrcTy)); |
5864 | |
5865 | // Is source op negative? |
5866 | // icmp ugt (sext X), C --> icmp slt X, 0 |
5867 | assert(ICmp.getPredicate() == ICmpInst::ICMP_UGT && "ICmp should be folded!" ); |
5868 | return new ICmpInst(CmpInst::ICMP_SLT, X, Constant::getNullValue(Ty: SrcTy)); |
5869 | } |
5870 | |
5871 | /// Handle icmp (cast x), (cast or constant). |
5872 | Instruction *InstCombinerImpl::foldICmpWithCastOp(ICmpInst &ICmp) { |
5873 | // If any operand of ICmp is a inttoptr roundtrip cast then remove it as |
5874 | // icmp compares only pointer's value. |
5875 | // icmp (inttoptr (ptrtoint p1)), p2 --> icmp p1, p2. |
5876 | Value *SimplifiedOp0 = simplifyIntToPtrRoundTripCast(Val: ICmp.getOperand(i_nocapture: 0)); |
5877 | Value *SimplifiedOp1 = simplifyIntToPtrRoundTripCast(Val: ICmp.getOperand(i_nocapture: 1)); |
5878 | if (SimplifiedOp0 || SimplifiedOp1) |
5879 | return new ICmpInst(ICmp.getPredicate(), |
5880 | SimplifiedOp0 ? SimplifiedOp0 : ICmp.getOperand(i_nocapture: 0), |
5881 | SimplifiedOp1 ? SimplifiedOp1 : ICmp.getOperand(i_nocapture: 1)); |
5882 | |
5883 | auto *CastOp0 = dyn_cast<CastInst>(Val: ICmp.getOperand(i_nocapture: 0)); |
5884 | if (!CastOp0) |
5885 | return nullptr; |
5886 | if (!isa<Constant>(Val: ICmp.getOperand(i_nocapture: 1)) && !isa<CastInst>(Val: ICmp.getOperand(i_nocapture: 1))) |
5887 | return nullptr; |
5888 | |
5889 | Value *Op0Src = CastOp0->getOperand(i_nocapture: 0); |
5890 | Type *SrcTy = CastOp0->getSrcTy(); |
5891 | Type *DestTy = CastOp0->getDestTy(); |
5892 | |
5893 | // Turn icmp (ptrtoint x), (ptrtoint/c) into a compare of the input if the |
5894 | // integer type is the same size as the pointer type. |
5895 | auto CompatibleSizes = [&](Type *SrcTy, Type *DestTy) { |
5896 | if (isa<VectorType>(Val: SrcTy)) { |
5897 | SrcTy = cast<VectorType>(Val: SrcTy)->getElementType(); |
5898 | DestTy = cast<VectorType>(Val: DestTy)->getElementType(); |
5899 | } |
5900 | return DL.getPointerTypeSizeInBits(SrcTy) == DestTy->getIntegerBitWidth(); |
5901 | }; |
5902 | if (CastOp0->getOpcode() == Instruction::PtrToInt && |
5903 | CompatibleSizes(SrcTy, DestTy)) { |
5904 | Value *NewOp1 = nullptr; |
5905 | if (auto *PtrToIntOp1 = dyn_cast<PtrToIntOperator>(Val: ICmp.getOperand(i_nocapture: 1))) { |
5906 | Value *PtrSrc = PtrToIntOp1->getOperand(i_nocapture: 0); |
5907 | if (PtrSrc->getType() == Op0Src->getType()) |
5908 | NewOp1 = PtrToIntOp1->getOperand(i_nocapture: 0); |
5909 | } else if (auto *RHSC = dyn_cast<Constant>(Val: ICmp.getOperand(i_nocapture: 1))) { |
5910 | NewOp1 = ConstantExpr::getIntToPtr(C: RHSC, Ty: SrcTy); |
5911 | } |
5912 | |
5913 | if (NewOp1) |
5914 | return new ICmpInst(ICmp.getPredicate(), Op0Src, NewOp1); |
5915 | } |
5916 | |
5917 | if (Instruction *R = foldICmpWithTrunc(ICmp)) |
5918 | return R; |
5919 | |
5920 | return foldICmpWithZextOrSext(ICmp); |
5921 | } |
5922 | |
5923 | static bool isNeutralValue(Instruction::BinaryOps BinaryOp, Value *RHS, bool IsSigned) { |
5924 | switch (BinaryOp) { |
5925 | default: |
5926 | llvm_unreachable("Unsupported binary op" ); |
5927 | case Instruction::Add: |
5928 | case Instruction::Sub: |
5929 | return match(V: RHS, P: m_Zero()); |
5930 | case Instruction::Mul: |
5931 | return !(RHS->getType()->isIntOrIntVectorTy(BitWidth: 1) && IsSigned) && |
5932 | match(V: RHS, P: m_One()); |
5933 | } |
5934 | } |
5935 | |
5936 | OverflowResult |
5937 | InstCombinerImpl::computeOverflow(Instruction::BinaryOps BinaryOp, |
5938 | bool IsSigned, Value *LHS, Value *RHS, |
5939 | Instruction *CxtI) const { |
5940 | switch (BinaryOp) { |
5941 | default: |
5942 | llvm_unreachable("Unsupported binary op" ); |
5943 | case Instruction::Add: |
5944 | if (IsSigned) |
5945 | return computeOverflowForSignedAdd(LHS, RHS, CxtI); |
5946 | else |
5947 | return computeOverflowForUnsignedAdd(LHS, RHS, CxtI); |
5948 | case Instruction::Sub: |
5949 | if (IsSigned) |
5950 | return computeOverflowForSignedSub(LHS, RHS, CxtI); |
5951 | else |
5952 | return computeOverflowForUnsignedSub(LHS, RHS, CxtI); |
5953 | case Instruction::Mul: |
5954 | if (IsSigned) |
5955 | return computeOverflowForSignedMul(LHS, RHS, CxtI); |
5956 | else |
5957 | return computeOverflowForUnsignedMul(LHS, RHS, CxtI); |
5958 | } |
5959 | } |
5960 | |
5961 | bool InstCombinerImpl::OptimizeOverflowCheck(Instruction::BinaryOps BinaryOp, |
5962 | bool IsSigned, Value *LHS, |
5963 | Value *RHS, Instruction &OrigI, |
5964 | Value *&Result, |
5965 | Constant *&Overflow) { |
5966 | if (OrigI.isCommutative() && isa<Constant>(Val: LHS) && !isa<Constant>(Val: RHS)) |
5967 | std::swap(a&: LHS, b&: RHS); |
5968 | |
5969 | // If the overflow check was an add followed by a compare, the insertion point |
5970 | // may be pointing to the compare. We want to insert the new instructions |
5971 | // before the add in case there are uses of the add between the add and the |
5972 | // compare. |
5973 | Builder.SetInsertPoint(&OrigI); |
5974 | |
5975 | Type *OverflowTy = Type::getInt1Ty(C&: LHS->getContext()); |
5976 | if (auto *LHSTy = dyn_cast<VectorType>(Val: LHS->getType())) |
5977 | OverflowTy = VectorType::get(ElementType: OverflowTy, EC: LHSTy->getElementCount()); |
5978 | |
5979 | if (isNeutralValue(BinaryOp, RHS, IsSigned)) { |
5980 | Result = LHS; |
5981 | Overflow = ConstantInt::getFalse(Ty: OverflowTy); |
5982 | return true; |
5983 | } |
5984 | |
5985 | switch (computeOverflow(BinaryOp, IsSigned, LHS, RHS, CxtI: &OrigI)) { |
5986 | case OverflowResult::MayOverflow: |
5987 | return false; |
5988 | case OverflowResult::AlwaysOverflowsLow: |
5989 | case OverflowResult::AlwaysOverflowsHigh: |
5990 | Result = Builder.CreateBinOp(Opc: BinaryOp, LHS, RHS); |
5991 | Result->takeName(V: &OrigI); |
5992 | Overflow = ConstantInt::getTrue(Ty: OverflowTy); |
5993 | return true; |
5994 | case OverflowResult::NeverOverflows: |
5995 | Result = Builder.CreateBinOp(Opc: BinaryOp, LHS, RHS); |
5996 | Result->takeName(V: &OrigI); |
5997 | Overflow = ConstantInt::getFalse(Ty: OverflowTy); |
5998 | if (auto *Inst = dyn_cast<Instruction>(Val: Result)) { |
5999 | if (IsSigned) |
6000 | Inst->setHasNoSignedWrap(); |
6001 | else |
6002 | Inst->setHasNoUnsignedWrap(); |
6003 | } |
6004 | return true; |
6005 | } |
6006 | |
6007 | llvm_unreachable("Unexpected overflow result" ); |
6008 | } |
6009 | |
6010 | /// Recognize and process idiom involving test for multiplication |
6011 | /// overflow. |
6012 | /// |
6013 | /// The caller has matched a pattern of the form: |
6014 | /// I = cmp u (mul(zext A, zext B), V |
6015 | /// The function checks if this is a test for overflow and if so replaces |
6016 | /// multiplication with call to 'mul.with.overflow' intrinsic. |
6017 | /// |
6018 | /// \param I Compare instruction. |
6019 | /// \param MulVal Result of 'mult' instruction. It is one of the arguments of |
6020 | /// the compare instruction. Must be of integer type. |
6021 | /// \param OtherVal The other argument of compare instruction. |
6022 | /// \returns Instruction which must replace the compare instruction, NULL if no |
6023 | /// replacement required. |
6024 | static Instruction *processUMulZExtIdiom(ICmpInst &I, Value *MulVal, |
6025 | const APInt *OtherVal, |
6026 | InstCombinerImpl &IC) { |
6027 | // Don't bother doing this transformation for pointers, don't do it for |
6028 | // vectors. |
6029 | if (!isa<IntegerType>(Val: MulVal->getType())) |
6030 | return nullptr; |
6031 | |
6032 | auto *MulInstr = dyn_cast<Instruction>(Val: MulVal); |
6033 | if (!MulInstr) |
6034 | return nullptr; |
6035 | assert(MulInstr->getOpcode() == Instruction::Mul); |
6036 | |
6037 | auto *LHS = cast<ZExtInst>(Val: MulInstr->getOperand(i: 0)), |
6038 | *RHS = cast<ZExtInst>(Val: MulInstr->getOperand(i: 1)); |
6039 | assert(LHS->getOpcode() == Instruction::ZExt); |
6040 | assert(RHS->getOpcode() == Instruction::ZExt); |
6041 | Value *A = LHS->getOperand(i_nocapture: 0), *B = RHS->getOperand(i_nocapture: 0); |
6042 | |
6043 | // Calculate type and width of the result produced by mul.with.overflow. |
6044 | Type *TyA = A->getType(), *TyB = B->getType(); |
6045 | unsigned WidthA = TyA->getPrimitiveSizeInBits(), |
6046 | WidthB = TyB->getPrimitiveSizeInBits(); |
6047 | unsigned MulWidth; |
6048 | Type *MulType; |
6049 | if (WidthB > WidthA) { |
6050 | MulWidth = WidthB; |
6051 | MulType = TyB; |
6052 | } else { |
6053 | MulWidth = WidthA; |
6054 | MulType = TyA; |
6055 | } |
6056 | |
6057 | // In order to replace the original mul with a narrower mul.with.overflow, |
6058 | // all uses must ignore upper bits of the product. The number of used low |
6059 | // bits must be not greater than the width of mul.with.overflow. |
6060 | if (MulVal->hasNUsesOrMore(N: 2)) |
6061 | for (User *U : MulVal->users()) { |
6062 | if (U == &I) |
6063 | continue; |
6064 | if (TruncInst *TI = dyn_cast<TruncInst>(Val: U)) { |
6065 | // Check if truncation ignores bits above MulWidth. |
6066 | unsigned TruncWidth = TI->getType()->getPrimitiveSizeInBits(); |
6067 | if (TruncWidth > MulWidth) |
6068 | return nullptr; |
6069 | } else if (BinaryOperator *BO = dyn_cast<BinaryOperator>(Val: U)) { |
6070 | // Check if AND ignores bits above MulWidth. |
6071 | if (BO->getOpcode() != Instruction::And) |
6072 | return nullptr; |
6073 | if (ConstantInt *CI = dyn_cast<ConstantInt>(Val: BO->getOperand(i_nocapture: 1))) { |
6074 | const APInt &CVal = CI->getValue(); |
6075 | if (CVal.getBitWidth() - CVal.countl_zero() > MulWidth) |
6076 | return nullptr; |
6077 | } else { |
6078 | // In this case we could have the operand of the binary operation |
6079 | // being defined in another block, and performing the replacement |
6080 | // could break the dominance relation. |
6081 | return nullptr; |
6082 | } |
6083 | } else { |
6084 | // Other uses prohibit this transformation. |
6085 | return nullptr; |
6086 | } |
6087 | } |
6088 | |
6089 | // Recognize patterns |
6090 | switch (I.getPredicate()) { |
6091 | case ICmpInst::ICMP_UGT: { |
6092 | // Recognize pattern: |
6093 | // mulval = mul(zext A, zext B) |
6094 | // cmp ugt mulval, max |
6095 | APInt MaxVal = APInt::getMaxValue(numBits: MulWidth); |
6096 | MaxVal = MaxVal.zext(width: OtherVal->getBitWidth()); |
6097 | if (MaxVal.eq(RHS: *OtherVal)) |
6098 | break; // Recognized |
6099 | return nullptr; |
6100 | } |
6101 | |
6102 | case ICmpInst::ICMP_ULT: { |
6103 | // Recognize pattern: |
6104 | // mulval = mul(zext A, zext B) |
6105 | // cmp ule mulval, max + 1 |
6106 | APInt MaxVal = APInt::getOneBitSet(numBits: OtherVal->getBitWidth(), BitNo: MulWidth); |
6107 | if (MaxVal.eq(RHS: *OtherVal)) |
6108 | break; // Recognized |
6109 | return nullptr; |
6110 | } |
6111 | |
6112 | default: |
6113 | return nullptr; |
6114 | } |
6115 | |
6116 | InstCombiner::BuilderTy &Builder = IC.Builder; |
6117 | Builder.SetInsertPoint(MulInstr); |
6118 | |
6119 | // Replace: mul(zext A, zext B) --> mul.with.overflow(A, B) |
6120 | Value *MulA = A, *MulB = B; |
6121 | if (WidthA < MulWidth) |
6122 | MulA = Builder.CreateZExt(V: A, DestTy: MulType); |
6123 | if (WidthB < MulWidth) |
6124 | MulB = Builder.CreateZExt(V: B, DestTy: MulType); |
6125 | Function *F = Intrinsic::getDeclaration( |
6126 | M: I.getModule(), Intrinsic::id: umul_with_overflow, Tys: MulType); |
6127 | CallInst *Call = Builder.CreateCall(Callee: F, Args: {MulA, MulB}, Name: "umul" ); |
6128 | IC.addToWorklist(I: MulInstr); |
6129 | |
6130 | // If there are uses of mul result other than the comparison, we know that |
6131 | // they are truncation or binary AND. Change them to use result of |
6132 | // mul.with.overflow and adjust properly mask/size. |
6133 | if (MulVal->hasNUsesOrMore(N: 2)) { |
6134 | Value *Mul = Builder.CreateExtractValue(Agg: Call, Idxs: 0, Name: "umul.value" ); |
6135 | for (User *U : make_early_inc_range(Range: MulVal->users())) { |
6136 | if (U == &I) |
6137 | continue; |
6138 | if (TruncInst *TI = dyn_cast<TruncInst>(Val: U)) { |
6139 | if (TI->getType()->getPrimitiveSizeInBits() == MulWidth) |
6140 | IC.replaceInstUsesWith(I&: *TI, V: Mul); |
6141 | else |
6142 | TI->setOperand(i_nocapture: 0, Val_nocapture: Mul); |
6143 | } else if (BinaryOperator *BO = dyn_cast<BinaryOperator>(Val: U)) { |
6144 | assert(BO->getOpcode() == Instruction::And); |
6145 | // Replace (mul & mask) --> zext (mul.with.overflow & short_mask) |
6146 | ConstantInt *CI = cast<ConstantInt>(Val: BO->getOperand(i_nocapture: 1)); |
6147 | APInt ShortMask = CI->getValue().trunc(width: MulWidth); |
6148 | Value *ShortAnd = Builder.CreateAnd(LHS: Mul, RHS: ShortMask); |
6149 | Value *Zext = Builder.CreateZExt(V: ShortAnd, DestTy: BO->getType()); |
6150 | IC.replaceInstUsesWith(I&: *BO, V: Zext); |
6151 | } else { |
6152 | llvm_unreachable("Unexpected Binary operation" ); |
6153 | } |
6154 | IC.addToWorklist(I: cast<Instruction>(Val: U)); |
6155 | } |
6156 | } |
6157 | |
6158 | // The original icmp gets replaced with the overflow value, maybe inverted |
6159 | // depending on predicate. |
6160 | if (I.getPredicate() == ICmpInst::ICMP_ULT) { |
6161 | Value *Res = Builder.CreateExtractValue(Agg: Call, Idxs: 1); |
6162 | return BinaryOperator::CreateNot(Op: Res); |
6163 | } |
6164 | |
6165 | return ExtractValueInst::Create(Agg: Call, Idxs: 1); |
6166 | } |
6167 | |
6168 | /// When performing a comparison against a constant, it is possible that not all |
6169 | /// the bits in the LHS are demanded. This helper method computes the mask that |
6170 | /// IS demanded. |
6171 | static APInt getDemandedBitsLHSMask(ICmpInst &I, unsigned BitWidth) { |
6172 | const APInt *RHS; |
6173 | if (!match(V: I.getOperand(i_nocapture: 1), P: m_APInt(Res&: RHS))) |
6174 | return APInt::getAllOnes(numBits: BitWidth); |
6175 | |
6176 | // If this is a normal comparison, it demands all bits. If it is a sign bit |
6177 | // comparison, it only demands the sign bit. |
6178 | bool UnusedBit; |
6179 | if (isSignBitCheck(Pred: I.getPredicate(), RHS: *RHS, TrueIfSigned&: UnusedBit)) |
6180 | return APInt::getSignMask(BitWidth); |
6181 | |
6182 | switch (I.getPredicate()) { |
6183 | // For a UGT comparison, we don't care about any bits that |
6184 | // correspond to the trailing ones of the comparand. The value of these |
6185 | // bits doesn't impact the outcome of the comparison, because any value |
6186 | // greater than the RHS must differ in a bit higher than these due to carry. |
6187 | case ICmpInst::ICMP_UGT: |
6188 | return APInt::getBitsSetFrom(numBits: BitWidth, loBit: RHS->countr_one()); |
6189 | |
6190 | // Similarly, for a ULT comparison, we don't care about the trailing zeros. |
6191 | // Any value less than the RHS must differ in a higher bit because of carries. |
6192 | case ICmpInst::ICMP_ULT: |
6193 | return APInt::getBitsSetFrom(numBits: BitWidth, loBit: RHS->countr_zero()); |
6194 | |
6195 | default: |
6196 | return APInt::getAllOnes(numBits: BitWidth); |
6197 | } |
6198 | } |
6199 | |
6200 | /// Check that one use is in the same block as the definition and all |
6201 | /// other uses are in blocks dominated by a given block. |
6202 | /// |
6203 | /// \param DI Definition |
6204 | /// \param UI Use |
6205 | /// \param DB Block that must dominate all uses of \p DI outside |
6206 | /// the parent block |
6207 | /// \return true when \p UI is the only use of \p DI in the parent block |
6208 | /// and all other uses of \p DI are in blocks dominated by \p DB. |
6209 | /// |
6210 | bool InstCombinerImpl::dominatesAllUses(const Instruction *DI, |
6211 | const Instruction *UI, |
6212 | const BasicBlock *DB) const { |
6213 | assert(DI && UI && "Instruction not defined\n" ); |
6214 | // Ignore incomplete definitions. |
6215 | if (!DI->getParent()) |
6216 | return false; |
6217 | // DI and UI must be in the same block. |
6218 | if (DI->getParent() != UI->getParent()) |
6219 | return false; |
6220 | // Protect from self-referencing blocks. |
6221 | if (DI->getParent() == DB) |
6222 | return false; |
6223 | for (const User *U : DI->users()) { |
6224 | auto *Usr = cast<Instruction>(Val: U); |
6225 | if (Usr != UI && !DT.dominates(A: DB, B: Usr->getParent())) |
6226 | return false; |
6227 | } |
6228 | return true; |
6229 | } |
6230 | |
6231 | /// Return true when the instruction sequence within a block is select-cmp-br. |
6232 | static bool isChainSelectCmpBranch(const SelectInst *SI) { |
6233 | const BasicBlock *BB = SI->getParent(); |
6234 | if (!BB) |
6235 | return false; |
6236 | auto *BI = dyn_cast_or_null<BranchInst>(Val: BB->getTerminator()); |
6237 | if (!BI || BI->getNumSuccessors() != 2) |
6238 | return false; |
6239 | auto *IC = dyn_cast<ICmpInst>(Val: BI->getCondition()); |
6240 | if (!IC || (IC->getOperand(i_nocapture: 0) != SI && IC->getOperand(i_nocapture: 1) != SI)) |
6241 | return false; |
6242 | return true; |
6243 | } |
6244 | |
6245 | /// True when a select result is replaced by one of its operands |
6246 | /// in select-icmp sequence. This will eventually result in the elimination |
6247 | /// of the select. |
6248 | /// |
6249 | /// \param SI Select instruction |
6250 | /// \param Icmp Compare instruction |
6251 | /// \param SIOpd Operand that replaces the select |
6252 | /// |
6253 | /// Notes: |
6254 | /// - The replacement is global and requires dominator information |
6255 | /// - The caller is responsible for the actual replacement |
6256 | /// |
6257 | /// Example: |
6258 | /// |
6259 | /// entry: |
6260 | /// %4 = select i1 %3, %C* %0, %C* null |
6261 | /// %5 = icmp eq %C* %4, null |
6262 | /// br i1 %5, label %9, label %7 |
6263 | /// ... |
6264 | /// ; <label>:7 ; preds = %entry |
6265 | /// %8 = getelementptr inbounds %C* %4, i64 0, i32 0 |
6266 | /// ... |
6267 | /// |
6268 | /// can be transformed to |
6269 | /// |
6270 | /// %5 = icmp eq %C* %0, null |
6271 | /// %6 = select i1 %3, i1 %5, i1 true |
6272 | /// br i1 %6, label %9, label %7 |
6273 | /// ... |
6274 | /// ; <label>:7 ; preds = %entry |
6275 | /// %8 = getelementptr inbounds %C* %0, i64 0, i32 0 // replace by %0! |
6276 | /// |
6277 | /// Similar when the first operand of the select is a constant or/and |
6278 | /// the compare is for not equal rather than equal. |
6279 | /// |
6280 | /// NOTE: The function is only called when the select and compare constants |
6281 | /// are equal, the optimization can work only for EQ predicates. This is not a |
6282 | /// major restriction since a NE compare should be 'normalized' to an equal |
6283 | /// compare, which usually happens in the combiner and test case |
6284 | /// select-cmp-br.ll checks for it. |
6285 | bool InstCombinerImpl::replacedSelectWithOperand(SelectInst *SI, |
6286 | const ICmpInst *Icmp, |
6287 | const unsigned SIOpd) { |
6288 | assert((SIOpd == 1 || SIOpd == 2) && "Invalid select operand!" ); |
6289 | if (isChainSelectCmpBranch(SI) && Icmp->getPredicate() == ICmpInst::ICMP_EQ) { |
6290 | BasicBlock *Succ = SI->getParent()->getTerminator()->getSuccessor(Idx: 1); |
6291 | // The check for the single predecessor is not the best that can be |
6292 | // done. But it protects efficiently against cases like when SI's |
6293 | // home block has two successors, Succ and Succ1, and Succ1 predecessor |
6294 | // of Succ. Then SI can't be replaced by SIOpd because the use that gets |
6295 | // replaced can be reached on either path. So the uniqueness check |
6296 | // guarantees that the path all uses of SI (outside SI's parent) are on |
6297 | // is disjoint from all other paths out of SI. But that information |
6298 | // is more expensive to compute, and the trade-off here is in favor |
6299 | // of compile-time. It should also be noticed that we check for a single |
6300 | // predecessor and not only uniqueness. This to handle the situation when |
6301 | // Succ and Succ1 points to the same basic block. |
6302 | if (Succ->getSinglePredecessor() && dominatesAllUses(DI: SI, UI: Icmp, DB: Succ)) { |
6303 | NumSel++; |
6304 | SI->replaceUsesOutsideBlock(V: SI->getOperand(i_nocapture: SIOpd), BB: SI->getParent()); |
6305 | return true; |
6306 | } |
6307 | } |
6308 | return false; |
6309 | } |
6310 | |
6311 | /// Try to fold the comparison based on range information we can get by checking |
6312 | /// whether bits are known to be zero or one in the inputs. |
6313 | Instruction *InstCombinerImpl::foldICmpUsingKnownBits(ICmpInst &I) { |
6314 | Value *Op0 = I.getOperand(i_nocapture: 0), *Op1 = I.getOperand(i_nocapture: 1); |
6315 | Type *Ty = Op0->getType(); |
6316 | ICmpInst::Predicate Pred = I.getPredicate(); |
6317 | |
6318 | // Get scalar or pointer size. |
6319 | unsigned BitWidth = Ty->isIntOrIntVectorTy() |
6320 | ? Ty->getScalarSizeInBits() |
6321 | : DL.getPointerTypeSizeInBits(Ty->getScalarType()); |
6322 | |
6323 | if (!BitWidth) |
6324 | return nullptr; |
6325 | |
6326 | KnownBits Op0Known(BitWidth); |
6327 | KnownBits Op1Known(BitWidth); |
6328 | |
6329 | { |
6330 | // Don't use dominating conditions when folding icmp using known bits. This |
6331 | // may convert signed into unsigned predicates in ways that other passes |
6332 | // (especially IndVarSimplify) may not be able to reliably undo. |
6333 | SQ.DC = nullptr; |
6334 | auto _ = make_scope_exit(F: [&]() { SQ.DC = &DC; }); |
6335 | if (SimplifyDemandedBits(I: &I, Op: 0, DemandedMask: getDemandedBitsLHSMask(I, BitWidth), |
6336 | Known&: Op0Known, Depth: 0)) |
6337 | return &I; |
6338 | |
6339 | if (SimplifyDemandedBits(I: &I, Op: 1, DemandedMask: APInt::getAllOnes(numBits: BitWidth), Known&: Op1Known, Depth: 0)) |
6340 | return &I; |
6341 | } |
6342 | |
6343 | // Given the known and unknown bits, compute a range that the LHS could be |
6344 | // in. Compute the Min, Max and RHS values based on the known bits. For the |
6345 | // EQ and NE we use unsigned values. |
6346 | APInt Op0Min(BitWidth, 0), Op0Max(BitWidth, 0); |
6347 | APInt Op1Min(BitWidth, 0), Op1Max(BitWidth, 0); |
6348 | if (I.isSigned()) { |
6349 | Op0Min = Op0Known.getSignedMinValue(); |
6350 | Op0Max = Op0Known.getSignedMaxValue(); |
6351 | Op1Min = Op1Known.getSignedMinValue(); |
6352 | Op1Max = Op1Known.getSignedMaxValue(); |
6353 | } else { |
6354 | Op0Min = Op0Known.getMinValue(); |
6355 | Op0Max = Op0Known.getMaxValue(); |
6356 | Op1Min = Op1Known.getMinValue(); |
6357 | Op1Max = Op1Known.getMaxValue(); |
6358 | } |
6359 | |
6360 | // If Min and Max are known to be the same, then SimplifyDemandedBits figured |
6361 | // out that the LHS or RHS is a constant. Constant fold this now, so that |
6362 | // code below can assume that Min != Max. |
6363 | if (!isa<Constant>(Val: Op0) && Op0Min == Op0Max) |
6364 | return new ICmpInst(Pred, ConstantExpr::getIntegerValue(Ty, V: Op0Min), Op1); |
6365 | if (!isa<Constant>(Val: Op1) && Op1Min == Op1Max) |
6366 | return new ICmpInst(Pred, Op0, ConstantExpr::getIntegerValue(Ty, V: Op1Min)); |
6367 | |
6368 | // Don't break up a clamp pattern -- (min(max X, Y), Z) -- by replacing a |
6369 | // min/max canonical compare with some other compare. That could lead to |
6370 | // conflict with select canonicalization and infinite looping. |
6371 | // FIXME: This constraint may go away if min/max intrinsics are canonical. |
6372 | auto isMinMaxCmp = [&](Instruction &Cmp) { |
6373 | if (!Cmp.hasOneUse()) |
6374 | return false; |
6375 | Value *A, *B; |
6376 | SelectPatternFlavor SPF = matchSelectPattern(V: Cmp.user_back(), LHS&: A, RHS&: B).Flavor; |
6377 | if (!SelectPatternResult::isMinOrMax(SPF)) |
6378 | return false; |
6379 | return match(V: Op0, P: m_MaxOrMin(L: m_Value(), R: m_Value())) || |
6380 | match(V: Op1, P: m_MaxOrMin(L: m_Value(), R: m_Value())); |
6381 | }; |
6382 | if (!isMinMaxCmp(I)) { |
6383 | switch (Pred) { |
6384 | default: |
6385 | break; |
6386 | case ICmpInst::ICMP_ULT: { |
6387 | if (Op1Min == Op0Max) // A <u B -> A != B if max(A) == min(B) |
6388 | return new ICmpInst(ICmpInst::ICMP_NE, Op0, Op1); |
6389 | const APInt *CmpC; |
6390 | if (match(V: Op1, P: m_APInt(Res&: CmpC))) { |
6391 | // A <u C -> A == C-1 if min(A)+1 == C |
6392 | if (*CmpC == Op0Min + 1) |
6393 | return new ICmpInst(ICmpInst::ICMP_EQ, Op0, |
6394 | ConstantInt::get(Ty: Op1->getType(), V: *CmpC - 1)); |
6395 | // X <u C --> X == 0, if the number of zero bits in the bottom of X |
6396 | // exceeds the log2 of C. |
6397 | if (Op0Known.countMinTrailingZeros() >= CmpC->ceilLogBase2()) |
6398 | return new ICmpInst(ICmpInst::ICMP_EQ, Op0, |
6399 | Constant::getNullValue(Ty: Op1->getType())); |
6400 | } |
6401 | break; |
6402 | } |
6403 | case ICmpInst::ICMP_UGT: { |
6404 | if (Op1Max == Op0Min) // A >u B -> A != B if min(A) == max(B) |
6405 | return new ICmpInst(ICmpInst::ICMP_NE, Op0, Op1); |
6406 | const APInt *CmpC; |
6407 | if (match(V: Op1, P: m_APInt(Res&: CmpC))) { |
6408 | // A >u C -> A == C+1 if max(a)-1 == C |
6409 | if (*CmpC == Op0Max - 1) |
6410 | return new ICmpInst(ICmpInst::ICMP_EQ, Op0, |
6411 | ConstantInt::get(Ty: Op1->getType(), V: *CmpC + 1)); |
6412 | // X >u C --> X != 0, if the number of zero bits in the bottom of X |
6413 | // exceeds the log2 of C. |
6414 | if (Op0Known.countMinTrailingZeros() >= CmpC->getActiveBits()) |
6415 | return new ICmpInst(ICmpInst::ICMP_NE, Op0, |
6416 | Constant::getNullValue(Ty: Op1->getType())); |
6417 | } |
6418 | break; |
6419 | } |
6420 | case ICmpInst::ICMP_SLT: { |
6421 | if (Op1Min == Op0Max) // A <s B -> A != B if max(A) == min(B) |
6422 | return new ICmpInst(ICmpInst::ICMP_NE, Op0, Op1); |
6423 | const APInt *CmpC; |
6424 | if (match(V: Op1, P: m_APInt(Res&: CmpC))) { |
6425 | if (*CmpC == Op0Min + 1) // A <s C -> A == C-1 if min(A)+1 == C |
6426 | return new ICmpInst(ICmpInst::ICMP_EQ, Op0, |
6427 | ConstantInt::get(Ty: Op1->getType(), V: *CmpC - 1)); |
6428 | } |
6429 | break; |
6430 | } |
6431 | case ICmpInst::ICMP_SGT: { |
6432 | if (Op1Max == Op0Min) // A >s B -> A != B if min(A) == max(B) |
6433 | return new ICmpInst(ICmpInst::ICMP_NE, Op0, Op1); |
6434 | const APInt *CmpC; |
6435 | if (match(V: Op1, P: m_APInt(Res&: CmpC))) { |
6436 | if (*CmpC == Op0Max - 1) // A >s C -> A == C+1 if max(A)-1 == C |
6437 | return new ICmpInst(ICmpInst::ICMP_EQ, Op0, |
6438 | ConstantInt::get(Ty: Op1->getType(), V: *CmpC + 1)); |
6439 | } |
6440 | break; |
6441 | } |
6442 | } |
6443 | } |
6444 | |
6445 | // Based on the range information we know about the LHS, see if we can |
6446 | // simplify this comparison. For example, (x&4) < 8 is always true. |
6447 | switch (Pred) { |
6448 | default: |
6449 | llvm_unreachable("Unknown icmp opcode!" ); |
6450 | case ICmpInst::ICMP_EQ: |
6451 | case ICmpInst::ICMP_NE: { |
6452 | if (Op0Max.ult(RHS: Op1Min) || Op0Min.ugt(RHS: Op1Max)) |
6453 | return replaceInstUsesWith( |
6454 | I, V: ConstantInt::getBool(Ty: I.getType(), V: Pred == CmpInst::ICMP_NE)); |
6455 | |
6456 | // If all bits are known zero except for one, then we know at most one bit |
6457 | // is set. If the comparison is against zero, then this is a check to see if |
6458 | // *that* bit is set. |
6459 | APInt Op0KnownZeroInverted = ~Op0Known.Zero; |
6460 | if (Op1Known.isZero()) { |
6461 | // If the LHS is an AND with the same constant, look through it. |
6462 | Value *LHS = nullptr; |
6463 | const APInt *LHSC; |
6464 | if (!match(V: Op0, P: m_And(L: m_Value(V&: LHS), R: m_APInt(Res&: LHSC))) || |
6465 | *LHSC != Op0KnownZeroInverted) |
6466 | LHS = Op0; |
6467 | |
6468 | Value *X; |
6469 | const APInt *C1; |
6470 | if (match(V: LHS, P: m_Shl(L: m_Power2(V&: C1), R: m_Value(V&: X)))) { |
6471 | Type *XTy = X->getType(); |
6472 | unsigned Log2C1 = C1->countr_zero(); |
6473 | APInt C2 = Op0KnownZeroInverted; |
6474 | APInt C2Pow2 = (C2 & ~(*C1 - 1)) + *C1; |
6475 | if (C2Pow2.isPowerOf2()) { |
6476 | // iff (C1 is pow2) & ((C2 & ~(C1-1)) + C1) is pow2): |
6477 | // ((C1 << X) & C2) == 0 -> X >= (Log2(C2+C1) - Log2(C1)) |
6478 | // ((C1 << X) & C2) != 0 -> X < (Log2(C2+C1) - Log2(C1)) |
6479 | unsigned Log2C2 = C2Pow2.countr_zero(); |
6480 | auto *CmpC = ConstantInt::get(Ty: XTy, V: Log2C2 - Log2C1); |
6481 | auto NewPred = |
6482 | Pred == CmpInst::ICMP_EQ ? CmpInst::ICMP_UGE : CmpInst::ICMP_ULT; |
6483 | return new ICmpInst(NewPred, X, CmpC); |
6484 | } |
6485 | } |
6486 | } |
6487 | |
6488 | // Op0 eq C_Pow2 -> Op0 ne 0 if Op0 is known to be C_Pow2 or zero. |
6489 | if (Op1Known.isConstant() && Op1Known.getConstant().isPowerOf2() && |
6490 | (Op0Known & Op1Known) == Op0Known) |
6491 | return new ICmpInst(CmpInst::getInversePredicate(pred: Pred), Op0, |
6492 | ConstantInt::getNullValue(Ty: Op1->getType())); |
6493 | break; |
6494 | } |
6495 | case ICmpInst::ICMP_ULT: { |
6496 | if (Op0Max.ult(RHS: Op1Min)) // A <u B -> true if max(A) < min(B) |
6497 | return replaceInstUsesWith(I, V: ConstantInt::getTrue(Ty: I.getType())); |
6498 | if (Op0Min.uge(RHS: Op1Max)) // A <u B -> false if min(A) >= max(B) |
6499 | return replaceInstUsesWith(I, V: ConstantInt::getFalse(Ty: I.getType())); |
6500 | break; |
6501 | } |
6502 | case ICmpInst::ICMP_UGT: { |
6503 | if (Op0Min.ugt(RHS: Op1Max)) // A >u B -> true if min(A) > max(B) |
6504 | return replaceInstUsesWith(I, V: ConstantInt::getTrue(Ty: I.getType())); |
6505 | if (Op0Max.ule(RHS: Op1Min)) // A >u B -> false if max(A) <= max(B) |
6506 | return replaceInstUsesWith(I, V: ConstantInt::getFalse(Ty: I.getType())); |
6507 | break; |
6508 | } |
6509 | case ICmpInst::ICMP_SLT: { |
6510 | if (Op0Max.slt(RHS: Op1Min)) // A <s B -> true if max(A) < min(C) |
6511 | return replaceInstUsesWith(I, V: ConstantInt::getTrue(Ty: I.getType())); |
6512 | if (Op0Min.sge(RHS: Op1Max)) // A <s B -> false if min(A) >= max(C) |
6513 | return replaceInstUsesWith(I, V: ConstantInt::getFalse(Ty: I.getType())); |
6514 | break; |
6515 | } |
6516 | case ICmpInst::ICMP_SGT: { |
6517 | if (Op0Min.sgt(RHS: Op1Max)) // A >s B -> true if min(A) > max(B) |
6518 | return replaceInstUsesWith(I, V: ConstantInt::getTrue(Ty: I.getType())); |
6519 | if (Op0Max.sle(RHS: Op1Min)) // A >s B -> false if max(A) <= min(B) |
6520 | return replaceInstUsesWith(I, V: ConstantInt::getFalse(Ty: I.getType())); |
6521 | break; |
6522 | } |
6523 | case ICmpInst::ICMP_SGE: |
6524 | assert(!isa<ConstantInt>(Op1) && "ICMP_SGE with ConstantInt not folded!" ); |
6525 | if (Op0Min.sge(RHS: Op1Max)) // A >=s B -> true if min(A) >= max(B) |
6526 | return replaceInstUsesWith(I, V: ConstantInt::getTrue(Ty: I.getType())); |
6527 | if (Op0Max.slt(RHS: Op1Min)) // A >=s B -> false if max(A) < min(B) |
6528 | return replaceInstUsesWith(I, V: ConstantInt::getFalse(Ty: I.getType())); |
6529 | if (Op1Min == Op0Max) // A >=s B -> A == B if max(A) == min(B) |
6530 | return new ICmpInst(ICmpInst::ICMP_EQ, Op0, Op1); |
6531 | break; |
6532 | case ICmpInst::ICMP_SLE: |
6533 | assert(!isa<ConstantInt>(Op1) && "ICMP_SLE with ConstantInt not folded!" ); |
6534 | if (Op0Max.sle(RHS: Op1Min)) // A <=s B -> true if max(A) <= min(B) |
6535 | return replaceInstUsesWith(I, V: ConstantInt::getTrue(Ty: I.getType())); |
6536 | if (Op0Min.sgt(RHS: Op1Max)) // A <=s B -> false if min(A) > max(B) |
6537 | return replaceInstUsesWith(I, V: ConstantInt::getFalse(Ty: I.getType())); |
6538 | if (Op1Max == Op0Min) // A <=s B -> A == B if min(A) == max(B) |
6539 | return new ICmpInst(ICmpInst::ICMP_EQ, Op0, Op1); |
6540 | break; |
6541 | case ICmpInst::ICMP_UGE: |
6542 | assert(!isa<ConstantInt>(Op1) && "ICMP_UGE with ConstantInt not folded!" ); |
6543 | if (Op0Min.uge(RHS: Op1Max)) // A >=u B -> true if min(A) >= max(B) |
6544 | return replaceInstUsesWith(I, V: ConstantInt::getTrue(Ty: I.getType())); |
6545 | if (Op0Max.ult(RHS: Op1Min)) // A >=u B -> false if max(A) < min(B) |
6546 | return replaceInstUsesWith(I, V: ConstantInt::getFalse(Ty: I.getType())); |
6547 | if (Op1Min == Op0Max) // A >=u B -> A == B if max(A) == min(B) |
6548 | return new ICmpInst(ICmpInst::ICMP_EQ, Op0, Op1); |
6549 | break; |
6550 | case ICmpInst::ICMP_ULE: |
6551 | assert(!isa<ConstantInt>(Op1) && "ICMP_ULE with ConstantInt not folded!" ); |
6552 | if (Op0Max.ule(RHS: Op1Min)) // A <=u B -> true if max(A) <= min(B) |
6553 | return replaceInstUsesWith(I, V: ConstantInt::getTrue(Ty: I.getType())); |
6554 | if (Op0Min.ugt(RHS: Op1Max)) // A <=u B -> false if min(A) > max(B) |
6555 | return replaceInstUsesWith(I, V: ConstantInt::getFalse(Ty: I.getType())); |
6556 | if (Op1Max == Op0Min) // A <=u B -> A == B if min(A) == max(B) |
6557 | return new ICmpInst(ICmpInst::ICMP_EQ, Op0, Op1); |
6558 | break; |
6559 | } |
6560 | |
6561 | // Turn a signed comparison into an unsigned one if both operands are known to |
6562 | // have the same sign. |
6563 | if (I.isSigned() && |
6564 | ((Op0Known.Zero.isNegative() && Op1Known.Zero.isNegative()) || |
6565 | (Op0Known.One.isNegative() && Op1Known.One.isNegative()))) |
6566 | return new ICmpInst(I.getUnsignedPredicate(), Op0, Op1); |
6567 | |
6568 | return nullptr; |
6569 | } |
6570 | |
6571 | /// If one operand of an icmp is effectively a bool (value range of {0,1}), |
6572 | /// then try to reduce patterns based on that limit. |
6573 | Instruction *InstCombinerImpl::foldICmpUsingBoolRange(ICmpInst &I) { |
6574 | Value *X, *Y; |
6575 | ICmpInst::Predicate Pred; |
6576 | |
6577 | // X must be 0 and bool must be true for "ULT": |
6578 | // X <u (zext i1 Y) --> (X == 0) & Y |
6579 | if (match(V: &I, P: m_c_ICmp(Pred, L: m_Value(V&: X), R: m_OneUse(SubPattern: m_ZExt(Op: m_Value(V&: Y))))) && |
6580 | Y->getType()->isIntOrIntVectorTy(BitWidth: 1) && Pred == ICmpInst::ICMP_ULT) |
6581 | return BinaryOperator::CreateAnd(V1: Builder.CreateIsNull(Arg: X), V2: Y); |
6582 | |
6583 | // X must be 0 or bool must be true for "ULE": |
6584 | // X <=u (sext i1 Y) --> (X == 0) | Y |
6585 | if (match(V: &I, P: m_c_ICmp(Pred, L: m_Value(V&: X), R: m_OneUse(SubPattern: m_SExt(Op: m_Value(V&: Y))))) && |
6586 | Y->getType()->isIntOrIntVectorTy(BitWidth: 1) && Pred == ICmpInst::ICMP_ULE) |
6587 | return BinaryOperator::CreateOr(V1: Builder.CreateIsNull(Arg: X), V2: Y); |
6588 | |
6589 | // icmp eq/ne X, (zext/sext (icmp eq/ne X, C)) |
6590 | ICmpInst::Predicate Pred1, Pred2; |
6591 | const APInt *C; |
6592 | Instruction *ExtI; |
6593 | if (match(V: &I, P: m_c_ICmp(Pred&: Pred1, L: m_Value(V&: X), |
6594 | R: m_CombineAnd(L: m_Instruction(I&: ExtI), |
6595 | R: m_ZExtOrSExt(Op: m_ICmp(Pred&: Pred2, L: m_Deferred(V: X), |
6596 | R: m_APInt(Res&: C)))))) && |
6597 | ICmpInst::isEquality(P: Pred1) && ICmpInst::isEquality(P: Pred2)) { |
6598 | bool IsSExt = ExtI->getOpcode() == Instruction::SExt; |
6599 | bool HasOneUse = ExtI->hasOneUse() && ExtI->getOperand(i: 0)->hasOneUse(); |
6600 | auto CreateRangeCheck = [&] { |
6601 | Value *CmpV1 = |
6602 | Builder.CreateICmp(P: Pred1, LHS: X, RHS: Constant::getNullValue(Ty: X->getType())); |
6603 | Value *CmpV2 = Builder.CreateICmp( |
6604 | P: Pred1, LHS: X, RHS: ConstantInt::getSigned(Ty: X->getType(), V: IsSExt ? -1 : 1)); |
6605 | return BinaryOperator::Create( |
6606 | Op: Pred1 == ICmpInst::ICMP_EQ ? Instruction::Or : Instruction::And, |
6607 | S1: CmpV1, S2: CmpV2); |
6608 | }; |
6609 | if (C->isZero()) { |
6610 | if (Pred2 == ICmpInst::ICMP_EQ) { |
6611 | // icmp eq X, (zext/sext (icmp eq X, 0)) --> false |
6612 | // icmp ne X, (zext/sext (icmp eq X, 0)) --> true |
6613 | return replaceInstUsesWith( |
6614 | I, V: ConstantInt::getBool(Ty: I.getType(), V: Pred1 == ICmpInst::ICMP_NE)); |
6615 | } else if (!IsSExt || HasOneUse) { |
6616 | // icmp eq X, (zext (icmp ne X, 0)) --> X == 0 || X == 1 |
6617 | // icmp ne X, (zext (icmp ne X, 0)) --> X != 0 && X != 1 |
6618 | // icmp eq X, (sext (icmp ne X, 0)) --> X == 0 || X == -1 |
6619 | // icmp ne X, (sext (icmp ne X, 0)) --> X != 0 && X == -1 |
6620 | return CreateRangeCheck(); |
6621 | } |
6622 | } else if (IsSExt ? C->isAllOnes() : C->isOne()) { |
6623 | if (Pred2 == ICmpInst::ICMP_NE) { |
6624 | // icmp eq X, (zext (icmp ne X, 1)) --> false |
6625 | // icmp ne X, (zext (icmp ne X, 1)) --> true |
6626 | // icmp eq X, (sext (icmp ne X, -1)) --> false |
6627 | // icmp ne X, (sext (icmp ne X, -1)) --> true |
6628 | return replaceInstUsesWith( |
6629 | I, V: ConstantInt::getBool(Ty: I.getType(), V: Pred1 == ICmpInst::ICMP_NE)); |
6630 | } else if (!IsSExt || HasOneUse) { |
6631 | // icmp eq X, (zext (icmp eq X, 1)) --> X == 0 || X == 1 |
6632 | // icmp ne X, (zext (icmp eq X, 1)) --> X != 0 && X != 1 |
6633 | // icmp eq X, (sext (icmp eq X, -1)) --> X == 0 || X == -1 |
6634 | // icmp ne X, (sext (icmp eq X, -1)) --> X != 0 && X == -1 |
6635 | return CreateRangeCheck(); |
6636 | } |
6637 | } else { |
6638 | // when C != 0 && C != 1: |
6639 | // icmp eq X, (zext (icmp eq X, C)) --> icmp eq X, 0 |
6640 | // icmp eq X, (zext (icmp ne X, C)) --> icmp eq X, 1 |
6641 | // icmp ne X, (zext (icmp eq X, C)) --> icmp ne X, 0 |
6642 | // icmp ne X, (zext (icmp ne X, C)) --> icmp ne X, 1 |
6643 | // when C != 0 && C != -1: |
6644 | // icmp eq X, (sext (icmp eq X, C)) --> icmp eq X, 0 |
6645 | // icmp eq X, (sext (icmp ne X, C)) --> icmp eq X, -1 |
6646 | // icmp ne X, (sext (icmp eq X, C)) --> icmp ne X, 0 |
6647 | // icmp ne X, (sext (icmp ne X, C)) --> icmp ne X, -1 |
6648 | return ICmpInst::Create( |
6649 | Op: Instruction::ICmp, Pred: Pred1, S1: X, |
6650 | S2: ConstantInt::getSigned(Ty: X->getType(), V: Pred2 == ICmpInst::ICMP_NE |
6651 | ? (IsSExt ? -1 : 1) |
6652 | : 0)); |
6653 | } |
6654 | } |
6655 | |
6656 | return nullptr; |
6657 | } |
6658 | |
6659 | std::optional<std::pair<CmpInst::Predicate, Constant *>> |
6660 | InstCombiner::getFlippedStrictnessPredicateAndConstant(CmpInst::Predicate Pred, |
6661 | Constant *C) { |
6662 | assert(ICmpInst::isRelational(Pred) && ICmpInst::isIntPredicate(Pred) && |
6663 | "Only for relational integer predicates." ); |
6664 | |
6665 | Type *Type = C->getType(); |
6666 | bool IsSigned = ICmpInst::isSigned(predicate: Pred); |
6667 | |
6668 | CmpInst::Predicate UnsignedPred = ICmpInst::getUnsignedPredicate(pred: Pred); |
6669 | bool WillIncrement = |
6670 | UnsignedPred == ICmpInst::ICMP_ULE || UnsignedPred == ICmpInst::ICMP_UGT; |
6671 | |
6672 | // Check if the constant operand can be safely incremented/decremented |
6673 | // without overflowing/underflowing. |
6674 | auto ConstantIsOk = [WillIncrement, IsSigned](ConstantInt *C) { |
6675 | return WillIncrement ? !C->isMaxValue(IsSigned) : !C->isMinValue(IsSigned); |
6676 | }; |
6677 | |
6678 | Constant *SafeReplacementConstant = nullptr; |
6679 | if (auto *CI = dyn_cast<ConstantInt>(Val: C)) { |
6680 | // Bail out if the constant can't be safely incremented/decremented. |
6681 | if (!ConstantIsOk(CI)) |
6682 | return std::nullopt; |
6683 | } else if (auto *FVTy = dyn_cast<FixedVectorType>(Val: Type)) { |
6684 | unsigned NumElts = FVTy->getNumElements(); |
6685 | for (unsigned i = 0; i != NumElts; ++i) { |
6686 | Constant *Elt = C->getAggregateElement(Elt: i); |
6687 | if (!Elt) |
6688 | return std::nullopt; |
6689 | |
6690 | if (isa<UndefValue>(Val: Elt)) |
6691 | continue; |
6692 | |
6693 | // Bail out if we can't determine if this constant is min/max or if we |
6694 | // know that this constant is min/max. |
6695 | auto *CI = dyn_cast<ConstantInt>(Val: Elt); |
6696 | if (!CI || !ConstantIsOk(CI)) |
6697 | return std::nullopt; |
6698 | |
6699 | if (!SafeReplacementConstant) |
6700 | SafeReplacementConstant = CI; |
6701 | } |
6702 | } else if (isa<VectorType>(Val: C->getType())) { |
6703 | // Handle scalable splat |
6704 | Value *SplatC = C->getSplatValue(); |
6705 | auto *CI = dyn_cast_or_null<ConstantInt>(Val: SplatC); |
6706 | // Bail out if the constant can't be safely incremented/decremented. |
6707 | if (!CI || !ConstantIsOk(CI)) |
6708 | return std::nullopt; |
6709 | } else { |
6710 | // ConstantExpr? |
6711 | return std::nullopt; |
6712 | } |
6713 | |
6714 | // It may not be safe to change a compare predicate in the presence of |
6715 | // undefined elements, so replace those elements with the first safe constant |
6716 | // that we found. |
6717 | // TODO: in case of poison, it is safe; let's replace undefs only. |
6718 | if (C->containsUndefOrPoisonElement()) { |
6719 | assert(SafeReplacementConstant && "Replacement constant not set" ); |
6720 | C = Constant::replaceUndefsWith(C, Replacement: SafeReplacementConstant); |
6721 | } |
6722 | |
6723 | CmpInst::Predicate NewPred = CmpInst::getFlippedStrictnessPredicate(pred: Pred); |
6724 | |
6725 | // Increment or decrement the constant. |
6726 | Constant *OneOrNegOne = ConstantInt::get(Ty: Type, V: WillIncrement ? 1 : -1, IsSigned: true); |
6727 | Constant *NewC = ConstantExpr::getAdd(C1: C, C2: OneOrNegOne); |
6728 | |
6729 | return std::make_pair(x&: NewPred, y&: NewC); |
6730 | } |
6731 | |
6732 | /// If we have an icmp le or icmp ge instruction with a constant operand, turn |
6733 | /// it into the appropriate icmp lt or icmp gt instruction. This transform |
6734 | /// allows them to be folded in visitICmpInst. |
6735 | static ICmpInst *canonicalizeCmpWithConstant(ICmpInst &I) { |
6736 | ICmpInst::Predicate Pred = I.getPredicate(); |
6737 | if (ICmpInst::isEquality(P: Pred) || !ICmpInst::isIntPredicate(P: Pred) || |
6738 | InstCombiner::isCanonicalPredicate(Pred)) |
6739 | return nullptr; |
6740 | |
6741 | Value *Op0 = I.getOperand(i_nocapture: 0); |
6742 | Value *Op1 = I.getOperand(i_nocapture: 1); |
6743 | auto *Op1C = dyn_cast<Constant>(Val: Op1); |
6744 | if (!Op1C) |
6745 | return nullptr; |
6746 | |
6747 | auto FlippedStrictness = |
6748 | InstCombiner::getFlippedStrictnessPredicateAndConstant(Pred, C: Op1C); |
6749 | if (!FlippedStrictness) |
6750 | return nullptr; |
6751 | |
6752 | return new ICmpInst(FlippedStrictness->first, Op0, FlippedStrictness->second); |
6753 | } |
6754 | |
6755 | /// If we have a comparison with a non-canonical predicate, if we can update |
6756 | /// all the users, invert the predicate and adjust all the users. |
6757 | CmpInst *InstCombinerImpl::canonicalizeICmpPredicate(CmpInst &I) { |
6758 | // Is the predicate already canonical? |
6759 | CmpInst::Predicate Pred = I.getPredicate(); |
6760 | if (InstCombiner::isCanonicalPredicate(Pred)) |
6761 | return nullptr; |
6762 | |
6763 | // Can all users be adjusted to predicate inversion? |
6764 | if (!InstCombiner::canFreelyInvertAllUsersOf(V: &I, /*IgnoredUser=*/nullptr)) |
6765 | return nullptr; |
6766 | |
6767 | // Ok, we can canonicalize comparison! |
6768 | // Let's first invert the comparison's predicate. |
6769 | I.setPredicate(CmpInst::getInversePredicate(pred: Pred)); |
6770 | I.setName(I.getName() + ".not" ); |
6771 | |
6772 | // And, adapt users. |
6773 | freelyInvertAllUsersOf(V: &I); |
6774 | |
6775 | return &I; |
6776 | } |
6777 | |
6778 | /// Integer compare with boolean values can always be turned into bitwise ops. |
6779 | static Instruction *canonicalizeICmpBool(ICmpInst &I, |
6780 | InstCombiner::BuilderTy &Builder) { |
6781 | Value *A = I.getOperand(i_nocapture: 0), *B = I.getOperand(i_nocapture: 1); |
6782 | assert(A->getType()->isIntOrIntVectorTy(1) && "Bools only" ); |
6783 | |
6784 | // A boolean compared to true/false can be simplified to Op0/true/false in |
6785 | // 14 out of the 20 (10 predicates * 2 constants) possible combinations. |
6786 | // Cases not handled by InstSimplify are always 'not' of Op0. |
6787 | if (match(V: B, P: m_Zero())) { |
6788 | switch (I.getPredicate()) { |
6789 | case CmpInst::ICMP_EQ: // A == 0 -> !A |
6790 | case CmpInst::ICMP_ULE: // A <=u 0 -> !A |
6791 | case CmpInst::ICMP_SGE: // A >=s 0 -> !A |
6792 | return BinaryOperator::CreateNot(Op: A); |
6793 | default: |
6794 | llvm_unreachable("ICmp i1 X, C not simplified as expected." ); |
6795 | } |
6796 | } else if (match(V: B, P: m_One())) { |
6797 | switch (I.getPredicate()) { |
6798 | case CmpInst::ICMP_NE: // A != 1 -> !A |
6799 | case CmpInst::ICMP_ULT: // A <u 1 -> !A |
6800 | case CmpInst::ICMP_SGT: // A >s -1 -> !A |
6801 | return BinaryOperator::CreateNot(Op: A); |
6802 | default: |
6803 | llvm_unreachable("ICmp i1 X, C not simplified as expected." ); |
6804 | } |
6805 | } |
6806 | |
6807 | switch (I.getPredicate()) { |
6808 | default: |
6809 | llvm_unreachable("Invalid icmp instruction!" ); |
6810 | case ICmpInst::ICMP_EQ: |
6811 | // icmp eq i1 A, B -> ~(A ^ B) |
6812 | return BinaryOperator::CreateNot(Op: Builder.CreateXor(LHS: A, RHS: B)); |
6813 | |
6814 | case ICmpInst::ICMP_NE: |
6815 | // icmp ne i1 A, B -> A ^ B |
6816 | return BinaryOperator::CreateXor(V1: A, V2: B); |
6817 | |
6818 | case ICmpInst::ICMP_UGT: |
6819 | // icmp ugt -> icmp ult |
6820 | std::swap(a&: A, b&: B); |
6821 | [[fallthrough]]; |
6822 | case ICmpInst::ICMP_ULT: |
6823 | // icmp ult i1 A, B -> ~A & B |
6824 | return BinaryOperator::CreateAnd(V1: Builder.CreateNot(V: A), V2: B); |
6825 | |
6826 | case ICmpInst::ICMP_SGT: |
6827 | // icmp sgt -> icmp slt |
6828 | std::swap(a&: A, b&: B); |
6829 | [[fallthrough]]; |
6830 | case ICmpInst::ICMP_SLT: |
6831 | // icmp slt i1 A, B -> A & ~B |
6832 | return BinaryOperator::CreateAnd(V1: Builder.CreateNot(V: B), V2: A); |
6833 | |
6834 | case ICmpInst::ICMP_UGE: |
6835 | // icmp uge -> icmp ule |
6836 | std::swap(a&: A, b&: B); |
6837 | [[fallthrough]]; |
6838 | case ICmpInst::ICMP_ULE: |
6839 | // icmp ule i1 A, B -> ~A | B |
6840 | return BinaryOperator::CreateOr(V1: Builder.CreateNot(V: A), V2: B); |
6841 | |
6842 | case ICmpInst::ICMP_SGE: |
6843 | // icmp sge -> icmp sle |
6844 | std::swap(a&: A, b&: B); |
6845 | [[fallthrough]]; |
6846 | case ICmpInst::ICMP_SLE: |
6847 | // icmp sle i1 A, B -> A | ~B |
6848 | return BinaryOperator::CreateOr(V1: Builder.CreateNot(V: B), V2: A); |
6849 | } |
6850 | } |
6851 | |
6852 | // Transform pattern like: |
6853 | // (1 << Y) u<= X or ~(-1 << Y) u< X or ((1 << Y)+(-1)) u< X |
6854 | // (1 << Y) u> X or ~(-1 << Y) u>= X or ((1 << Y)+(-1)) u>= X |
6855 | // Into: |
6856 | // (X l>> Y) != 0 |
6857 | // (X l>> Y) == 0 |
6858 | static Instruction *foldICmpWithHighBitMask(ICmpInst &Cmp, |
6859 | InstCombiner::BuilderTy &Builder) { |
6860 | ICmpInst::Predicate Pred, NewPred; |
6861 | Value *X, *Y; |
6862 | if (match(V: &Cmp, |
6863 | P: m_c_ICmp(Pred, L: m_OneUse(SubPattern: m_Shl(L: m_One(), R: m_Value(V&: Y))), R: m_Value(V&: X)))) { |
6864 | switch (Pred) { |
6865 | case ICmpInst::ICMP_ULE: |
6866 | NewPred = ICmpInst::ICMP_NE; |
6867 | break; |
6868 | case ICmpInst::ICMP_UGT: |
6869 | NewPred = ICmpInst::ICMP_EQ; |
6870 | break; |
6871 | default: |
6872 | return nullptr; |
6873 | } |
6874 | } else if (match(V: &Cmp, P: m_c_ICmp(Pred, |
6875 | L: m_OneUse(SubPattern: m_CombineOr( |
6876 | L: m_Not(V: m_Shl(L: m_AllOnes(), R: m_Value(V&: Y))), |
6877 | R: m_Add(L: m_Shl(L: m_One(), R: m_Value(V&: Y)), |
6878 | R: m_AllOnes()))), |
6879 | R: m_Value(V&: X)))) { |
6880 | // The variant with 'add' is not canonical, (the variant with 'not' is) |
6881 | // we only get it because it has extra uses, and can't be canonicalized, |
6882 | |
6883 | switch (Pred) { |
6884 | case ICmpInst::ICMP_ULT: |
6885 | NewPred = ICmpInst::ICMP_NE; |
6886 | break; |
6887 | case ICmpInst::ICMP_UGE: |
6888 | NewPred = ICmpInst::ICMP_EQ; |
6889 | break; |
6890 | default: |
6891 | return nullptr; |
6892 | } |
6893 | } else |
6894 | return nullptr; |
6895 | |
6896 | Value *NewX = Builder.CreateLShr(LHS: X, RHS: Y, Name: X->getName() + ".highbits" ); |
6897 | Constant *Zero = Constant::getNullValue(Ty: NewX->getType()); |
6898 | return CmpInst::Create(Op: Instruction::ICmp, Pred: NewPred, S1: NewX, S2: Zero); |
6899 | } |
6900 | |
6901 | static Instruction *foldVectorCmp(CmpInst &Cmp, |
6902 | InstCombiner::BuilderTy &Builder) { |
6903 | const CmpInst::Predicate Pred = Cmp.getPredicate(); |
6904 | Value *LHS = Cmp.getOperand(i_nocapture: 0), *RHS = Cmp.getOperand(i_nocapture: 1); |
6905 | Value *V1, *V2; |
6906 | |
6907 | auto createCmpReverse = [&](CmpInst::Predicate Pred, Value *X, Value *Y) { |
6908 | Value *V = Builder.CreateCmp(Pred, LHS: X, RHS: Y, Name: Cmp.getName()); |
6909 | if (auto *I = dyn_cast<Instruction>(Val: V)) |
6910 | I->copyIRFlags(V: &Cmp); |
6911 | Module *M = Cmp.getModule(); |
6912 | Function *F = Intrinsic::getDeclaration( |
6913 | M, Intrinsic::id: experimental_vector_reverse, Tys: V->getType()); |
6914 | return CallInst::Create(F, V); |
6915 | }; |
6916 | |
6917 | if (match(V: LHS, P: m_VecReverse(Op0: m_Value(V&: V1)))) { |
6918 | // cmp Pred, rev(V1), rev(V2) --> rev(cmp Pred, V1, V2) |
6919 | if (match(V: RHS, P: m_VecReverse(Op0: m_Value(V&: V2))) && |
6920 | (LHS->hasOneUse() || RHS->hasOneUse())) |
6921 | return createCmpReverse(Pred, V1, V2); |
6922 | |
6923 | // cmp Pred, rev(V1), RHSSplat --> rev(cmp Pred, V1, RHSSplat) |
6924 | if (LHS->hasOneUse() && isSplatValue(V: RHS)) |
6925 | return createCmpReverse(Pred, V1, RHS); |
6926 | } |
6927 | // cmp Pred, LHSSplat, rev(V2) --> rev(cmp Pred, LHSSplat, V2) |
6928 | else if (isSplatValue(V: LHS) && match(V: RHS, P: m_OneUse(SubPattern: m_VecReverse(Op0: m_Value(V&: V2))))) |
6929 | return createCmpReverse(Pred, LHS, V2); |
6930 | |
6931 | ArrayRef<int> M; |
6932 | if (!match(V: LHS, P: m_Shuffle(v1: m_Value(V&: V1), v2: m_Undef(), mask: m_Mask(M)))) |
6933 | return nullptr; |
6934 | |
6935 | // If both arguments of the cmp are shuffles that use the same mask and |
6936 | // shuffle within a single vector, move the shuffle after the cmp: |
6937 | // cmp (shuffle V1, M), (shuffle V2, M) --> shuffle (cmp V1, V2), M |
6938 | Type *V1Ty = V1->getType(); |
6939 | if (match(V: RHS, P: m_Shuffle(v1: m_Value(V&: V2), v2: m_Undef(), mask: m_SpecificMask(M))) && |
6940 | V1Ty == V2->getType() && (LHS->hasOneUse() || RHS->hasOneUse())) { |
6941 | Value *NewCmp = Builder.CreateCmp(Pred, LHS: V1, RHS: V2); |
6942 | return new ShuffleVectorInst(NewCmp, M); |
6943 | } |
6944 | |
6945 | // Try to canonicalize compare with splatted operand and splat constant. |
6946 | // TODO: We could generalize this for more than splats. See/use the code in |
6947 | // InstCombiner::foldVectorBinop(). |
6948 | Constant *C; |
6949 | if (!LHS->hasOneUse() || !match(V: RHS, P: m_Constant(C))) |
6950 | return nullptr; |
6951 | |
6952 | // Length-changing splats are ok, so adjust the constants as needed: |
6953 | // cmp (shuffle V1, M), C --> shuffle (cmp V1, C'), M |
6954 | Constant *ScalarC = C->getSplatValue(/* AllowPoison */ true); |
6955 | int MaskSplatIndex; |
6956 | if (ScalarC && match(Mask: M, P: m_SplatOrPoisonMask(MaskSplatIndex))) { |
6957 | // We allow poison in matching, but this transform removes it for safety. |
6958 | // Demanded elements analysis should be able to recover some/all of that. |
6959 | C = ConstantVector::getSplat(EC: cast<VectorType>(Val: V1Ty)->getElementCount(), |
6960 | Elt: ScalarC); |
6961 | SmallVector<int, 8> NewM(M.size(), MaskSplatIndex); |
6962 | Value *NewCmp = Builder.CreateCmp(Pred, LHS: V1, RHS: C); |
6963 | return new ShuffleVectorInst(NewCmp, NewM); |
6964 | } |
6965 | |
6966 | return nullptr; |
6967 | } |
6968 | |
6969 | // extract(uadd.with.overflow(A, B), 0) ult A |
6970 | // -> extract(uadd.with.overflow(A, B), 1) |
6971 | static Instruction *foldICmpOfUAddOv(ICmpInst &I) { |
6972 | CmpInst::Predicate Pred = I.getPredicate(); |
6973 | Value *Op0 = I.getOperand(i_nocapture: 0), *Op1 = I.getOperand(i_nocapture: 1); |
6974 | |
6975 | Value *UAddOv; |
6976 | Value *A, *B; |
6977 | auto UAddOvResultPat = m_ExtractValue<0>( |
6978 | m_Intrinsic<Intrinsic::uadd_with_overflow>(m_Value(A), m_Value(B))); |
6979 | if (match(Op0, UAddOvResultPat) && |
6980 | ((Pred == ICmpInst::ICMP_ULT && (Op1 == A || Op1 == B)) || |
6981 | (Pred == ICmpInst::ICMP_EQ && match(V: Op1, P: m_ZeroInt()) && |
6982 | (match(V: A, P: m_One()) || match(V: B, P: m_One()))) || |
6983 | (Pred == ICmpInst::ICMP_NE && match(V: Op1, P: m_AllOnes()) && |
6984 | (match(V: A, P: m_AllOnes()) || match(V: B, P: m_AllOnes()))))) |
6985 | // extract(uadd.with.overflow(A, B), 0) < A |
6986 | // extract(uadd.with.overflow(A, 1), 0) == 0 |
6987 | // extract(uadd.with.overflow(A, -1), 0) != -1 |
6988 | UAddOv = cast<ExtractValueInst>(Val: Op0)->getAggregateOperand(); |
6989 | else if (match(Op1, UAddOvResultPat) && |
6990 | Pred == ICmpInst::ICMP_UGT && (Op0 == A || Op0 == B)) |
6991 | // A > extract(uadd.with.overflow(A, B), 0) |
6992 | UAddOv = cast<ExtractValueInst>(Val: Op1)->getAggregateOperand(); |
6993 | else |
6994 | return nullptr; |
6995 | |
6996 | return ExtractValueInst::Create(Agg: UAddOv, Idxs: 1); |
6997 | } |
6998 | |
6999 | static Instruction *foldICmpInvariantGroup(ICmpInst &I) { |
7000 | if (!I.getOperand(i_nocapture: 0)->getType()->isPointerTy() || |
7001 | NullPointerIsDefined( |
7002 | F: I.getParent()->getParent(), |
7003 | AS: I.getOperand(i_nocapture: 0)->getType()->getPointerAddressSpace())) { |
7004 | return nullptr; |
7005 | } |
7006 | Instruction *Op; |
7007 | if (match(V: I.getOperand(i_nocapture: 0), P: m_Instruction(I&: Op)) && |
7008 | match(V: I.getOperand(i_nocapture: 1), P: m_Zero()) && |
7009 | Op->isLaunderOrStripInvariantGroup()) { |
7010 | return ICmpInst::Create(Op: Instruction::ICmp, Pred: I.getPredicate(), |
7011 | S1: Op->getOperand(i: 0), S2: I.getOperand(i_nocapture: 1)); |
7012 | } |
7013 | return nullptr; |
7014 | } |
7015 | |
7016 | /// This function folds patterns produced by lowering of reduce idioms, such as |
7017 | /// llvm.vector.reduce.and which are lowered into instruction chains. This code |
7018 | /// attempts to generate fewer number of scalar comparisons instead of vector |
7019 | /// comparisons when possible. |
7020 | static Instruction *foldReductionIdiom(ICmpInst &I, |
7021 | InstCombiner::BuilderTy &Builder, |
7022 | const DataLayout &DL) { |
7023 | if (I.getType()->isVectorTy()) |
7024 | return nullptr; |
7025 | ICmpInst::Predicate OuterPred, InnerPred; |
7026 | Value *LHS, *RHS; |
7027 | |
7028 | // Match lowering of @llvm.vector.reduce.and. Turn |
7029 | /// %vec_ne = icmp ne <8 x i8> %lhs, %rhs |
7030 | /// %scalar_ne = bitcast <8 x i1> %vec_ne to i8 |
7031 | /// %res = icmp <pred> i8 %scalar_ne, 0 |
7032 | /// |
7033 | /// into |
7034 | /// |
7035 | /// %lhs.scalar = bitcast <8 x i8> %lhs to i64 |
7036 | /// %rhs.scalar = bitcast <8 x i8> %rhs to i64 |
7037 | /// %res = icmp <pred> i64 %lhs.scalar, %rhs.scalar |
7038 | /// |
7039 | /// for <pred> in {ne, eq}. |
7040 | if (!match(V: &I, P: m_ICmp(Pred&: OuterPred, |
7041 | L: m_OneUse(SubPattern: m_BitCast(Op: m_OneUse( |
7042 | SubPattern: m_ICmp(Pred&: InnerPred, L: m_Value(V&: LHS), R: m_Value(V&: RHS))))), |
7043 | R: m_Zero()))) |
7044 | return nullptr; |
7045 | auto *LHSTy = dyn_cast<FixedVectorType>(Val: LHS->getType()); |
7046 | if (!LHSTy || !LHSTy->getElementType()->isIntegerTy()) |
7047 | return nullptr; |
7048 | unsigned NumBits = |
7049 | LHSTy->getNumElements() * LHSTy->getElementType()->getIntegerBitWidth(); |
7050 | // TODO: Relax this to "not wider than max legal integer type"? |
7051 | if (!DL.isLegalInteger(Width: NumBits)) |
7052 | return nullptr; |
7053 | |
7054 | if (ICmpInst::isEquality(P: OuterPred) && InnerPred == ICmpInst::ICMP_NE) { |
7055 | auto *ScalarTy = Builder.getIntNTy(N: NumBits); |
7056 | LHS = Builder.CreateBitCast(V: LHS, DestTy: ScalarTy, Name: LHS->getName() + ".scalar" ); |
7057 | RHS = Builder.CreateBitCast(V: RHS, DestTy: ScalarTy, Name: RHS->getName() + ".scalar" ); |
7058 | return ICmpInst::Create(Op: Instruction::ICmp, Pred: OuterPred, S1: LHS, S2: RHS, |
7059 | Name: I.getName()); |
7060 | } |
7061 | |
7062 | return nullptr; |
7063 | } |
7064 | |
7065 | // This helper will be called with icmp operands in both orders. |
7066 | Instruction *InstCombinerImpl::foldICmpCommutative(ICmpInst::Predicate Pred, |
7067 | Value *Op0, Value *Op1, |
7068 | ICmpInst &CxtI) { |
7069 | // Try to optimize 'icmp GEP, P' or 'icmp P, GEP'. |
7070 | if (auto *GEP = dyn_cast<GEPOperator>(Val: Op0)) |
7071 | if (Instruction *NI = foldGEPICmp(GEPLHS: GEP, RHS: Op1, Cond: Pred, I&: CxtI)) |
7072 | return NI; |
7073 | |
7074 | if (auto *SI = dyn_cast<SelectInst>(Val: Op0)) |
7075 | if (Instruction *NI = foldSelectICmp(Pred, SI, RHS: Op1, I: CxtI)) |
7076 | return NI; |
7077 | |
7078 | if (auto *MinMax = dyn_cast<MinMaxIntrinsic>(Val: Op0)) |
7079 | if (Instruction *Res = foldICmpWithMinMax(I&: CxtI, MinMax, Z: Op1, Pred)) |
7080 | return Res; |
7081 | |
7082 | { |
7083 | Value *X; |
7084 | const APInt *C; |
7085 | // icmp X+Cst, X |
7086 | if (match(V: Op0, P: m_Add(L: m_Value(V&: X), R: m_APInt(Res&: C))) && Op1 == X) |
7087 | return foldICmpAddOpConst(X, C: *C, Pred); |
7088 | } |
7089 | |
7090 | // abs(X) >= X --> true |
7091 | // abs(X) u<= X --> true |
7092 | // abs(X) < X --> false |
7093 | // abs(X) u> X --> false |
7094 | // abs(X) u>= X --> IsIntMinPosion ? `X > -1`: `X u<= INTMIN` |
7095 | // abs(X) <= X --> IsIntMinPosion ? `X > -1`: `X u<= INTMIN` |
7096 | // abs(X) == X --> IsIntMinPosion ? `X > -1`: `X u<= INTMIN` |
7097 | // abs(X) u< X --> IsIntMinPosion ? `X < 0` : `X > INTMIN` |
7098 | // abs(X) > X --> IsIntMinPosion ? `X < 0` : `X > INTMIN` |
7099 | // abs(X) != X --> IsIntMinPosion ? `X < 0` : `X > INTMIN` |
7100 | { |
7101 | Value *X; |
7102 | Constant *C; |
7103 | if (match(Op0, m_Intrinsic<Intrinsic::abs>(m_Value(X), m_Constant(C))) && |
7104 | match(Op1, m_Specific(X))) { |
7105 | Value *NullValue = Constant::getNullValue(Ty: X->getType()); |
7106 | Value *AllOnesValue = Constant::getAllOnesValue(Ty: X->getType()); |
7107 | const APInt SMin = |
7108 | APInt::getSignedMinValue(numBits: X->getType()->getScalarSizeInBits()); |
7109 | bool IsIntMinPosion = C->isAllOnesValue(); |
7110 | switch (Pred) { |
7111 | case CmpInst::ICMP_ULE: |
7112 | case CmpInst::ICMP_SGE: |
7113 | return replaceInstUsesWith(I&: CxtI, V: ConstantInt::getTrue(Ty: CxtI.getType())); |
7114 | case CmpInst::ICMP_UGT: |
7115 | case CmpInst::ICMP_SLT: |
7116 | return replaceInstUsesWith(I&: CxtI, V: ConstantInt::getFalse(Ty: CxtI.getType())); |
7117 | case CmpInst::ICMP_UGE: |
7118 | case CmpInst::ICMP_SLE: |
7119 | case CmpInst::ICMP_EQ: { |
7120 | return replaceInstUsesWith( |
7121 | I&: CxtI, V: IsIntMinPosion |
7122 | ? Builder.CreateICmpSGT(LHS: X, RHS: AllOnesValue) |
7123 | : Builder.CreateICmpULT( |
7124 | LHS: X, RHS: ConstantInt::get(Ty: X->getType(), V: SMin + 1))); |
7125 | } |
7126 | case CmpInst::ICMP_ULT: |
7127 | case CmpInst::ICMP_SGT: |
7128 | case CmpInst::ICMP_NE: { |
7129 | return replaceInstUsesWith( |
7130 | I&: CxtI, V: IsIntMinPosion |
7131 | ? Builder.CreateICmpSLT(LHS: X, RHS: NullValue) |
7132 | : Builder.CreateICmpUGT( |
7133 | LHS: X, RHS: ConstantInt::get(Ty: X->getType(), V: SMin))); |
7134 | } |
7135 | default: |
7136 | llvm_unreachable("Invalid predicate!" ); |
7137 | } |
7138 | } |
7139 | } |
7140 | |
7141 | const SimplifyQuery Q = SQ.getWithInstruction(I: &CxtI); |
7142 | if (Value *V = foldICmpWithLowBitMaskedVal(Pred, Op0, Op1, Q, IC&: *this)) |
7143 | return replaceInstUsesWith(I&: CxtI, V); |
7144 | |
7145 | // Folding (X / Y) pred X => X swap(pred) 0 for constant Y other than 0 or 1 |
7146 | { |
7147 | const APInt *Divisor; |
7148 | if (match(V: Op0, P: m_UDiv(L: m_Specific(V: Op1), R: m_APInt(Res&: Divisor))) && |
7149 | Divisor->ugt(RHS: 1)) { |
7150 | return new ICmpInst(ICmpInst::getSwappedPredicate(pred: Pred), Op1, |
7151 | Constant::getNullValue(Ty: Op1->getType())); |
7152 | } |
7153 | |
7154 | if (!ICmpInst::isUnsigned(predicate: Pred) && |
7155 | match(V: Op0, P: m_SDiv(L: m_Specific(V: Op1), R: m_APInt(Res&: Divisor))) && |
7156 | Divisor->ugt(RHS: 1)) { |
7157 | return new ICmpInst(ICmpInst::getSwappedPredicate(pred: Pred), Op1, |
7158 | Constant::getNullValue(Ty: Op1->getType())); |
7159 | } |
7160 | } |
7161 | |
7162 | // Another case of this fold is (X >> Y) pred X => X swap(pred) 0 if Y != 0 |
7163 | { |
7164 | const APInt *Shift; |
7165 | if (match(V: Op0, P: m_LShr(L: m_Specific(V: Op1), R: m_APInt(Res&: Shift))) && |
7166 | !Shift->isZero()) { |
7167 | return new ICmpInst(ICmpInst::getSwappedPredicate(pred: Pred), Op1, |
7168 | Constant::getNullValue(Ty: Op1->getType())); |
7169 | } |
7170 | |
7171 | if ((Pred == CmpInst::ICMP_SLT || Pred == CmpInst::ICMP_SGE) && |
7172 | match(V: Op0, P: m_AShr(L: m_Specific(V: Op1), R: m_APInt(Res&: Shift))) && |
7173 | !Shift->isZero()) { |
7174 | return new ICmpInst(ICmpInst::getSwappedPredicate(pred: Pred), Op1, |
7175 | Constant::getNullValue(Ty: Op1->getType())); |
7176 | } |
7177 | } |
7178 | |
7179 | return nullptr; |
7180 | } |
7181 | |
7182 | Instruction *InstCombinerImpl::visitICmpInst(ICmpInst &I) { |
7183 | bool Changed = false; |
7184 | const SimplifyQuery Q = SQ.getWithInstruction(I: &I); |
7185 | Value *Op0 = I.getOperand(i_nocapture: 0), *Op1 = I.getOperand(i_nocapture: 1); |
7186 | unsigned Op0Cplxity = getComplexity(V: Op0); |
7187 | unsigned Op1Cplxity = getComplexity(V: Op1); |
7188 | |
7189 | /// Orders the operands of the compare so that they are listed from most |
7190 | /// complex to least complex. This puts constants before unary operators, |
7191 | /// before binary operators. |
7192 | if (Op0Cplxity < Op1Cplxity) { |
7193 | I.swapOperands(); |
7194 | std::swap(a&: Op0, b&: Op1); |
7195 | Changed = true; |
7196 | } |
7197 | |
7198 | if (Value *V = simplifyICmpInst(Predicate: I.getPredicate(), LHS: Op0, RHS: Op1, Q)) |
7199 | return replaceInstUsesWith(I, V); |
7200 | |
7201 | // Comparing -val or val with non-zero is the same as just comparing val |
7202 | // ie, abs(val) != 0 -> val != 0 |
7203 | if (I.getPredicate() == ICmpInst::ICMP_NE && match(V: Op1, P: m_Zero())) { |
7204 | Value *Cond, *SelectTrue, *SelectFalse; |
7205 | if (match(V: Op0, P: m_Select(C: m_Value(V&: Cond), L: m_Value(V&: SelectTrue), |
7206 | R: m_Value(V&: SelectFalse)))) { |
7207 | if (Value *V = dyn_castNegVal(V: SelectTrue)) { |
7208 | if (V == SelectFalse) |
7209 | return CmpInst::Create(Op: Instruction::ICmp, Pred: I.getPredicate(), S1: V, S2: Op1); |
7210 | } |
7211 | else if (Value *V = dyn_castNegVal(V: SelectFalse)) { |
7212 | if (V == SelectTrue) |
7213 | return CmpInst::Create(Op: Instruction::ICmp, Pred: I.getPredicate(), S1: V, S2: Op1); |
7214 | } |
7215 | } |
7216 | } |
7217 | |
7218 | if (Op0->getType()->isIntOrIntVectorTy(BitWidth: 1)) |
7219 | if (Instruction *Res = canonicalizeICmpBool(I, Builder)) |
7220 | return Res; |
7221 | |
7222 | if (Instruction *Res = canonicalizeCmpWithConstant(I)) |
7223 | return Res; |
7224 | |
7225 | if (Instruction *Res = canonicalizeICmpPredicate(I)) |
7226 | return Res; |
7227 | |
7228 | if (Instruction *Res = foldICmpWithConstant(Cmp&: I)) |
7229 | return Res; |
7230 | |
7231 | if (Instruction *Res = foldICmpWithDominatingICmp(Cmp&: I)) |
7232 | return Res; |
7233 | |
7234 | if (Instruction *Res = foldICmpUsingBoolRange(I)) |
7235 | return Res; |
7236 | |
7237 | if (Instruction *Res = foldICmpUsingKnownBits(I)) |
7238 | return Res; |
7239 | |
7240 | if (Instruction *Res = foldICmpTruncWithTruncOrExt(Cmp&: I, Q)) |
7241 | return Res; |
7242 | |
7243 | // Test if the ICmpInst instruction is used exclusively by a select as |
7244 | // part of a minimum or maximum operation. If so, refrain from doing |
7245 | // any other folding. This helps out other analyses which understand |
7246 | // non-obfuscated minimum and maximum idioms, such as ScalarEvolution |
7247 | // and CodeGen. And in this case, at least one of the comparison |
7248 | // operands has at least one user besides the compare (the select), |
7249 | // which would often largely negate the benefit of folding anyway. |
7250 | // |
7251 | // Do the same for the other patterns recognized by matchSelectPattern. |
7252 | if (I.hasOneUse()) |
7253 | if (SelectInst *SI = dyn_cast<SelectInst>(Val: I.user_back())) { |
7254 | Value *A, *B; |
7255 | SelectPatternResult SPR = matchSelectPattern(V: SI, LHS&: A, RHS&: B); |
7256 | if (SPR.Flavor != SPF_UNKNOWN) |
7257 | return nullptr; |
7258 | } |
7259 | |
7260 | // Do this after checking for min/max to prevent infinite looping. |
7261 | if (Instruction *Res = foldICmpWithZero(Cmp&: I)) |
7262 | return Res; |
7263 | |
7264 | // FIXME: We only do this after checking for min/max to prevent infinite |
7265 | // looping caused by a reverse canonicalization of these patterns for min/max. |
7266 | // FIXME: The organization of folds is a mess. These would naturally go into |
7267 | // canonicalizeCmpWithConstant(), but we can't move all of the above folds |
7268 | // down here after the min/max restriction. |
7269 | ICmpInst::Predicate Pred = I.getPredicate(); |
7270 | const APInt *C; |
7271 | if (match(V: Op1, P: m_APInt(Res&: C))) { |
7272 | // For i32: x >u 2147483647 -> x <s 0 -> true if sign bit set |
7273 | if (Pred == ICmpInst::ICMP_UGT && C->isMaxSignedValue()) { |
7274 | Constant *Zero = Constant::getNullValue(Ty: Op0->getType()); |
7275 | return new ICmpInst(ICmpInst::ICMP_SLT, Op0, Zero); |
7276 | } |
7277 | |
7278 | // For i32: x <u 2147483648 -> x >s -1 -> true if sign bit clear |
7279 | if (Pred == ICmpInst::ICMP_ULT && C->isMinSignedValue()) { |
7280 | Constant *AllOnes = Constant::getAllOnesValue(Ty: Op0->getType()); |
7281 | return new ICmpInst(ICmpInst::ICMP_SGT, Op0, AllOnes); |
7282 | } |
7283 | } |
7284 | |
7285 | // The folds in here may rely on wrapping flags and special constants, so |
7286 | // they can break up min/max idioms in some cases but not seemingly similar |
7287 | // patterns. |
7288 | // FIXME: It may be possible to enhance select folding to make this |
7289 | // unnecessary. It may also be moot if we canonicalize to min/max |
7290 | // intrinsics. |
7291 | if (Instruction *Res = foldICmpBinOp(I, SQ: Q)) |
7292 | return Res; |
7293 | |
7294 | if (Instruction *Res = foldICmpInstWithConstant(Cmp&: I)) |
7295 | return Res; |
7296 | |
7297 | // Try to match comparison as a sign bit test. Intentionally do this after |
7298 | // foldICmpInstWithConstant() to potentially let other folds to happen first. |
7299 | if (Instruction *New = foldSignBitTest(I)) |
7300 | return New; |
7301 | |
7302 | if (Instruction *Res = foldICmpInstWithConstantNotInt(I)) |
7303 | return Res; |
7304 | |
7305 | if (Instruction *Res = foldICmpCommutative(Pred: I.getPredicate(), Op0, Op1, CxtI&: I)) |
7306 | return Res; |
7307 | if (Instruction *Res = |
7308 | foldICmpCommutative(Pred: I.getSwappedPredicate(), Op0: Op1, Op1: Op0, CxtI&: I)) |
7309 | return Res; |
7310 | |
7311 | if (I.isCommutative()) { |
7312 | if (auto Pair = matchSymmetricPair(LHS: I.getOperand(i_nocapture: 0), RHS: I.getOperand(i_nocapture: 1))) { |
7313 | replaceOperand(I, OpNum: 0, V: Pair->first); |
7314 | replaceOperand(I, OpNum: 1, V: Pair->second); |
7315 | return &I; |
7316 | } |
7317 | } |
7318 | |
7319 | // In case of a comparison with two select instructions having the same |
7320 | // condition, check whether one of the resulting branches can be simplified. |
7321 | // If so, just compare the other branch and select the appropriate result. |
7322 | // For example: |
7323 | // %tmp1 = select i1 %cmp, i32 %y, i32 %x |
7324 | // %tmp2 = select i1 %cmp, i32 %z, i32 %x |
7325 | // %cmp2 = icmp slt i32 %tmp2, %tmp1 |
7326 | // The icmp will result false for the false value of selects and the result |
7327 | // will depend upon the comparison of true values of selects if %cmp is |
7328 | // true. Thus, transform this into: |
7329 | // %cmp = icmp slt i32 %y, %z |
7330 | // %sel = select i1 %cond, i1 %cmp, i1 false |
7331 | // This handles similar cases to transform. |
7332 | { |
7333 | Value *Cond, *A, *B, *C, *D; |
7334 | if (match(V: Op0, P: m_Select(C: m_Value(V&: Cond), L: m_Value(V&: A), R: m_Value(V&: B))) && |
7335 | match(V: Op1, P: m_Select(C: m_Specific(V: Cond), L: m_Value(V&: C), R: m_Value(V&: D))) && |
7336 | (Op0->hasOneUse() || Op1->hasOneUse())) { |
7337 | // Check whether comparison of TrueValues can be simplified |
7338 | if (Value *Res = simplifyICmpInst(Predicate: Pred, LHS: A, RHS: C, Q: SQ)) { |
7339 | Value *NewICMP = Builder.CreateICmp(P: Pred, LHS: B, RHS: D); |
7340 | return SelectInst::Create(C: Cond, S1: Res, S2: NewICMP); |
7341 | } |
7342 | // Check whether comparison of FalseValues can be simplified |
7343 | if (Value *Res = simplifyICmpInst(Predicate: Pred, LHS: B, RHS: D, Q: SQ)) { |
7344 | Value *NewICMP = Builder.CreateICmp(P: Pred, LHS: A, RHS: C); |
7345 | return SelectInst::Create(C: Cond, S1: NewICMP, S2: Res); |
7346 | } |
7347 | } |
7348 | } |
7349 | |
7350 | // Try to optimize equality comparisons against alloca-based pointers. |
7351 | if (Op0->getType()->isPointerTy() && I.isEquality()) { |
7352 | assert(Op1->getType()->isPointerTy() && "Comparing pointer with non-pointer?" ); |
7353 | if (auto *Alloca = dyn_cast<AllocaInst>(Val: getUnderlyingObject(V: Op0))) |
7354 | if (foldAllocaCmp(Alloca)) |
7355 | return nullptr; |
7356 | if (auto *Alloca = dyn_cast<AllocaInst>(Val: getUnderlyingObject(V: Op1))) |
7357 | if (foldAllocaCmp(Alloca)) |
7358 | return nullptr; |
7359 | } |
7360 | |
7361 | if (Instruction *Res = foldICmpBitCast(Cmp&: I)) |
7362 | return Res; |
7363 | |
7364 | // TODO: Hoist this above the min/max bailout. |
7365 | if (Instruction *R = foldICmpWithCastOp(ICmp&: I)) |
7366 | return R; |
7367 | |
7368 | { |
7369 | Value *X, *Y; |
7370 | // Transform (X & ~Y) == 0 --> (X & Y) != 0 |
7371 | // and (X & ~Y) != 0 --> (X & Y) == 0 |
7372 | // if A is a power of 2. |
7373 | if (match(V: Op0, P: m_And(L: m_Value(V&: X), R: m_Not(V: m_Value(V&: Y)))) && |
7374 | match(V: Op1, P: m_Zero()) && isKnownToBeAPowerOfTwo(V: X, OrZero: false, Depth: 0, CxtI: &I) && |
7375 | I.isEquality()) |
7376 | return new ICmpInst(I.getInversePredicate(), Builder.CreateAnd(LHS: X, RHS: Y), |
7377 | Op1); |
7378 | |
7379 | // Op0 pred Op1 -> ~Op1 pred ~Op0, if this allows us to drop an instruction. |
7380 | if (Op0->getType()->isIntOrIntVectorTy()) { |
7381 | bool ConsumesOp0, ConsumesOp1; |
7382 | if (isFreeToInvert(V: Op0, WillInvertAllUses: Op0->hasOneUse(), DoesConsume&: ConsumesOp0) && |
7383 | isFreeToInvert(V: Op1, WillInvertAllUses: Op1->hasOneUse(), DoesConsume&: ConsumesOp1) && |
7384 | (ConsumesOp0 || ConsumesOp1)) { |
7385 | Value *InvOp0 = getFreelyInverted(V: Op0, WillInvertAllUses: Op0->hasOneUse(), Builder: &Builder); |
7386 | Value *InvOp1 = getFreelyInverted(V: Op1, WillInvertAllUses: Op1->hasOneUse(), Builder: &Builder); |
7387 | assert(InvOp0 && InvOp1 && |
7388 | "Mismatch between isFreeToInvert and getFreelyInverted" ); |
7389 | return new ICmpInst(I.getSwappedPredicate(), InvOp0, InvOp1); |
7390 | } |
7391 | } |
7392 | |
7393 | Instruction *AddI = nullptr; |
7394 | if (match(V: &I, P: m_UAddWithOverflow(L: m_Value(V&: X), R: m_Value(V&: Y), |
7395 | S: m_Instruction(I&: AddI))) && |
7396 | isa<IntegerType>(Val: X->getType())) { |
7397 | Value *Result; |
7398 | Constant *Overflow; |
7399 | // m_UAddWithOverflow can match patterns that do not include an explicit |
7400 | // "add" instruction, so check the opcode of the matched op. |
7401 | if (AddI->getOpcode() == Instruction::Add && |
7402 | OptimizeOverflowCheck(BinaryOp: Instruction::Add, /*Signed*/ IsSigned: false, LHS: X, RHS: Y, OrigI&: *AddI, |
7403 | Result, Overflow)) { |
7404 | replaceInstUsesWith(I&: *AddI, V: Result); |
7405 | eraseInstFromFunction(I&: *AddI); |
7406 | return replaceInstUsesWith(I, V: Overflow); |
7407 | } |
7408 | } |
7409 | |
7410 | // (zext X) * (zext Y) --> llvm.umul.with.overflow. |
7411 | if (match(V: Op0, P: m_NUWMul(L: m_ZExt(Op: m_Value(V&: X)), R: m_ZExt(Op: m_Value(V&: Y)))) && |
7412 | match(V: Op1, P: m_APInt(Res&: C))) { |
7413 | if (Instruction *R = processUMulZExtIdiom(I, MulVal: Op0, OtherVal: C, IC&: *this)) |
7414 | return R; |
7415 | } |
7416 | |
7417 | // Signbit test folds |
7418 | // Fold (X u>> BitWidth - 1 Pred ZExt(i1)) --> X s< 0 Pred i1 |
7419 | // Fold (X s>> BitWidth - 1 Pred SExt(i1)) --> X s< 0 Pred i1 |
7420 | Instruction *ExtI; |
7421 | if ((I.isUnsigned() || I.isEquality()) && |
7422 | match(V: Op1, |
7423 | P: m_CombineAnd(L: m_Instruction(I&: ExtI), R: m_ZExtOrSExt(Op: m_Value(V&: Y)))) && |
7424 | Y->getType()->getScalarSizeInBits() == 1 && |
7425 | (Op0->hasOneUse() || Op1->hasOneUse())) { |
7426 | unsigned OpWidth = Op0->getType()->getScalarSizeInBits(); |
7427 | Instruction *ShiftI; |
7428 | if (match(V: Op0, P: m_CombineAnd(L: m_Instruction(I&: ShiftI), |
7429 | R: m_Shr(L: m_Value(V&: X), R: m_SpecificIntAllowPoison( |
7430 | V: OpWidth - 1))))) { |
7431 | unsigned ExtOpc = ExtI->getOpcode(); |
7432 | unsigned ShiftOpc = ShiftI->getOpcode(); |
7433 | if ((ExtOpc == Instruction::ZExt && ShiftOpc == Instruction::LShr) || |
7434 | (ExtOpc == Instruction::SExt && ShiftOpc == Instruction::AShr)) { |
7435 | Value *SLTZero = |
7436 | Builder.CreateICmpSLT(LHS: X, RHS: Constant::getNullValue(Ty: X->getType())); |
7437 | Value *Cmp = Builder.CreateICmp(P: Pred, LHS: SLTZero, RHS: Y, Name: I.getName()); |
7438 | return replaceInstUsesWith(I, V: Cmp); |
7439 | } |
7440 | } |
7441 | } |
7442 | } |
7443 | |
7444 | if (Instruction *Res = foldICmpEquality(I)) |
7445 | return Res; |
7446 | |
7447 | if (Instruction *Res = foldICmpPow2Test(I, Builder)) |
7448 | return Res; |
7449 | |
7450 | if (Instruction *Res = foldICmpOfUAddOv(I)) |
7451 | return Res; |
7452 | |
7453 | // The 'cmpxchg' instruction returns an aggregate containing the old value and |
7454 | // an i1 which indicates whether or not we successfully did the swap. |
7455 | // |
7456 | // Replace comparisons between the old value and the expected value with the |
7457 | // indicator that 'cmpxchg' returns. |
7458 | // |
7459 | // N.B. This transform is only valid when the 'cmpxchg' is not permitted to |
7460 | // spuriously fail. In those cases, the old value may equal the expected |
7461 | // value but it is possible for the swap to not occur. |
7462 | if (I.getPredicate() == ICmpInst::ICMP_EQ) |
7463 | if (auto *EVI = dyn_cast<ExtractValueInst>(Val: Op0)) |
7464 | if (auto *ACXI = dyn_cast<AtomicCmpXchgInst>(Val: EVI->getAggregateOperand())) |
7465 | if (EVI->getIndices()[0] == 0 && ACXI->getCompareOperand() == Op1 && |
7466 | !ACXI->isWeak()) |
7467 | return ExtractValueInst::Create(Agg: ACXI, Idxs: 1); |
7468 | |
7469 | if (Instruction *Res = foldICmpWithHighBitMask(Cmp&: I, Builder)) |
7470 | return Res; |
7471 | |
7472 | if (I.getType()->isVectorTy()) |
7473 | if (Instruction *Res = foldVectorCmp(Cmp&: I, Builder)) |
7474 | return Res; |
7475 | |
7476 | if (Instruction *Res = foldICmpInvariantGroup(I)) |
7477 | return Res; |
7478 | |
7479 | if (Instruction *Res = foldReductionIdiom(I, Builder, DL)) |
7480 | return Res; |
7481 | |
7482 | return Changed ? &I : nullptr; |
7483 | } |
7484 | |
7485 | /// Fold fcmp ([us]itofp x, cst) if possible. |
7486 | Instruction *InstCombinerImpl::foldFCmpIntToFPConst(FCmpInst &I, |
7487 | Instruction *LHSI, |
7488 | Constant *RHSC) { |
7489 | const APFloat *RHS; |
7490 | if (!match(V: RHSC, P: m_APFloat(Res&: RHS))) |
7491 | return nullptr; |
7492 | |
7493 | // Get the width of the mantissa. We don't want to hack on conversions that |
7494 | // might lose information from the integer, e.g. "i64 -> float" |
7495 | int MantissaWidth = LHSI->getType()->getFPMantissaWidth(); |
7496 | if (MantissaWidth == -1) return nullptr; // Unknown. |
7497 | |
7498 | Type *IntTy = LHSI->getOperand(i: 0)->getType(); |
7499 | unsigned IntWidth = IntTy->getScalarSizeInBits(); |
7500 | bool LHSUnsigned = isa<UIToFPInst>(Val: LHSI); |
7501 | |
7502 | if (I.isEquality()) { |
7503 | FCmpInst::Predicate P = I.getPredicate(); |
7504 | bool IsExact = false; |
7505 | APSInt RHSCvt(IntWidth, LHSUnsigned); |
7506 | RHS->convertToInteger(Result&: RHSCvt, RM: APFloat::rmNearestTiesToEven, IsExact: &IsExact); |
7507 | |
7508 | // If the floating point constant isn't an integer value, we know if we will |
7509 | // ever compare equal / not equal to it. |
7510 | if (!IsExact) { |
7511 | // TODO: Can never be -0.0 and other non-representable values |
7512 | APFloat RHSRoundInt(*RHS); |
7513 | RHSRoundInt.roundToIntegral(RM: APFloat::rmNearestTiesToEven); |
7514 | if (*RHS != RHSRoundInt) { |
7515 | if (P == FCmpInst::FCMP_OEQ || P == FCmpInst::FCMP_UEQ) |
7516 | return replaceInstUsesWith(I, V: ConstantInt::getFalse(Ty: I.getType())); |
7517 | |
7518 | assert(P == FCmpInst::FCMP_ONE || P == FCmpInst::FCMP_UNE); |
7519 | return replaceInstUsesWith(I, V: ConstantInt::getTrue(Ty: I.getType())); |
7520 | } |
7521 | } |
7522 | |
7523 | // TODO: If the constant is exactly representable, is it always OK to do |
7524 | // equality compares as integer? |
7525 | } |
7526 | |
7527 | // Check to see that the input is converted from an integer type that is small |
7528 | // enough that preserves all bits. TODO: check here for "known" sign bits. |
7529 | // This would allow us to handle (fptosi (x >>s 62) to float) if x is i64 f.e. |
7530 | |
7531 | // Following test does NOT adjust IntWidth downwards for signed inputs, |
7532 | // because the most negative value still requires all the mantissa bits |
7533 | // to distinguish it from one less than that value. |
7534 | if ((int)IntWidth > MantissaWidth) { |
7535 | // Conversion would lose accuracy. Check if loss can impact comparison. |
7536 | int Exp = ilogb(Arg: *RHS); |
7537 | if (Exp == APFloat::IEK_Inf) { |
7538 | int MaxExponent = ilogb(Arg: APFloat::getLargest(Sem: RHS->getSemantics())); |
7539 | if (MaxExponent < (int)IntWidth - !LHSUnsigned) |
7540 | // Conversion could create infinity. |
7541 | return nullptr; |
7542 | } else { |
7543 | // Note that if RHS is zero or NaN, then Exp is negative |
7544 | // and first condition is trivially false. |
7545 | if (MantissaWidth <= Exp && Exp <= (int)IntWidth - !LHSUnsigned) |
7546 | // Conversion could affect comparison. |
7547 | return nullptr; |
7548 | } |
7549 | } |
7550 | |
7551 | // Otherwise, we can potentially simplify the comparison. We know that it |
7552 | // will always come through as an integer value and we know the constant is |
7553 | // not a NAN (it would have been previously simplified). |
7554 | assert(!RHS->isNaN() && "NaN comparison not already folded!" ); |
7555 | |
7556 | ICmpInst::Predicate Pred; |
7557 | switch (I.getPredicate()) { |
7558 | default: llvm_unreachable("Unexpected predicate!" ); |
7559 | case FCmpInst::FCMP_UEQ: |
7560 | case FCmpInst::FCMP_OEQ: |
7561 | Pred = ICmpInst::ICMP_EQ; |
7562 | break; |
7563 | case FCmpInst::FCMP_UGT: |
7564 | case FCmpInst::FCMP_OGT: |
7565 | Pred = LHSUnsigned ? ICmpInst::ICMP_UGT : ICmpInst::ICMP_SGT; |
7566 | break; |
7567 | case FCmpInst::FCMP_UGE: |
7568 | case FCmpInst::FCMP_OGE: |
7569 | Pred = LHSUnsigned ? ICmpInst::ICMP_UGE : ICmpInst::ICMP_SGE; |
7570 | break; |
7571 | case FCmpInst::FCMP_ULT: |
7572 | case FCmpInst::FCMP_OLT: |
7573 | Pred = LHSUnsigned ? ICmpInst::ICMP_ULT : ICmpInst::ICMP_SLT; |
7574 | break; |
7575 | case FCmpInst::FCMP_ULE: |
7576 | case FCmpInst::FCMP_OLE: |
7577 | Pred = LHSUnsigned ? ICmpInst::ICMP_ULE : ICmpInst::ICMP_SLE; |
7578 | break; |
7579 | case FCmpInst::FCMP_UNE: |
7580 | case FCmpInst::FCMP_ONE: |
7581 | Pred = ICmpInst::ICMP_NE; |
7582 | break; |
7583 | case FCmpInst::FCMP_ORD: |
7584 | return replaceInstUsesWith(I, V: ConstantInt::getTrue(Ty: I.getType())); |
7585 | case FCmpInst::FCMP_UNO: |
7586 | return replaceInstUsesWith(I, V: ConstantInt::getFalse(Ty: I.getType())); |
7587 | } |
7588 | |
7589 | // Now we know that the APFloat is a normal number, zero or inf. |
7590 | |
7591 | // See if the FP constant is too large for the integer. For example, |
7592 | // comparing an i8 to 300.0. |
7593 | if (!LHSUnsigned) { |
7594 | // If the RHS value is > SignedMax, fold the comparison. This handles +INF |
7595 | // and large values. |
7596 | APFloat SMax(RHS->getSemantics()); |
7597 | SMax.convertFromAPInt(Input: APInt::getSignedMaxValue(numBits: IntWidth), IsSigned: true, |
7598 | RM: APFloat::rmNearestTiesToEven); |
7599 | if (SMax < *RHS) { // smax < 13123.0 |
7600 | if (Pred == ICmpInst::ICMP_NE || Pred == ICmpInst::ICMP_SLT || |
7601 | Pred == ICmpInst::ICMP_SLE) |
7602 | return replaceInstUsesWith(I, V: ConstantInt::getTrue(Ty: I.getType())); |
7603 | return replaceInstUsesWith(I, V: ConstantInt::getFalse(Ty: I.getType())); |
7604 | } |
7605 | } else { |
7606 | // If the RHS value is > UnsignedMax, fold the comparison. This handles |
7607 | // +INF and large values. |
7608 | APFloat UMax(RHS->getSemantics()); |
7609 | UMax.convertFromAPInt(Input: APInt::getMaxValue(numBits: IntWidth), IsSigned: false, |
7610 | RM: APFloat::rmNearestTiesToEven); |
7611 | if (UMax < *RHS) { // umax < 13123.0 |
7612 | if (Pred == ICmpInst::ICMP_NE || Pred == ICmpInst::ICMP_ULT || |
7613 | Pred == ICmpInst::ICMP_ULE) |
7614 | return replaceInstUsesWith(I, V: ConstantInt::getTrue(Ty: I.getType())); |
7615 | return replaceInstUsesWith(I, V: ConstantInt::getFalse(Ty: I.getType())); |
7616 | } |
7617 | } |
7618 | |
7619 | if (!LHSUnsigned) { |
7620 | // See if the RHS value is < SignedMin. |
7621 | APFloat SMin(RHS->getSemantics()); |
7622 | SMin.convertFromAPInt(Input: APInt::getSignedMinValue(numBits: IntWidth), IsSigned: true, |
7623 | RM: APFloat::rmNearestTiesToEven); |
7624 | if (SMin > *RHS) { // smin > 12312.0 |
7625 | if (Pred == ICmpInst::ICMP_NE || Pred == ICmpInst::ICMP_SGT || |
7626 | Pred == ICmpInst::ICMP_SGE) |
7627 | return replaceInstUsesWith(I, V: ConstantInt::getTrue(Ty: I.getType())); |
7628 | return replaceInstUsesWith(I, V: ConstantInt::getFalse(Ty: I.getType())); |
7629 | } |
7630 | } else { |
7631 | // See if the RHS value is < UnsignedMin. |
7632 | APFloat UMin(RHS->getSemantics()); |
7633 | UMin.convertFromAPInt(Input: APInt::getMinValue(numBits: IntWidth), IsSigned: false, |
7634 | RM: APFloat::rmNearestTiesToEven); |
7635 | if (UMin > *RHS) { // umin > 12312.0 |
7636 | if (Pred == ICmpInst::ICMP_NE || Pred == ICmpInst::ICMP_UGT || |
7637 | Pred == ICmpInst::ICMP_UGE) |
7638 | return replaceInstUsesWith(I, V: ConstantInt::getTrue(Ty: I.getType())); |
7639 | return replaceInstUsesWith(I, V: ConstantInt::getFalse(Ty: I.getType())); |
7640 | } |
7641 | } |
7642 | |
7643 | // Okay, now we know that the FP constant fits in the range [SMIN, SMAX] or |
7644 | // [0, UMAX], but it may still be fractional. Check whether this is the case |
7645 | // using the IsExact flag. |
7646 | // Don't do this for zero, because -0.0 is not fractional. |
7647 | APSInt RHSInt(IntWidth, LHSUnsigned); |
7648 | bool IsExact; |
7649 | RHS->convertToInteger(Result&: RHSInt, RM: APFloat::rmTowardZero, IsExact: &IsExact); |
7650 | if (!RHS->isZero()) { |
7651 | if (!IsExact) { |
7652 | // If we had a comparison against a fractional value, we have to adjust |
7653 | // the compare predicate and sometimes the value. RHSC is rounded towards |
7654 | // zero at this point. |
7655 | switch (Pred) { |
7656 | default: llvm_unreachable("Unexpected integer comparison!" ); |
7657 | case ICmpInst::ICMP_NE: // (float)int != 4.4 --> true |
7658 | return replaceInstUsesWith(I, V: ConstantInt::getTrue(Ty: I.getType())); |
7659 | case ICmpInst::ICMP_EQ: // (float)int == 4.4 --> false |
7660 | return replaceInstUsesWith(I, V: ConstantInt::getFalse(Ty: I.getType())); |
7661 | case ICmpInst::ICMP_ULE: |
7662 | // (float)int <= 4.4 --> int <= 4 |
7663 | // (float)int <= -4.4 --> false |
7664 | if (RHS->isNegative()) |
7665 | return replaceInstUsesWith(I, V: ConstantInt::getFalse(Ty: I.getType())); |
7666 | break; |
7667 | case ICmpInst::ICMP_SLE: |
7668 | // (float)int <= 4.4 --> int <= 4 |
7669 | // (float)int <= -4.4 --> int < -4 |
7670 | if (RHS->isNegative()) |
7671 | Pred = ICmpInst::ICMP_SLT; |
7672 | break; |
7673 | case ICmpInst::ICMP_ULT: |
7674 | // (float)int < -4.4 --> false |
7675 | // (float)int < 4.4 --> int <= 4 |
7676 | if (RHS->isNegative()) |
7677 | return replaceInstUsesWith(I, V: ConstantInt::getFalse(Ty: I.getType())); |
7678 | Pred = ICmpInst::ICMP_ULE; |
7679 | break; |
7680 | case ICmpInst::ICMP_SLT: |
7681 | // (float)int < -4.4 --> int < -4 |
7682 | // (float)int < 4.4 --> int <= 4 |
7683 | if (!RHS->isNegative()) |
7684 | Pred = ICmpInst::ICMP_SLE; |
7685 | break; |
7686 | case ICmpInst::ICMP_UGT: |
7687 | // (float)int > 4.4 --> int > 4 |
7688 | // (float)int > -4.4 --> true |
7689 | if (RHS->isNegative()) |
7690 | return replaceInstUsesWith(I, V: ConstantInt::getTrue(Ty: I.getType())); |
7691 | break; |
7692 | case ICmpInst::ICMP_SGT: |
7693 | // (float)int > 4.4 --> int > 4 |
7694 | // (float)int > -4.4 --> int >= -4 |
7695 | if (RHS->isNegative()) |
7696 | Pred = ICmpInst::ICMP_SGE; |
7697 | break; |
7698 | case ICmpInst::ICMP_UGE: |
7699 | // (float)int >= -4.4 --> true |
7700 | // (float)int >= 4.4 --> int > 4 |
7701 | if (RHS->isNegative()) |
7702 | return replaceInstUsesWith(I, V: ConstantInt::getTrue(Ty: I.getType())); |
7703 | Pred = ICmpInst::ICMP_UGT; |
7704 | break; |
7705 | case ICmpInst::ICMP_SGE: |
7706 | // (float)int >= -4.4 --> int >= -4 |
7707 | // (float)int >= 4.4 --> int > 4 |
7708 | if (!RHS->isNegative()) |
7709 | Pred = ICmpInst::ICMP_SGT; |
7710 | break; |
7711 | } |
7712 | } |
7713 | } |
7714 | |
7715 | // Lower this FP comparison into an appropriate integer version of the |
7716 | // comparison. |
7717 | return new ICmpInst(Pred, LHSI->getOperand(i: 0), |
7718 | ConstantInt::get(Ty: LHSI->getOperand(i: 0)->getType(), V: RHSInt)); |
7719 | } |
7720 | |
7721 | /// Fold (C / X) < 0.0 --> X < 0.0 if possible. Swap predicate if necessary. |
7722 | static Instruction *foldFCmpReciprocalAndZero(FCmpInst &I, Instruction *LHSI, |
7723 | Constant *RHSC) { |
7724 | // When C is not 0.0 and infinities are not allowed: |
7725 | // (C / X) < 0.0 is a sign-bit test of X |
7726 | // (C / X) < 0.0 --> X < 0.0 (if C is positive) |
7727 | // (C / X) < 0.0 --> X > 0.0 (if C is negative, swap the predicate) |
7728 | // |
7729 | // Proof: |
7730 | // Multiply (C / X) < 0.0 by X * X / C. |
7731 | // - X is non zero, if it is the flag 'ninf' is violated. |
7732 | // - C defines the sign of X * X * C. Thus it also defines whether to swap |
7733 | // the predicate. C is also non zero by definition. |
7734 | // |
7735 | // Thus X * X / C is non zero and the transformation is valid. [qed] |
7736 | |
7737 | FCmpInst::Predicate Pred = I.getPredicate(); |
7738 | |
7739 | // Check that predicates are valid. |
7740 | if ((Pred != FCmpInst::FCMP_OGT) && (Pred != FCmpInst::FCMP_OLT) && |
7741 | (Pred != FCmpInst::FCMP_OGE) && (Pred != FCmpInst::FCMP_OLE)) |
7742 | return nullptr; |
7743 | |
7744 | // Check that RHS operand is zero. |
7745 | if (!match(V: RHSC, P: m_AnyZeroFP())) |
7746 | return nullptr; |
7747 | |
7748 | // Check fastmath flags ('ninf'). |
7749 | if (!LHSI->hasNoInfs() || !I.hasNoInfs()) |
7750 | return nullptr; |
7751 | |
7752 | // Check the properties of the dividend. It must not be zero to avoid a |
7753 | // division by zero (see Proof). |
7754 | const APFloat *C; |
7755 | if (!match(V: LHSI->getOperand(i: 0), P: m_APFloat(Res&: C))) |
7756 | return nullptr; |
7757 | |
7758 | if (C->isZero()) |
7759 | return nullptr; |
7760 | |
7761 | // Get swapped predicate if necessary. |
7762 | if (C->isNegative()) |
7763 | Pred = I.getSwappedPredicate(); |
7764 | |
7765 | return new FCmpInst(Pred, LHSI->getOperand(i: 1), RHSC, "" , &I); |
7766 | } |
7767 | |
7768 | /// Optimize fabs(X) compared with zero. |
7769 | static Instruction *foldFabsWithFcmpZero(FCmpInst &I, InstCombinerImpl &IC) { |
7770 | Value *X; |
7771 | if (!match(V: I.getOperand(i_nocapture: 0), P: m_FAbs(Op0: m_Value(V&: X)))) |
7772 | return nullptr; |
7773 | |
7774 | const APFloat *C; |
7775 | if (!match(V: I.getOperand(i_nocapture: 1), P: m_APFloat(Res&: C))) |
7776 | return nullptr; |
7777 | |
7778 | if (!C->isPosZero()) { |
7779 | if (!C->isSmallestNormalized()) |
7780 | return nullptr; |
7781 | |
7782 | const Function *F = I.getFunction(); |
7783 | DenormalMode Mode = F->getDenormalMode(FPType: C->getSemantics()); |
7784 | if (Mode.Input == DenormalMode::PreserveSign || |
7785 | Mode.Input == DenormalMode::PositiveZero) { |
7786 | |
7787 | auto replaceFCmp = [](FCmpInst *I, FCmpInst::Predicate P, Value *X) { |
7788 | Constant *Zero = ConstantFP::getZero(Ty: X->getType()); |
7789 | return new FCmpInst(P, X, Zero, "" , I); |
7790 | }; |
7791 | |
7792 | switch (I.getPredicate()) { |
7793 | case FCmpInst::FCMP_OLT: |
7794 | // fcmp olt fabs(x), smallest_normalized_number -> fcmp oeq x, 0.0 |
7795 | return replaceFCmp(&I, FCmpInst::FCMP_OEQ, X); |
7796 | case FCmpInst::FCMP_UGE: |
7797 | // fcmp uge fabs(x), smallest_normalized_number -> fcmp une x, 0.0 |
7798 | return replaceFCmp(&I, FCmpInst::FCMP_UNE, X); |
7799 | case FCmpInst::FCMP_OGE: |
7800 | // fcmp oge fabs(x), smallest_normalized_number -> fcmp one x, 0.0 |
7801 | return replaceFCmp(&I, FCmpInst::FCMP_ONE, X); |
7802 | case FCmpInst::FCMP_ULT: |
7803 | // fcmp ult fabs(x), smallest_normalized_number -> fcmp ueq x, 0.0 |
7804 | return replaceFCmp(&I, FCmpInst::FCMP_UEQ, X); |
7805 | default: |
7806 | break; |
7807 | } |
7808 | } |
7809 | |
7810 | return nullptr; |
7811 | } |
7812 | |
7813 | auto replacePredAndOp0 = [&IC](FCmpInst *I, FCmpInst::Predicate P, Value *X) { |
7814 | I->setPredicate(P); |
7815 | return IC.replaceOperand(I&: *I, OpNum: 0, V: X); |
7816 | }; |
7817 | |
7818 | switch (I.getPredicate()) { |
7819 | case FCmpInst::FCMP_UGE: |
7820 | case FCmpInst::FCMP_OLT: |
7821 | // fabs(X) >= 0.0 --> true |
7822 | // fabs(X) < 0.0 --> false |
7823 | llvm_unreachable("fcmp should have simplified" ); |
7824 | |
7825 | case FCmpInst::FCMP_OGT: |
7826 | // fabs(X) > 0.0 --> X != 0.0 |
7827 | return replacePredAndOp0(&I, FCmpInst::FCMP_ONE, X); |
7828 | |
7829 | case FCmpInst::FCMP_UGT: |
7830 | // fabs(X) u> 0.0 --> X u!= 0.0 |
7831 | return replacePredAndOp0(&I, FCmpInst::FCMP_UNE, X); |
7832 | |
7833 | case FCmpInst::FCMP_OLE: |
7834 | // fabs(X) <= 0.0 --> X == 0.0 |
7835 | return replacePredAndOp0(&I, FCmpInst::FCMP_OEQ, X); |
7836 | |
7837 | case FCmpInst::FCMP_ULE: |
7838 | // fabs(X) u<= 0.0 --> X u== 0.0 |
7839 | return replacePredAndOp0(&I, FCmpInst::FCMP_UEQ, X); |
7840 | |
7841 | case FCmpInst::FCMP_OGE: |
7842 | // fabs(X) >= 0.0 --> !isnan(X) |
7843 | assert(!I.hasNoNaNs() && "fcmp should have simplified" ); |
7844 | return replacePredAndOp0(&I, FCmpInst::FCMP_ORD, X); |
7845 | |
7846 | case FCmpInst::FCMP_ULT: |
7847 | // fabs(X) u< 0.0 --> isnan(X) |
7848 | assert(!I.hasNoNaNs() && "fcmp should have simplified" ); |
7849 | return replacePredAndOp0(&I, FCmpInst::FCMP_UNO, X); |
7850 | |
7851 | case FCmpInst::FCMP_OEQ: |
7852 | case FCmpInst::FCMP_UEQ: |
7853 | case FCmpInst::FCMP_ONE: |
7854 | case FCmpInst::FCMP_UNE: |
7855 | case FCmpInst::FCMP_ORD: |
7856 | case FCmpInst::FCMP_UNO: |
7857 | // Look through the fabs() because it doesn't change anything but the sign. |
7858 | // fabs(X) == 0.0 --> X == 0.0, |
7859 | // fabs(X) != 0.0 --> X != 0.0 |
7860 | // isnan(fabs(X)) --> isnan(X) |
7861 | // !isnan(fabs(X) --> !isnan(X) |
7862 | return replacePredAndOp0(&I, I.getPredicate(), X); |
7863 | |
7864 | default: |
7865 | return nullptr; |
7866 | } |
7867 | } |
7868 | |
7869 | static Instruction *foldFCmpFNegCommonOp(FCmpInst &I) { |
7870 | CmpInst::Predicate Pred = I.getPredicate(); |
7871 | Value *Op0 = I.getOperand(i_nocapture: 0), *Op1 = I.getOperand(i_nocapture: 1); |
7872 | |
7873 | // Canonicalize fneg as Op1. |
7874 | if (match(V: Op0, P: m_FNeg(X: m_Value())) && !match(V: Op1, P: m_FNeg(X: m_Value()))) { |
7875 | std::swap(a&: Op0, b&: Op1); |
7876 | Pred = I.getSwappedPredicate(); |
7877 | } |
7878 | |
7879 | if (!match(V: Op1, P: m_FNeg(X: m_Specific(V: Op0)))) |
7880 | return nullptr; |
7881 | |
7882 | // Replace the negated operand with 0.0: |
7883 | // fcmp Pred Op0, -Op0 --> fcmp Pred Op0, 0.0 |
7884 | Constant *Zero = ConstantFP::getZero(Ty: Op0->getType()); |
7885 | return new FCmpInst(Pred, Op0, Zero, "" , &I); |
7886 | } |
7887 | |
7888 | Instruction *InstCombinerImpl::visitFCmpInst(FCmpInst &I) { |
7889 | bool Changed = false; |
7890 | |
7891 | /// Orders the operands of the compare so that they are listed from most |
7892 | /// complex to least complex. This puts constants before unary operators, |
7893 | /// before binary operators. |
7894 | if (getComplexity(V: I.getOperand(i_nocapture: 0)) < getComplexity(V: I.getOperand(i_nocapture: 1))) { |
7895 | I.swapOperands(); |
7896 | Changed = true; |
7897 | } |
7898 | |
7899 | const CmpInst::Predicate Pred = I.getPredicate(); |
7900 | Value *Op0 = I.getOperand(i_nocapture: 0), *Op1 = I.getOperand(i_nocapture: 1); |
7901 | if (Value *V = simplifyFCmpInst(Predicate: Pred, LHS: Op0, RHS: Op1, FMF: I.getFastMathFlags(), |
7902 | Q: SQ.getWithInstruction(I: &I))) |
7903 | return replaceInstUsesWith(I, V); |
7904 | |
7905 | // Simplify 'fcmp pred X, X' |
7906 | Type *OpType = Op0->getType(); |
7907 | assert(OpType == Op1->getType() && "fcmp with different-typed operands?" ); |
7908 | if (Op0 == Op1) { |
7909 | switch (Pred) { |
7910 | default: break; |
7911 | case FCmpInst::FCMP_UNO: // True if unordered: isnan(X) | isnan(Y) |
7912 | case FCmpInst::FCMP_ULT: // True if unordered or less than |
7913 | case FCmpInst::FCMP_UGT: // True if unordered or greater than |
7914 | case FCmpInst::FCMP_UNE: // True if unordered or not equal |
7915 | // Canonicalize these to be 'fcmp uno %X, 0.0'. |
7916 | I.setPredicate(FCmpInst::FCMP_UNO); |
7917 | I.setOperand(i_nocapture: 1, Val_nocapture: Constant::getNullValue(Ty: OpType)); |
7918 | return &I; |
7919 | |
7920 | case FCmpInst::FCMP_ORD: // True if ordered (no nans) |
7921 | case FCmpInst::FCMP_OEQ: // True if ordered and equal |
7922 | case FCmpInst::FCMP_OGE: // True if ordered and greater than or equal |
7923 | case FCmpInst::FCMP_OLE: // True if ordered and less than or equal |
7924 | // Canonicalize these to be 'fcmp ord %X, 0.0'. |
7925 | I.setPredicate(FCmpInst::FCMP_ORD); |
7926 | I.setOperand(i_nocapture: 1, Val_nocapture: Constant::getNullValue(Ty: OpType)); |
7927 | return &I; |
7928 | } |
7929 | } |
7930 | |
7931 | if (I.isCommutative()) { |
7932 | if (auto Pair = matchSymmetricPair(LHS: I.getOperand(i_nocapture: 0), RHS: I.getOperand(i_nocapture: 1))) { |
7933 | replaceOperand(I, OpNum: 0, V: Pair->first); |
7934 | replaceOperand(I, OpNum: 1, V: Pair->second); |
7935 | return &I; |
7936 | } |
7937 | } |
7938 | |
7939 | // If we're just checking for a NaN (ORD/UNO) and have a non-NaN operand, |
7940 | // then canonicalize the operand to 0.0. |
7941 | if (Pred == CmpInst::FCMP_ORD || Pred == CmpInst::FCMP_UNO) { |
7942 | if (!match(V: Op0, P: m_PosZeroFP()) && |
7943 | isKnownNeverNaN(V: Op0, Depth: 0, SQ: getSimplifyQuery().getWithInstruction(I: &I))) |
7944 | return replaceOperand(I, OpNum: 0, V: ConstantFP::getZero(Ty: OpType)); |
7945 | |
7946 | if (!match(V: Op1, P: m_PosZeroFP()) && |
7947 | isKnownNeverNaN(V: Op1, Depth: 0, SQ: getSimplifyQuery().getWithInstruction(I: &I))) |
7948 | return replaceOperand(I, OpNum: 1, V: ConstantFP::getZero(Ty: OpType)); |
7949 | } |
7950 | |
7951 | // fcmp pred (fneg X), (fneg Y) -> fcmp swap(pred) X, Y |
7952 | Value *X, *Y; |
7953 | if (match(V: Op0, P: m_FNeg(X: m_Value(V&: X))) && match(V: Op1, P: m_FNeg(X: m_Value(V&: Y)))) |
7954 | return new FCmpInst(I.getSwappedPredicate(), X, Y, "" , &I); |
7955 | |
7956 | if (Instruction *R = foldFCmpFNegCommonOp(I)) |
7957 | return R; |
7958 | |
7959 | // Test if the FCmpInst instruction is used exclusively by a select as |
7960 | // part of a minimum or maximum operation. If so, refrain from doing |
7961 | // any other folding. This helps out other analyses which understand |
7962 | // non-obfuscated minimum and maximum idioms, such as ScalarEvolution |
7963 | // and CodeGen. And in this case, at least one of the comparison |
7964 | // operands has at least one user besides the compare (the select), |
7965 | // which would often largely negate the benefit of folding anyway. |
7966 | if (I.hasOneUse()) |
7967 | if (SelectInst *SI = dyn_cast<SelectInst>(Val: I.user_back())) { |
7968 | Value *A, *B; |
7969 | SelectPatternResult SPR = matchSelectPattern(V: SI, LHS&: A, RHS&: B); |
7970 | if (SPR.Flavor != SPF_UNKNOWN) |
7971 | return nullptr; |
7972 | } |
7973 | |
7974 | // The sign of 0.0 is ignored by fcmp, so canonicalize to +0.0: |
7975 | // fcmp Pred X, -0.0 --> fcmp Pred X, 0.0 |
7976 | if (match(V: Op1, P: m_AnyZeroFP()) && !match(V: Op1, P: m_PosZeroFP())) |
7977 | return replaceOperand(I, OpNum: 1, V: ConstantFP::getZero(Ty: OpType)); |
7978 | |
7979 | // Canonicalize: |
7980 | // fcmp olt X, +inf -> fcmp one X, +inf |
7981 | // fcmp ole X, +inf -> fcmp ord X, 0 |
7982 | // fcmp ogt X, +inf -> false |
7983 | // fcmp oge X, +inf -> fcmp oeq X, +inf |
7984 | // fcmp ult X, +inf -> fcmp une X, +inf |
7985 | // fcmp ule X, +inf -> true |
7986 | // fcmp ugt X, +inf -> fcmp uno X, 0 |
7987 | // fcmp uge X, +inf -> fcmp ueq X, +inf |
7988 | // fcmp olt X, -inf -> false |
7989 | // fcmp ole X, -inf -> fcmp oeq X, -inf |
7990 | // fcmp ogt X, -inf -> fcmp one X, -inf |
7991 | // fcmp oge X, -inf -> fcmp ord X, 0 |
7992 | // fcmp ult X, -inf -> fcmp uno X, 0 |
7993 | // fcmp ule X, -inf -> fcmp ueq X, -inf |
7994 | // fcmp ugt X, -inf -> fcmp une X, -inf |
7995 | // fcmp uge X, -inf -> true |
7996 | const APFloat *C; |
7997 | if (match(V: Op1, P: m_APFloat(Res&: C)) && C->isInfinity()) { |
7998 | switch (C->isNegative() ? FCmpInst::getSwappedPredicate(pred: Pred) : Pred) { |
7999 | default: |
8000 | break; |
8001 | case FCmpInst::FCMP_ORD: |
8002 | case FCmpInst::FCMP_UNO: |
8003 | case FCmpInst::FCMP_TRUE: |
8004 | case FCmpInst::FCMP_FALSE: |
8005 | case FCmpInst::FCMP_OGT: |
8006 | case FCmpInst::FCMP_ULE: |
8007 | llvm_unreachable("Should be simplified by InstSimplify" ); |
8008 | case FCmpInst::FCMP_OLT: |
8009 | return new FCmpInst(FCmpInst::FCMP_ONE, Op0, Op1, "" , &I); |
8010 | case FCmpInst::FCMP_OLE: |
8011 | return new FCmpInst(FCmpInst::FCMP_ORD, Op0, ConstantFP::getZero(Ty: OpType), |
8012 | "" , &I); |
8013 | case FCmpInst::FCMP_OGE: |
8014 | return new FCmpInst(FCmpInst::FCMP_OEQ, Op0, Op1, "" , &I); |
8015 | case FCmpInst::FCMP_ULT: |
8016 | return new FCmpInst(FCmpInst::FCMP_UNE, Op0, Op1, "" , &I); |
8017 | case FCmpInst::FCMP_UGT: |
8018 | return new FCmpInst(FCmpInst::FCMP_UNO, Op0, ConstantFP::getZero(Ty: OpType), |
8019 | "" , &I); |
8020 | case FCmpInst::FCMP_UGE: |
8021 | return new FCmpInst(FCmpInst::FCMP_UEQ, Op0, Op1, "" , &I); |
8022 | } |
8023 | } |
8024 | |
8025 | // Ignore signbit of bitcasted int when comparing equality to FP 0.0: |
8026 | // fcmp oeq/une (bitcast X), 0.0 --> (and X, SignMaskC) ==/!= 0 |
8027 | if (match(V: Op1, P: m_PosZeroFP()) && |
8028 | match(V: Op0, P: m_OneUse(SubPattern: m_ElementWiseBitCast(Op: m_Value(V&: X))))) { |
8029 | ICmpInst::Predicate IntPred = ICmpInst::BAD_ICMP_PREDICATE; |
8030 | if (Pred == FCmpInst::FCMP_OEQ) |
8031 | IntPred = ICmpInst::ICMP_EQ; |
8032 | else if (Pred == FCmpInst::FCMP_UNE) |
8033 | IntPred = ICmpInst::ICMP_NE; |
8034 | |
8035 | if (IntPred != ICmpInst::BAD_ICMP_PREDICATE) { |
8036 | Type *IntTy = X->getType(); |
8037 | const APInt &SignMask = ~APInt::getSignMask(BitWidth: IntTy->getScalarSizeInBits()); |
8038 | Value *MaskX = Builder.CreateAnd(LHS: X, RHS: ConstantInt::get(Ty: IntTy, V: SignMask)); |
8039 | return new ICmpInst(IntPred, MaskX, ConstantInt::getNullValue(Ty: IntTy)); |
8040 | } |
8041 | } |
8042 | |
8043 | // Handle fcmp with instruction LHS and constant RHS. |
8044 | Instruction *LHSI; |
8045 | Constant *RHSC; |
8046 | if (match(V: Op0, P: m_Instruction(I&: LHSI)) && match(V: Op1, P: m_Constant(C&: RHSC))) { |
8047 | switch (LHSI->getOpcode()) { |
8048 | case Instruction::Select: |
8049 | // fcmp eq (cond ? x : -x), 0 --> fcmp eq x, 0 |
8050 | if (FCmpInst::isEquality(Pred) && match(V: RHSC, P: m_AnyZeroFP()) && |
8051 | (match(V: LHSI, |
8052 | P: m_Select(C: m_Value(), L: m_Value(V&: X), R: m_FNeg(X: m_Deferred(V: X)))) || |
8053 | match(V: LHSI, P: m_Select(C: m_Value(), L: m_FNeg(X: m_Value(V&: X)), R: m_Deferred(V: X))))) |
8054 | return replaceOperand(I, OpNum: 0, V: X); |
8055 | if (Instruction *NV = FoldOpIntoSelect(Op&: I, SI: cast<SelectInst>(Val: LHSI))) |
8056 | return NV; |
8057 | break; |
8058 | case Instruction::PHI: |
8059 | if (Instruction *NV = foldOpIntoPhi(I, PN: cast<PHINode>(Val: LHSI))) |
8060 | return NV; |
8061 | break; |
8062 | case Instruction::SIToFP: |
8063 | case Instruction::UIToFP: |
8064 | if (Instruction *NV = foldFCmpIntToFPConst(I, LHSI, RHSC)) |
8065 | return NV; |
8066 | break; |
8067 | case Instruction::FDiv: |
8068 | if (Instruction *NV = foldFCmpReciprocalAndZero(I, LHSI, RHSC)) |
8069 | return NV; |
8070 | break; |
8071 | case Instruction::Load: |
8072 | if (auto *GEP = dyn_cast<GetElementPtrInst>(Val: LHSI->getOperand(i: 0))) |
8073 | if (auto *GV = dyn_cast<GlobalVariable>(Val: GEP->getOperand(i_nocapture: 0))) |
8074 | if (Instruction *Res = foldCmpLoadFromIndexedGlobal( |
8075 | LI: cast<LoadInst>(Val: LHSI), GEP, GV, ICI&: I)) |
8076 | return Res; |
8077 | break; |
8078 | } |
8079 | } |
8080 | |
8081 | if (Instruction *R = foldFabsWithFcmpZero(I, IC&: *this)) |
8082 | return R; |
8083 | |
8084 | if (match(V: Op0, P: m_FNeg(X: m_Value(V&: X)))) { |
8085 | // fcmp pred (fneg X), C --> fcmp swap(pred) X, -C |
8086 | Constant *C; |
8087 | if (match(V: Op1, P: m_Constant(C))) |
8088 | if (Constant *NegC = ConstantFoldUnaryOpOperand(Opcode: Instruction::FNeg, Op: C, DL)) |
8089 | return new FCmpInst(I.getSwappedPredicate(), X, NegC, "" , &I); |
8090 | } |
8091 | |
8092 | // fcmp (fadd X, 0.0), Y --> fcmp X, Y |
8093 | if (match(V: Op0, P: m_FAdd(L: m_Value(V&: X), R: m_AnyZeroFP()))) |
8094 | return new FCmpInst(Pred, X, Op1, "" , &I); |
8095 | |
8096 | // fcmp X, (fadd Y, 0.0) --> fcmp X, Y |
8097 | if (match(V: Op1, P: m_FAdd(L: m_Value(V&: Y), R: m_AnyZeroFP()))) |
8098 | return new FCmpInst(Pred, Op0, Y, "" , &I); |
8099 | |
8100 | if (match(V: Op0, P: m_FPExt(Op: m_Value(V&: X)))) { |
8101 | // fcmp (fpext X), (fpext Y) -> fcmp X, Y |
8102 | if (match(V: Op1, P: m_FPExt(Op: m_Value(V&: Y))) && X->getType() == Y->getType()) |
8103 | return new FCmpInst(Pred, X, Y, "" , &I); |
8104 | |
8105 | const APFloat *C; |
8106 | if (match(V: Op1, P: m_APFloat(Res&: C))) { |
8107 | const fltSemantics &FPSem = |
8108 | X->getType()->getScalarType()->getFltSemantics(); |
8109 | bool Lossy; |
8110 | APFloat TruncC = *C; |
8111 | TruncC.convert(ToSemantics: FPSem, RM: APFloat::rmNearestTiesToEven, losesInfo: &Lossy); |
8112 | |
8113 | if (Lossy) { |
8114 | // X can't possibly equal the higher-precision constant, so reduce any |
8115 | // equality comparison. |
8116 | // TODO: Other predicates can be handled via getFCmpCode(). |
8117 | switch (Pred) { |
8118 | case FCmpInst::FCMP_OEQ: |
8119 | // X is ordered and equal to an impossible constant --> false |
8120 | return replaceInstUsesWith(I, V: ConstantInt::getFalse(Ty: I.getType())); |
8121 | case FCmpInst::FCMP_ONE: |
8122 | // X is ordered and not equal to an impossible constant --> ordered |
8123 | return new FCmpInst(FCmpInst::FCMP_ORD, X, |
8124 | ConstantFP::getZero(Ty: X->getType())); |
8125 | case FCmpInst::FCMP_UEQ: |
8126 | // X is unordered or equal to an impossible constant --> unordered |
8127 | return new FCmpInst(FCmpInst::FCMP_UNO, X, |
8128 | ConstantFP::getZero(Ty: X->getType())); |
8129 | case FCmpInst::FCMP_UNE: |
8130 | // X is unordered or not equal to an impossible constant --> true |
8131 | return replaceInstUsesWith(I, V: ConstantInt::getTrue(Ty: I.getType())); |
8132 | default: |
8133 | break; |
8134 | } |
8135 | } |
8136 | |
8137 | // fcmp (fpext X), C -> fcmp X, (fptrunc C) if fptrunc is lossless |
8138 | // Avoid lossy conversions and denormals. |
8139 | // Zero is a special case that's OK to convert. |
8140 | APFloat Fabs = TruncC; |
8141 | Fabs.clearSign(); |
8142 | if (!Lossy && |
8143 | (Fabs.isZero() || !(Fabs < APFloat::getSmallestNormalized(Sem: FPSem)))) { |
8144 | Constant *NewC = ConstantFP::get(Ty: X->getType(), V: TruncC); |
8145 | return new FCmpInst(Pred, X, NewC, "" , &I); |
8146 | } |
8147 | } |
8148 | } |
8149 | |
8150 | // Convert a sign-bit test of an FP value into a cast and integer compare. |
8151 | // TODO: Simplify if the copysign constant is 0.0 or NaN. |
8152 | // TODO: Handle non-zero compare constants. |
8153 | // TODO: Handle other predicates. |
8154 | if (match(Op0, m_OneUse(m_Intrinsic<Intrinsic::copysign>(m_APFloat(C), |
8155 | m_Value(X)))) && |
8156 | match(Op1, m_AnyZeroFP()) && !C->isZero() && !C->isNaN()) { |
8157 | Type *IntType = Builder.getIntNTy(N: X->getType()->getScalarSizeInBits()); |
8158 | if (auto *VecTy = dyn_cast<VectorType>(Val: OpType)) |
8159 | IntType = VectorType::get(ElementType: IntType, EC: VecTy->getElementCount()); |
8160 | |
8161 | // copysign(non-zero constant, X) < 0.0 --> (bitcast X) < 0 |
8162 | if (Pred == FCmpInst::FCMP_OLT) { |
8163 | Value *IntX = Builder.CreateBitCast(V: X, DestTy: IntType); |
8164 | return new ICmpInst(ICmpInst::ICMP_SLT, IntX, |
8165 | ConstantInt::getNullValue(Ty: IntType)); |
8166 | } |
8167 | } |
8168 | |
8169 | { |
8170 | Value *CanonLHS = nullptr, *CanonRHS = nullptr; |
8171 | match(Op0, m_Intrinsic<Intrinsic::canonicalize>(m_Value(CanonLHS))); |
8172 | match(Op1, m_Intrinsic<Intrinsic::canonicalize>(m_Value(CanonRHS))); |
8173 | |
8174 | // (canonicalize(x) == x) => (x == x) |
8175 | if (CanonLHS == Op1) |
8176 | return new FCmpInst(Pred, Op1, Op1, "" , &I); |
8177 | |
8178 | // (x == canonicalize(x)) => (x == x) |
8179 | if (CanonRHS == Op0) |
8180 | return new FCmpInst(Pred, Op0, Op0, "" , &I); |
8181 | |
8182 | // (canonicalize(x) == canonicalize(y)) => (x == y) |
8183 | if (CanonLHS && CanonRHS) |
8184 | return new FCmpInst(Pred, CanonLHS, CanonRHS, "" , &I); |
8185 | } |
8186 | |
8187 | if (I.getType()->isVectorTy()) |
8188 | if (Instruction *Res = foldVectorCmp(Cmp&: I, Builder)) |
8189 | return Res; |
8190 | |
8191 | return Changed ? &I : nullptr; |
8192 | } |
8193 | |