1 | //===- AttributorAttributes.cpp - Attributes for Attributor deduction -----===// |
2 | // |
3 | // Part of the LLVM Project, under the Apache License v2.0 with LLVM Exceptions. |
4 | // See https://llvm.org/LICENSE.txt for license information. |
5 | // SPDX-License-Identifier: Apache-2.0 WITH LLVM-exception |
6 | // |
7 | //===----------------------------------------------------------------------===// |
8 | // |
9 | // See the Attributor.h file comment and the class descriptions in that file for |
10 | // more information. |
11 | // |
12 | //===----------------------------------------------------------------------===// |
13 | |
14 | #include "llvm/Transforms/IPO/Attributor.h" |
15 | |
16 | #include "llvm/ADT/APInt.h" |
17 | #include "llvm/ADT/ArrayRef.h" |
18 | #include "llvm/ADT/DenseMapInfo.h" |
19 | #include "llvm/ADT/MapVector.h" |
20 | #include "llvm/ADT/SCCIterator.h" |
21 | #include "llvm/ADT/STLExtras.h" |
22 | #include "llvm/ADT/SetOperations.h" |
23 | #include "llvm/ADT/SetVector.h" |
24 | #include "llvm/ADT/SmallPtrSet.h" |
25 | #include "llvm/ADT/SmallVector.h" |
26 | #include "llvm/ADT/Statistic.h" |
27 | #include "llvm/ADT/StringExtras.h" |
28 | #include "llvm/Analysis/AliasAnalysis.h" |
29 | #include "llvm/Analysis/AssumeBundleQueries.h" |
30 | #include "llvm/Analysis/AssumptionCache.h" |
31 | #include "llvm/Analysis/CaptureTracking.h" |
32 | #include "llvm/Analysis/CycleAnalysis.h" |
33 | #include "llvm/Analysis/InstructionSimplify.h" |
34 | #include "llvm/Analysis/LazyValueInfo.h" |
35 | #include "llvm/Analysis/MemoryBuiltins.h" |
36 | #include "llvm/Analysis/OptimizationRemarkEmitter.h" |
37 | #include "llvm/Analysis/ScalarEvolution.h" |
38 | #include "llvm/Analysis/TargetTransformInfo.h" |
39 | #include "llvm/Analysis/ValueTracking.h" |
40 | #include "llvm/IR/Argument.h" |
41 | #include "llvm/IR/Assumptions.h" |
42 | #include "llvm/IR/Attributes.h" |
43 | #include "llvm/IR/BasicBlock.h" |
44 | #include "llvm/IR/Constant.h" |
45 | #include "llvm/IR/Constants.h" |
46 | #include "llvm/IR/DataLayout.h" |
47 | #include "llvm/IR/DerivedTypes.h" |
48 | #include "llvm/IR/GlobalValue.h" |
49 | #include "llvm/IR/IRBuilder.h" |
50 | #include "llvm/IR/InlineAsm.h" |
51 | #include "llvm/IR/InstrTypes.h" |
52 | #include "llvm/IR/Instruction.h" |
53 | #include "llvm/IR/Instructions.h" |
54 | #include "llvm/IR/IntrinsicInst.h" |
55 | #include "llvm/IR/IntrinsicsAMDGPU.h" |
56 | #include "llvm/IR/IntrinsicsNVPTX.h" |
57 | #include "llvm/IR/LLVMContext.h" |
58 | #include "llvm/IR/MDBuilder.h" |
59 | #include "llvm/IR/NoFolder.h" |
60 | #include "llvm/IR/Value.h" |
61 | #include "llvm/IR/ValueHandle.h" |
62 | #include "llvm/Support/Alignment.h" |
63 | #include "llvm/Support/Casting.h" |
64 | #include "llvm/Support/CommandLine.h" |
65 | #include "llvm/Support/ErrorHandling.h" |
66 | #include "llvm/Support/GraphWriter.h" |
67 | #include "llvm/Support/MathExtras.h" |
68 | #include "llvm/Support/TypeSize.h" |
69 | #include "llvm/Support/raw_ostream.h" |
70 | #include "llvm/Transforms/Utils/BasicBlockUtils.h" |
71 | #include "llvm/Transforms/Utils/CallPromotionUtils.h" |
72 | #include "llvm/Transforms/Utils/Local.h" |
73 | #include "llvm/Transforms/Utils/ValueMapper.h" |
74 | #include <cassert> |
75 | #include <numeric> |
76 | #include <optional> |
77 | #include <string> |
78 | |
79 | using namespace llvm; |
80 | |
81 | #define DEBUG_TYPE "attributor" |
82 | |
83 | static cl::opt<bool> ManifestInternal( |
84 | "attributor-manifest-internal" , cl::Hidden, |
85 | cl::desc("Manifest Attributor internal string attributes." ), |
86 | cl::init(Val: false)); |
87 | |
88 | static cl::opt<int> MaxHeapToStackSize("max-heap-to-stack-size" , cl::init(Val: 128), |
89 | cl::Hidden); |
90 | |
91 | template <> |
92 | unsigned llvm::PotentialConstantIntValuesState::MaxPotentialValues = 0; |
93 | |
94 | template <> unsigned llvm::PotentialLLVMValuesState::MaxPotentialValues = -1; |
95 | |
96 | static cl::opt<unsigned, true> MaxPotentialValues( |
97 | "attributor-max-potential-values" , cl::Hidden, |
98 | cl::desc("Maximum number of potential values to be " |
99 | "tracked for each position." ), |
100 | cl::location(L&: llvm::PotentialConstantIntValuesState::MaxPotentialValues), |
101 | cl::init(Val: 7)); |
102 | |
103 | static cl::opt<int> MaxPotentialValuesIterations( |
104 | "attributor-max-potential-values-iterations" , cl::Hidden, |
105 | cl::desc( |
106 | "Maximum number of iterations we keep dismantling potential values." ), |
107 | cl::init(Val: 64)); |
108 | |
109 | STATISTIC(NumAAs, "Number of abstract attributes created" ); |
110 | |
111 | // Some helper macros to deal with statistics tracking. |
112 | // |
113 | // Usage: |
114 | // For simple IR attribute tracking overload trackStatistics in the abstract |
115 | // attribute and choose the right STATS_DECLTRACK_********* macro, |
116 | // e.g.,: |
117 | // void trackStatistics() const override { |
118 | // STATS_DECLTRACK_ARG_ATTR(returned) |
119 | // } |
120 | // If there is a single "increment" side one can use the macro |
121 | // STATS_DECLTRACK with a custom message. If there are multiple increment |
122 | // sides, STATS_DECL and STATS_TRACK can also be used separately. |
123 | // |
124 | #define BUILD_STAT_MSG_IR_ATTR(TYPE, NAME) \ |
125 | ("Number of " #TYPE " marked '" #NAME "'") |
126 | #define BUILD_STAT_NAME(NAME, TYPE) NumIR##TYPE##_##NAME |
127 | #define STATS_DECL_(NAME, MSG) STATISTIC(NAME, MSG); |
128 | #define STATS_DECL(NAME, TYPE, MSG) \ |
129 | STATS_DECL_(BUILD_STAT_NAME(NAME, TYPE), MSG); |
130 | #define STATS_TRACK(NAME, TYPE) ++(BUILD_STAT_NAME(NAME, TYPE)); |
131 | #define STATS_DECLTRACK(NAME, TYPE, MSG) \ |
132 | { \ |
133 | STATS_DECL(NAME, TYPE, MSG) \ |
134 | STATS_TRACK(NAME, TYPE) \ |
135 | } |
136 | #define STATS_DECLTRACK_ARG_ATTR(NAME) \ |
137 | STATS_DECLTRACK(NAME, Arguments, BUILD_STAT_MSG_IR_ATTR(arguments, NAME)) |
138 | #define STATS_DECLTRACK_CSARG_ATTR(NAME) \ |
139 | STATS_DECLTRACK(NAME, CSArguments, \ |
140 | BUILD_STAT_MSG_IR_ATTR(call site arguments, NAME)) |
141 | #define STATS_DECLTRACK_FN_ATTR(NAME) \ |
142 | STATS_DECLTRACK(NAME, Function, BUILD_STAT_MSG_IR_ATTR(functions, NAME)) |
143 | #define STATS_DECLTRACK_CS_ATTR(NAME) \ |
144 | STATS_DECLTRACK(NAME, CS, BUILD_STAT_MSG_IR_ATTR(call site, NAME)) |
145 | #define STATS_DECLTRACK_FNRET_ATTR(NAME) \ |
146 | STATS_DECLTRACK(NAME, FunctionReturn, \ |
147 | BUILD_STAT_MSG_IR_ATTR(function returns, NAME)) |
148 | #define STATS_DECLTRACK_CSRET_ATTR(NAME) \ |
149 | STATS_DECLTRACK(NAME, CSReturn, \ |
150 | BUILD_STAT_MSG_IR_ATTR(call site returns, NAME)) |
151 | #define STATS_DECLTRACK_FLOATING_ATTR(NAME) \ |
152 | STATS_DECLTRACK(NAME, Floating, \ |
153 | ("Number of floating values known to be '" #NAME "'")) |
154 | |
155 | // Specialization of the operator<< for abstract attributes subclasses. This |
156 | // disambiguates situations where multiple operators are applicable. |
157 | namespace llvm { |
158 | #define PIPE_OPERATOR(CLASS) \ |
159 | raw_ostream &operator<<(raw_ostream &OS, const CLASS &AA) { \ |
160 | return OS << static_cast<const AbstractAttribute &>(AA); \ |
161 | } |
162 | |
163 | PIPE_OPERATOR(AAIsDead) |
164 | PIPE_OPERATOR(AANoUnwind) |
165 | PIPE_OPERATOR(AANoSync) |
166 | PIPE_OPERATOR(AANoRecurse) |
167 | PIPE_OPERATOR(AANonConvergent) |
168 | PIPE_OPERATOR(AAWillReturn) |
169 | PIPE_OPERATOR(AANoReturn) |
170 | PIPE_OPERATOR(AANonNull) |
171 | PIPE_OPERATOR(AAMustProgress) |
172 | PIPE_OPERATOR(AANoAlias) |
173 | PIPE_OPERATOR(AADereferenceable) |
174 | PIPE_OPERATOR(AAAlign) |
175 | PIPE_OPERATOR(AAInstanceInfo) |
176 | PIPE_OPERATOR(AANoCapture) |
177 | PIPE_OPERATOR(AAValueSimplify) |
178 | PIPE_OPERATOR(AANoFree) |
179 | PIPE_OPERATOR(AAHeapToStack) |
180 | PIPE_OPERATOR(AAIntraFnReachability) |
181 | PIPE_OPERATOR(AAMemoryBehavior) |
182 | PIPE_OPERATOR(AAMemoryLocation) |
183 | PIPE_OPERATOR(AAValueConstantRange) |
184 | PIPE_OPERATOR(AAPrivatizablePtr) |
185 | PIPE_OPERATOR(AAUndefinedBehavior) |
186 | PIPE_OPERATOR(AAPotentialConstantValues) |
187 | PIPE_OPERATOR(AAPotentialValues) |
188 | PIPE_OPERATOR(AANoUndef) |
189 | PIPE_OPERATOR(AANoFPClass) |
190 | PIPE_OPERATOR(AACallEdges) |
191 | PIPE_OPERATOR(AAInterFnReachability) |
192 | PIPE_OPERATOR(AAPointerInfo) |
193 | PIPE_OPERATOR(AAAssumptionInfo) |
194 | PIPE_OPERATOR(AAUnderlyingObjects) |
195 | PIPE_OPERATOR(AAAddressSpace) |
196 | PIPE_OPERATOR(AAAllocationInfo) |
197 | PIPE_OPERATOR(AAIndirectCallInfo) |
198 | PIPE_OPERATOR(AAGlobalValueInfo) |
199 | PIPE_OPERATOR(AADenormalFPMath) |
200 | |
201 | #undef PIPE_OPERATOR |
202 | |
203 | template <> |
204 | ChangeStatus clampStateAndIndicateChange<DerefState>(DerefState &S, |
205 | const DerefState &R) { |
206 | ChangeStatus CS0 = |
207 | clampStateAndIndicateChange(S&: S.DerefBytesState, R: R.DerefBytesState); |
208 | ChangeStatus CS1 = clampStateAndIndicateChange(S&: S.GlobalState, R: R.GlobalState); |
209 | return CS0 | CS1; |
210 | } |
211 | |
212 | } // namespace llvm |
213 | |
214 | static bool mayBeInCycle(const CycleInfo *CI, const Instruction *I, |
215 | bool , Cycle **CPtr = nullptr) { |
216 | if (!CI) |
217 | return true; |
218 | auto *BB = I->getParent(); |
219 | auto *C = CI->getCycle(Block: BB); |
220 | if (!C) |
221 | return false; |
222 | if (CPtr) |
223 | *CPtr = C; |
224 | return !HeaderOnly || BB == C->getHeader(); |
225 | } |
226 | |
227 | /// Checks if a type could have padding bytes. |
228 | static bool isDenselyPacked(Type *Ty, const DataLayout &DL) { |
229 | // There is no size information, so be conservative. |
230 | if (!Ty->isSized()) |
231 | return false; |
232 | |
233 | // If the alloc size is not equal to the storage size, then there are padding |
234 | // bytes. For x86_fp80 on x86-64, size: 80 alloc size: 128. |
235 | if (DL.getTypeSizeInBits(Ty) != DL.getTypeAllocSizeInBits(Ty)) |
236 | return false; |
237 | |
238 | // FIXME: This isn't the right way to check for padding in vectors with |
239 | // non-byte-size elements. |
240 | if (VectorType *SeqTy = dyn_cast<VectorType>(Val: Ty)) |
241 | return isDenselyPacked(Ty: SeqTy->getElementType(), DL); |
242 | |
243 | // For array types, check for padding within members. |
244 | if (ArrayType *SeqTy = dyn_cast<ArrayType>(Val: Ty)) |
245 | return isDenselyPacked(Ty: SeqTy->getElementType(), DL); |
246 | |
247 | if (!isa<StructType>(Val: Ty)) |
248 | return true; |
249 | |
250 | // Check for padding within and between elements of a struct. |
251 | StructType *StructTy = cast<StructType>(Val: Ty); |
252 | const StructLayout *Layout = DL.getStructLayout(Ty: StructTy); |
253 | uint64_t StartPos = 0; |
254 | for (unsigned I = 0, E = StructTy->getNumElements(); I < E; ++I) { |
255 | Type *ElTy = StructTy->getElementType(N: I); |
256 | if (!isDenselyPacked(Ty: ElTy, DL)) |
257 | return false; |
258 | if (StartPos != Layout->getElementOffsetInBits(Idx: I)) |
259 | return false; |
260 | StartPos += DL.getTypeAllocSizeInBits(Ty: ElTy); |
261 | } |
262 | |
263 | return true; |
264 | } |
265 | |
266 | /// Get pointer operand of memory accessing instruction. If \p I is |
267 | /// not a memory accessing instruction, return nullptr. If \p AllowVolatile, |
268 | /// is set to false and the instruction is volatile, return nullptr. |
269 | static const Value *getPointerOperand(const Instruction *I, |
270 | bool AllowVolatile) { |
271 | if (!AllowVolatile && I->isVolatile()) |
272 | return nullptr; |
273 | |
274 | if (auto *LI = dyn_cast<LoadInst>(Val: I)) { |
275 | return LI->getPointerOperand(); |
276 | } |
277 | |
278 | if (auto *SI = dyn_cast<StoreInst>(Val: I)) { |
279 | return SI->getPointerOperand(); |
280 | } |
281 | |
282 | if (auto *CXI = dyn_cast<AtomicCmpXchgInst>(Val: I)) { |
283 | return CXI->getPointerOperand(); |
284 | } |
285 | |
286 | if (auto *RMWI = dyn_cast<AtomicRMWInst>(Val: I)) { |
287 | return RMWI->getPointerOperand(); |
288 | } |
289 | |
290 | return nullptr; |
291 | } |
292 | |
293 | /// Helper function to create a pointer based on \p Ptr, and advanced by \p |
294 | /// Offset bytes. |
295 | static Value *constructPointer(Value *Ptr, int64_t Offset, |
296 | IRBuilder<NoFolder> &IRB) { |
297 | LLVM_DEBUG(dbgs() << "Construct pointer: " << *Ptr << " + " << Offset |
298 | << "-bytes\n" ); |
299 | |
300 | if (Offset) |
301 | Ptr = IRB.CreatePtrAdd(Ptr, Offset: IRB.getInt64(C: Offset), |
302 | Name: Ptr->getName() + ".b" + Twine(Offset)); |
303 | return Ptr; |
304 | } |
305 | |
306 | static const Value * |
307 | stripAndAccumulateOffsets(Attributor &A, const AbstractAttribute &QueryingAA, |
308 | const Value *Val, const DataLayout &DL, APInt &Offset, |
309 | bool GetMinOffset, bool AllowNonInbounds, |
310 | bool UseAssumed = false) { |
311 | |
312 | auto AttributorAnalysis = [&](Value &V, APInt &ROffset) -> bool { |
313 | const IRPosition &Pos = IRPosition::value(V); |
314 | // Only track dependence if we are going to use the assumed info. |
315 | const AAValueConstantRange *ValueConstantRangeAA = |
316 | A.getAAFor<AAValueConstantRange>(QueryingAA, IRP: Pos, |
317 | DepClass: UseAssumed ? DepClassTy::OPTIONAL |
318 | : DepClassTy::NONE); |
319 | if (!ValueConstantRangeAA) |
320 | return false; |
321 | ConstantRange Range = UseAssumed ? ValueConstantRangeAA->getAssumed() |
322 | : ValueConstantRangeAA->getKnown(); |
323 | if (Range.isFullSet()) |
324 | return false; |
325 | |
326 | // We can only use the lower part of the range because the upper part can |
327 | // be higher than what the value can really be. |
328 | if (GetMinOffset) |
329 | ROffset = Range.getSignedMin(); |
330 | else |
331 | ROffset = Range.getSignedMax(); |
332 | return true; |
333 | }; |
334 | |
335 | return Val->stripAndAccumulateConstantOffsets(DL, Offset, AllowNonInbounds, |
336 | /* AllowInvariant */ AllowInvariantGroup: true, |
337 | ExternalAnalysis: AttributorAnalysis); |
338 | } |
339 | |
340 | static const Value * |
341 | getMinimalBaseOfPointer(Attributor &A, const AbstractAttribute &QueryingAA, |
342 | const Value *Ptr, int64_t &BytesOffset, |
343 | const DataLayout &DL, bool AllowNonInbounds = false) { |
344 | APInt OffsetAPInt(DL.getIndexTypeSizeInBits(Ty: Ptr->getType()), 0); |
345 | const Value *Base = |
346 | stripAndAccumulateOffsets(A, QueryingAA, Val: Ptr, DL, Offset&: OffsetAPInt, |
347 | /* GetMinOffset */ true, AllowNonInbounds); |
348 | |
349 | BytesOffset = OffsetAPInt.getSExtValue(); |
350 | return Base; |
351 | } |
352 | |
353 | /// Clamp the information known for all returned values of a function |
354 | /// (identified by \p QueryingAA) into \p S. |
355 | template <typename AAType, typename StateType = typename AAType::StateType, |
356 | Attribute::AttrKind IRAttributeKind = AAType::IRAttributeKind, |
357 | bool RecurseForSelectAndPHI = true> |
358 | static void clampReturnedValueStates( |
359 | Attributor &A, const AAType &QueryingAA, StateType &S, |
360 | const IRPosition::CallBaseContext *CBContext = nullptr) { |
361 | LLVM_DEBUG(dbgs() << "[Attributor] Clamp return value states for " |
362 | << QueryingAA << " into " << S << "\n" ); |
363 | |
364 | assert((QueryingAA.getIRPosition().getPositionKind() == |
365 | IRPosition::IRP_RETURNED || |
366 | QueryingAA.getIRPosition().getPositionKind() == |
367 | IRPosition::IRP_CALL_SITE_RETURNED) && |
368 | "Can only clamp returned value states for a function returned or call " |
369 | "site returned position!" ); |
370 | |
371 | // Use an optional state as there might not be any return values and we want |
372 | // to join (IntegerState::operator&) the state of all there are. |
373 | std::optional<StateType> T; |
374 | |
375 | // Callback for each possibly returned value. |
376 | auto CheckReturnValue = [&](Value &RV) -> bool { |
377 | const IRPosition &RVPos = IRPosition::value(V: RV, CBContext); |
378 | // If possible, use the hasAssumedIRAttr interface. |
379 | if (Attribute::isEnumAttrKind(Kind: IRAttributeKind)) { |
380 | bool IsKnown; |
381 | return AA::hasAssumedIRAttr<IRAttributeKind>( |
382 | A, &QueryingAA, RVPos, DepClassTy::REQUIRED, IsKnown); |
383 | } |
384 | |
385 | const AAType *AA = |
386 | A.getAAFor<AAType>(QueryingAA, RVPos, DepClassTy::REQUIRED); |
387 | if (!AA) |
388 | return false; |
389 | LLVM_DEBUG(dbgs() << "[Attributor] RV: " << RV |
390 | << " AA: " << AA->getAsStr(&A) << " @ " << RVPos << "\n" ); |
391 | const StateType &AAS = AA->getState(); |
392 | if (!T) |
393 | T = StateType::getBestState(AAS); |
394 | *T &= AAS; |
395 | LLVM_DEBUG(dbgs() << "[Attributor] AA State: " << AAS << " RV State: " << T |
396 | << "\n" ); |
397 | return T->isValidState(); |
398 | }; |
399 | |
400 | if (!A.checkForAllReturnedValues(Pred: CheckReturnValue, QueryingAA, |
401 | S: AA::ValueScope::Intraprocedural, |
402 | RecurseForSelectAndPHI)) |
403 | S.indicatePessimisticFixpoint(); |
404 | else if (T) |
405 | S ^= *T; |
406 | } |
407 | |
408 | namespace { |
409 | /// Helper class for generic deduction: return value -> returned position. |
410 | template <typename AAType, typename BaseType, |
411 | typename StateType = typename BaseType::StateType, |
412 | bool PropagateCallBaseContext = false, |
413 | Attribute::AttrKind IRAttributeKind = AAType::IRAttributeKind, |
414 | bool RecurseForSelectAndPHI = true> |
415 | struct AAReturnedFromReturnedValues : public BaseType { |
416 | AAReturnedFromReturnedValues(const IRPosition &IRP, Attributor &A) |
417 | : BaseType(IRP, A) {} |
418 | |
419 | /// See AbstractAttribute::updateImpl(...). |
420 | ChangeStatus updateImpl(Attributor &A) override { |
421 | StateType S(StateType::getBestState(this->getState())); |
422 | clampReturnedValueStates<AAType, StateType, IRAttributeKind, RecurseForSelectAndPHI>( |
423 | A, *this, S, |
424 | PropagateCallBaseContext ? this->getCallBaseContext() : nullptr); |
425 | // TODO: If we know we visited all returned values, thus no are assumed |
426 | // dead, we can take the known information from the state T. |
427 | return clampStateAndIndicateChange<StateType>(this->getState(), S); |
428 | } |
429 | }; |
430 | |
431 | /// Clamp the information known at all call sites for a given argument |
432 | /// (identified by \p QueryingAA) into \p S. |
433 | template <typename AAType, typename StateType = typename AAType::StateType, |
434 | Attribute::AttrKind IRAttributeKind = AAType::IRAttributeKind> |
435 | static void clampCallSiteArgumentStates(Attributor &A, const AAType &QueryingAA, |
436 | StateType &S) { |
437 | LLVM_DEBUG(dbgs() << "[Attributor] Clamp call site argument states for " |
438 | << QueryingAA << " into " << S << "\n" ); |
439 | |
440 | assert(QueryingAA.getIRPosition().getPositionKind() == |
441 | IRPosition::IRP_ARGUMENT && |
442 | "Can only clamp call site argument states for an argument position!" ); |
443 | |
444 | // Use an optional state as there might not be any return values and we want |
445 | // to join (IntegerState::operator&) the state of all there are. |
446 | std::optional<StateType> T; |
447 | |
448 | // The argument number which is also the call site argument number. |
449 | unsigned ArgNo = QueryingAA.getIRPosition().getCallSiteArgNo(); |
450 | |
451 | auto CallSiteCheck = [&](AbstractCallSite ACS) { |
452 | const IRPosition &ACSArgPos = IRPosition::callsite_argument(ACS, ArgNo); |
453 | // Check if a coresponding argument was found or if it is on not associated |
454 | // (which can happen for callback calls). |
455 | if (ACSArgPos.getPositionKind() == IRPosition::IRP_INVALID) |
456 | return false; |
457 | |
458 | // If possible, use the hasAssumedIRAttr interface. |
459 | if (Attribute::isEnumAttrKind(Kind: IRAttributeKind)) { |
460 | bool IsKnown; |
461 | return AA::hasAssumedIRAttr<IRAttributeKind>( |
462 | A, &QueryingAA, ACSArgPos, DepClassTy::REQUIRED, IsKnown); |
463 | } |
464 | |
465 | const AAType *AA = |
466 | A.getAAFor<AAType>(QueryingAA, ACSArgPos, DepClassTy::REQUIRED); |
467 | if (!AA) |
468 | return false; |
469 | LLVM_DEBUG(dbgs() << "[Attributor] ACS: " << *ACS.getInstruction() |
470 | << " AA: " << AA->getAsStr(&A) << " @" << ACSArgPos |
471 | << "\n" ); |
472 | const StateType &AAS = AA->getState(); |
473 | if (!T) |
474 | T = StateType::getBestState(AAS); |
475 | *T &= AAS; |
476 | LLVM_DEBUG(dbgs() << "[Attributor] AA State: " << AAS << " CSA State: " << T |
477 | << "\n" ); |
478 | return T->isValidState(); |
479 | }; |
480 | |
481 | bool UsedAssumedInformation = false; |
482 | if (!A.checkForAllCallSites(CallSiteCheck, QueryingAA, true, |
483 | UsedAssumedInformation)) |
484 | S.indicatePessimisticFixpoint(); |
485 | else if (T) |
486 | S ^= *T; |
487 | } |
488 | |
489 | /// This function is the bridge between argument position and the call base |
490 | /// context. |
491 | template <typename AAType, typename BaseType, |
492 | typename StateType = typename AAType::StateType, |
493 | Attribute::AttrKind IRAttributeKind = AAType::IRAttributeKind> |
494 | bool getArgumentStateFromCallBaseContext(Attributor &A, |
495 | BaseType &QueryingAttribute, |
496 | IRPosition &Pos, StateType &State) { |
497 | assert((Pos.getPositionKind() == IRPosition::IRP_ARGUMENT) && |
498 | "Expected an 'argument' position !" ); |
499 | const CallBase *CBContext = Pos.getCallBaseContext(); |
500 | if (!CBContext) |
501 | return false; |
502 | |
503 | int ArgNo = Pos.getCallSiteArgNo(); |
504 | assert(ArgNo >= 0 && "Invalid Arg No!" ); |
505 | const IRPosition CBArgPos = IRPosition::callsite_argument(CB: *CBContext, ArgNo); |
506 | |
507 | // If possible, use the hasAssumedIRAttr interface. |
508 | if (Attribute::isEnumAttrKind(Kind: IRAttributeKind)) { |
509 | bool IsKnown; |
510 | return AA::hasAssumedIRAttr<IRAttributeKind>( |
511 | A, &QueryingAttribute, CBArgPos, DepClassTy::REQUIRED, IsKnown); |
512 | } |
513 | |
514 | const auto *AA = |
515 | A.getAAFor<AAType>(QueryingAttribute, CBArgPos, DepClassTy::REQUIRED); |
516 | if (!AA) |
517 | return false; |
518 | const StateType &CBArgumentState = |
519 | static_cast<const StateType &>(AA->getState()); |
520 | |
521 | LLVM_DEBUG(dbgs() << "[Attributor] Briding Call site context to argument" |
522 | << "Position:" << Pos << "CB Arg state:" << CBArgumentState |
523 | << "\n" ); |
524 | |
525 | // NOTE: If we want to do call site grouping it should happen here. |
526 | State ^= CBArgumentState; |
527 | return true; |
528 | } |
529 | |
530 | /// Helper class for generic deduction: call site argument -> argument position. |
531 | template <typename AAType, typename BaseType, |
532 | typename StateType = typename AAType::StateType, |
533 | bool BridgeCallBaseContext = false, |
534 | Attribute::AttrKind IRAttributeKind = AAType::IRAttributeKind> |
535 | struct AAArgumentFromCallSiteArguments : public BaseType { |
536 | AAArgumentFromCallSiteArguments(const IRPosition &IRP, Attributor &A) |
537 | : BaseType(IRP, A) {} |
538 | |
539 | /// See AbstractAttribute::updateImpl(...). |
540 | ChangeStatus updateImpl(Attributor &A) override { |
541 | StateType S = StateType::getBestState(this->getState()); |
542 | |
543 | if (BridgeCallBaseContext) { |
544 | bool Success = |
545 | getArgumentStateFromCallBaseContext<AAType, BaseType, StateType, |
546 | IRAttributeKind>( |
547 | A, *this, this->getIRPosition(), S); |
548 | if (Success) |
549 | return clampStateAndIndicateChange<StateType>(this->getState(), S); |
550 | } |
551 | clampCallSiteArgumentStates<AAType, StateType, IRAttributeKind>(A, *this, |
552 | S); |
553 | |
554 | // TODO: If we know we visited all incoming values, thus no are assumed |
555 | // dead, we can take the known information from the state T. |
556 | return clampStateAndIndicateChange<StateType>(this->getState(), S); |
557 | } |
558 | }; |
559 | |
560 | /// Helper class for generic replication: function returned -> cs returned. |
561 | template <typename AAType, typename BaseType, |
562 | typename StateType = typename BaseType::StateType, |
563 | bool IntroduceCallBaseContext = false, |
564 | Attribute::AttrKind IRAttributeKind = AAType::IRAttributeKind> |
565 | struct AACalleeToCallSite : public BaseType { |
566 | AACalleeToCallSite(const IRPosition &IRP, Attributor &A) : BaseType(IRP, A) {} |
567 | |
568 | /// See AbstractAttribute::updateImpl(...). |
569 | ChangeStatus updateImpl(Attributor &A) override { |
570 | auto IRPKind = this->getIRPosition().getPositionKind(); |
571 | assert((IRPKind == IRPosition::IRP_CALL_SITE_RETURNED || |
572 | IRPKind == IRPosition::IRP_CALL_SITE) && |
573 | "Can only wrap function returned positions for call site " |
574 | "returned positions!" ); |
575 | auto &S = this->getState(); |
576 | |
577 | CallBase &CB = cast<CallBase>(this->getAnchorValue()); |
578 | if (IntroduceCallBaseContext) |
579 | LLVM_DEBUG(dbgs() << "[Attributor] Introducing call base context:" << CB |
580 | << "\n" ); |
581 | |
582 | ChangeStatus Changed = ChangeStatus::UNCHANGED; |
583 | auto CalleePred = [&](ArrayRef<const Function *> Callees) { |
584 | for (const Function *Callee : Callees) { |
585 | IRPosition FnPos = |
586 | IRPKind == llvm::IRPosition::IRP_CALL_SITE_RETURNED |
587 | ? IRPosition::returned(F: *Callee, |
588 | CBContext: IntroduceCallBaseContext ? &CB : nullptr) |
589 | : IRPosition::function( |
590 | F: *Callee, CBContext: IntroduceCallBaseContext ? &CB : nullptr); |
591 | // If possible, use the hasAssumedIRAttr interface. |
592 | if (Attribute::isEnumAttrKind(Kind: IRAttributeKind)) { |
593 | bool IsKnown; |
594 | if (!AA::hasAssumedIRAttr<IRAttributeKind>( |
595 | A, this, FnPos, DepClassTy::REQUIRED, IsKnown)) |
596 | return false; |
597 | continue; |
598 | } |
599 | |
600 | const AAType *AA = |
601 | A.getAAFor<AAType>(*this, FnPos, DepClassTy::REQUIRED); |
602 | if (!AA) |
603 | return false; |
604 | Changed |= clampStateAndIndicateChange(S, AA->getState()); |
605 | if (S.isAtFixpoint()) |
606 | return S.isValidState(); |
607 | } |
608 | return true; |
609 | }; |
610 | if (!A.checkForAllCallees(Pred: CalleePred, QueryingAA: *this, CB)) |
611 | return S.indicatePessimisticFixpoint(); |
612 | return Changed; |
613 | } |
614 | }; |
615 | |
616 | /// Helper function to accumulate uses. |
617 | template <class AAType, typename StateType = typename AAType::StateType> |
618 | static void followUsesInContext(AAType &AA, Attributor &A, |
619 | MustBeExecutedContextExplorer &Explorer, |
620 | const Instruction *CtxI, |
621 | SetVector<const Use *> &Uses, |
622 | StateType &State) { |
623 | auto EIt = Explorer.begin(PP: CtxI), EEnd = Explorer.end(CtxI); |
624 | for (unsigned u = 0; u < Uses.size(); ++u) { |
625 | const Use *U = Uses[u]; |
626 | if (const Instruction *UserI = dyn_cast<Instruction>(Val: U->getUser())) { |
627 | bool Found = Explorer.findInContextOf(I: UserI, EIt, EEnd); |
628 | if (Found && AA.followUseInMBEC(A, U, UserI, State)) |
629 | for (const Use &Us : UserI->uses()) |
630 | Uses.insert(X: &Us); |
631 | } |
632 | } |
633 | } |
634 | |
635 | /// Use the must-be-executed-context around \p I to add information into \p S. |
636 | /// The AAType class is required to have `followUseInMBEC` method with the |
637 | /// following signature and behaviour: |
638 | /// |
639 | /// bool followUseInMBEC(Attributor &A, const Use *U, const Instruction *I) |
640 | /// U - Underlying use. |
641 | /// I - The user of the \p U. |
642 | /// Returns true if the value should be tracked transitively. |
643 | /// |
644 | template <class AAType, typename StateType = typename AAType::StateType> |
645 | static void followUsesInMBEC(AAType &AA, Attributor &A, StateType &S, |
646 | Instruction &CtxI) { |
647 | MustBeExecutedContextExplorer *Explorer = |
648 | A.getInfoCache().getMustBeExecutedContextExplorer(); |
649 | if (!Explorer) |
650 | return; |
651 | |
652 | // Container for (transitive) uses of the associated value. |
653 | SetVector<const Use *> Uses; |
654 | for (const Use &U : AA.getIRPosition().getAssociatedValue().uses()) |
655 | Uses.insert(X: &U); |
656 | |
657 | followUsesInContext<AAType>(AA, A, *Explorer, &CtxI, Uses, S); |
658 | |
659 | if (S.isAtFixpoint()) |
660 | return; |
661 | |
662 | SmallVector<const BranchInst *, 4> BrInsts; |
663 | auto Pred = [&](const Instruction *I) { |
664 | if (const BranchInst *Br = dyn_cast<BranchInst>(Val: I)) |
665 | if (Br->isConditional()) |
666 | BrInsts.push_back(Elt: Br); |
667 | return true; |
668 | }; |
669 | |
670 | // Here, accumulate conditional branch instructions in the context. We |
671 | // explore the child paths and collect the known states. The disjunction of |
672 | // those states can be merged to its own state. Let ParentState_i be a state |
673 | // to indicate the known information for an i-th branch instruction in the |
674 | // context. ChildStates are created for its successors respectively. |
675 | // |
676 | // ParentS_1 = ChildS_{1, 1} /\ ChildS_{1, 2} /\ ... /\ ChildS_{1, n_1} |
677 | // ParentS_2 = ChildS_{2, 1} /\ ChildS_{2, 2} /\ ... /\ ChildS_{2, n_2} |
678 | // ... |
679 | // ParentS_m = ChildS_{m, 1} /\ ChildS_{m, 2} /\ ... /\ ChildS_{m, n_m} |
680 | // |
681 | // Known State |= ParentS_1 \/ ParentS_2 \/... \/ ParentS_m |
682 | // |
683 | // FIXME: Currently, recursive branches are not handled. For example, we |
684 | // can't deduce that ptr must be dereferenced in below function. |
685 | // |
686 | // void f(int a, int c, int *ptr) { |
687 | // if(a) |
688 | // if (b) { |
689 | // *ptr = 0; |
690 | // } else { |
691 | // *ptr = 1; |
692 | // } |
693 | // else { |
694 | // if (b) { |
695 | // *ptr = 0; |
696 | // } else { |
697 | // *ptr = 1; |
698 | // } |
699 | // } |
700 | // } |
701 | |
702 | Explorer->checkForAllContext(PP: &CtxI, Pred); |
703 | for (const BranchInst *Br : BrInsts) { |
704 | StateType ParentState; |
705 | |
706 | // The known state of the parent state is a conjunction of children's |
707 | // known states so it is initialized with a best state. |
708 | ParentState.indicateOptimisticFixpoint(); |
709 | |
710 | for (const BasicBlock *BB : Br->successors()) { |
711 | StateType ChildState; |
712 | |
713 | size_t BeforeSize = Uses.size(); |
714 | followUsesInContext(AA, A, *Explorer, &BB->front(), Uses, ChildState); |
715 | |
716 | // Erase uses which only appear in the child. |
717 | for (auto It = Uses.begin() + BeforeSize; It != Uses.end();) |
718 | It = Uses.erase(I: It); |
719 | |
720 | ParentState &= ChildState; |
721 | } |
722 | |
723 | // Use only known state. |
724 | S += ParentState; |
725 | } |
726 | } |
727 | } // namespace |
728 | |
729 | /// ------------------------ PointerInfo --------------------------------------- |
730 | |
731 | namespace llvm { |
732 | namespace AA { |
733 | namespace PointerInfo { |
734 | |
735 | struct State; |
736 | |
737 | } // namespace PointerInfo |
738 | } // namespace AA |
739 | |
740 | /// Helper for AA::PointerInfo::Access DenseMap/Set usage. |
741 | template <> |
742 | struct DenseMapInfo<AAPointerInfo::Access> : DenseMapInfo<Instruction *> { |
743 | using Access = AAPointerInfo::Access; |
744 | static inline Access getEmptyKey(); |
745 | static inline Access getTombstoneKey(); |
746 | static unsigned getHashValue(const Access &A); |
747 | static bool isEqual(const Access &LHS, const Access &RHS); |
748 | }; |
749 | |
750 | /// Helper that allows RangeTy as a key in a DenseMap. |
751 | template <> struct DenseMapInfo<AA::RangeTy> { |
752 | static inline AA::RangeTy getEmptyKey() { |
753 | auto EmptyKey = DenseMapInfo<int64_t>::getEmptyKey(); |
754 | return AA::RangeTy{EmptyKey, EmptyKey}; |
755 | } |
756 | |
757 | static inline AA::RangeTy getTombstoneKey() { |
758 | auto TombstoneKey = DenseMapInfo<int64_t>::getTombstoneKey(); |
759 | return AA::RangeTy{TombstoneKey, TombstoneKey}; |
760 | } |
761 | |
762 | static unsigned getHashValue(const AA::RangeTy &Range) { |
763 | return detail::combineHashValue( |
764 | a: DenseMapInfo<int64_t>::getHashValue(Val: Range.Offset), |
765 | b: DenseMapInfo<int64_t>::getHashValue(Val: Range.Size)); |
766 | } |
767 | |
768 | static bool isEqual(const AA::RangeTy &A, const AA::RangeTy B) { |
769 | return A == B; |
770 | } |
771 | }; |
772 | |
773 | /// Helper for AA::PointerInfo::Access DenseMap/Set usage ignoring everythign |
774 | /// but the instruction |
775 | struct AccessAsInstructionInfo : DenseMapInfo<Instruction *> { |
776 | using Base = DenseMapInfo<Instruction *>; |
777 | using Access = AAPointerInfo::Access; |
778 | static inline Access getEmptyKey(); |
779 | static inline Access getTombstoneKey(); |
780 | static unsigned getHashValue(const Access &A); |
781 | static bool isEqual(const Access &LHS, const Access &RHS); |
782 | }; |
783 | |
784 | } // namespace llvm |
785 | |
786 | /// A type to track pointer/struct usage and accesses for AAPointerInfo. |
787 | struct AA::PointerInfo::State : public AbstractState { |
788 | /// Return the best possible representable state. |
789 | static State getBestState(const State &SIS) { return State(); } |
790 | |
791 | /// Return the worst possible representable state. |
792 | static State getWorstState(const State &SIS) { |
793 | State R; |
794 | R.indicatePessimisticFixpoint(); |
795 | return R; |
796 | } |
797 | |
798 | State() = default; |
799 | State(State &&SIS) = default; |
800 | |
801 | const State &getAssumed() const { return *this; } |
802 | |
803 | /// See AbstractState::isValidState(). |
804 | bool isValidState() const override { return BS.isValidState(); } |
805 | |
806 | /// See AbstractState::isAtFixpoint(). |
807 | bool isAtFixpoint() const override { return BS.isAtFixpoint(); } |
808 | |
809 | /// See AbstractState::indicateOptimisticFixpoint(). |
810 | ChangeStatus indicateOptimisticFixpoint() override { |
811 | BS.indicateOptimisticFixpoint(); |
812 | return ChangeStatus::UNCHANGED; |
813 | } |
814 | |
815 | /// See AbstractState::indicatePessimisticFixpoint(). |
816 | ChangeStatus indicatePessimisticFixpoint() override { |
817 | BS.indicatePessimisticFixpoint(); |
818 | return ChangeStatus::CHANGED; |
819 | } |
820 | |
821 | State &operator=(const State &R) { |
822 | if (this == &R) |
823 | return *this; |
824 | BS = R.BS; |
825 | AccessList = R.AccessList; |
826 | OffsetBins = R.OffsetBins; |
827 | RemoteIMap = R.RemoteIMap; |
828 | return *this; |
829 | } |
830 | |
831 | State &operator=(State &&R) { |
832 | if (this == &R) |
833 | return *this; |
834 | std::swap(a&: BS, b&: R.BS); |
835 | std::swap(LHS&: AccessList, RHS&: R.AccessList); |
836 | std::swap(a&: OffsetBins, b&: R.OffsetBins); |
837 | std::swap(a&: RemoteIMap, b&: R.RemoteIMap); |
838 | return *this; |
839 | } |
840 | |
841 | /// Add a new Access to the state at offset \p Offset and with size \p Size. |
842 | /// The access is associated with \p I, writes \p Content (if anything), and |
843 | /// is of kind \p Kind. If an Access already exists for the same \p I and same |
844 | /// \p RemoteI, the two are combined, potentially losing information about |
845 | /// offset and size. The resulting access must now be moved from its original |
846 | /// OffsetBin to the bin for its new offset. |
847 | /// |
848 | /// \Returns CHANGED, if the state changed, UNCHANGED otherwise. |
849 | ChangeStatus addAccess(Attributor &A, const AAPointerInfo::RangeList &Ranges, |
850 | Instruction &I, std::optional<Value *> Content, |
851 | AAPointerInfo::AccessKind Kind, Type *Ty, |
852 | Instruction *RemoteI = nullptr); |
853 | |
854 | AAPointerInfo::const_bin_iterator begin() const { return OffsetBins.begin(); } |
855 | AAPointerInfo::const_bin_iterator end() const { return OffsetBins.end(); } |
856 | int64_t numOffsetBins() const { return OffsetBins.size(); } |
857 | |
858 | const AAPointerInfo::Access &getAccess(unsigned Index) const { |
859 | return AccessList[Index]; |
860 | } |
861 | |
862 | protected: |
863 | // Every memory instruction results in an Access object. We maintain a list of |
864 | // all Access objects that we own, along with the following maps: |
865 | // |
866 | // - OffsetBins: RangeTy -> { Access } |
867 | // - RemoteIMap: RemoteI x LocalI -> Access |
868 | // |
869 | // A RemoteI is any instruction that accesses memory. RemoteI is different |
870 | // from LocalI if and only if LocalI is a call; then RemoteI is some |
871 | // instruction in the callgraph starting from LocalI. Multiple paths in the |
872 | // callgraph from LocalI to RemoteI may produce multiple accesses, but these |
873 | // are all combined into a single Access object. This may result in loss of |
874 | // information in RangeTy in the Access object. |
875 | SmallVector<AAPointerInfo::Access> AccessList; |
876 | AAPointerInfo::OffsetBinsTy OffsetBins; |
877 | DenseMap<const Instruction *, SmallVector<unsigned>> RemoteIMap; |
878 | |
879 | /// See AAPointerInfo::forallInterferingAccesses. |
880 | bool forallInterferingAccesses( |
881 | AA::RangeTy Range, |
882 | function_ref<bool(const AAPointerInfo::Access &, bool)> CB) const { |
883 | if (!isValidState()) |
884 | return false; |
885 | |
886 | for (const auto &It : OffsetBins) { |
887 | AA::RangeTy ItRange = It.getFirst(); |
888 | if (!Range.mayOverlap(Range: ItRange)) |
889 | continue; |
890 | bool IsExact = Range == ItRange && !Range.offsetOrSizeAreUnknown(); |
891 | for (auto Index : It.getSecond()) { |
892 | auto &Access = AccessList[Index]; |
893 | if (!CB(Access, IsExact)) |
894 | return false; |
895 | } |
896 | } |
897 | return true; |
898 | } |
899 | |
900 | /// See AAPointerInfo::forallInterferingAccesses. |
901 | bool forallInterferingAccesses( |
902 | Instruction &I, |
903 | function_ref<bool(const AAPointerInfo::Access &, bool)> CB, |
904 | AA::RangeTy &Range) const { |
905 | if (!isValidState()) |
906 | return false; |
907 | |
908 | auto LocalList = RemoteIMap.find(Val: &I); |
909 | if (LocalList == RemoteIMap.end()) { |
910 | return true; |
911 | } |
912 | |
913 | for (unsigned Index : LocalList->getSecond()) { |
914 | for (auto &R : AccessList[Index]) { |
915 | Range &= R; |
916 | if (Range.offsetAndSizeAreUnknown()) |
917 | break; |
918 | } |
919 | } |
920 | return forallInterferingAccesses(Range, CB); |
921 | } |
922 | |
923 | private: |
924 | /// State to track fixpoint and validity. |
925 | BooleanState BS; |
926 | }; |
927 | |
928 | ChangeStatus AA::PointerInfo::State::addAccess( |
929 | Attributor &A, const AAPointerInfo::RangeList &Ranges, Instruction &I, |
930 | std::optional<Value *> Content, AAPointerInfo::AccessKind Kind, Type *Ty, |
931 | Instruction *RemoteI) { |
932 | RemoteI = RemoteI ? RemoteI : &I; |
933 | |
934 | // Check if we have an access for this instruction, if not, simply add it. |
935 | auto &LocalList = RemoteIMap[RemoteI]; |
936 | bool AccExists = false; |
937 | unsigned AccIndex = AccessList.size(); |
938 | for (auto Index : LocalList) { |
939 | auto &A = AccessList[Index]; |
940 | if (A.getLocalInst() == &I) { |
941 | AccExists = true; |
942 | AccIndex = Index; |
943 | break; |
944 | } |
945 | } |
946 | |
947 | auto AddToBins = [&](const AAPointerInfo::RangeList &ToAdd) { |
948 | LLVM_DEBUG(if (ToAdd.size()) dbgs() |
949 | << "[AAPointerInfo] Inserting access in new offset bins\n" ;); |
950 | |
951 | for (auto Key : ToAdd) { |
952 | LLVM_DEBUG(dbgs() << " key " << Key << "\n" ); |
953 | OffsetBins[Key].insert(V: AccIndex); |
954 | } |
955 | }; |
956 | |
957 | if (!AccExists) { |
958 | AccessList.emplace_back(Args: &I, Args&: RemoteI, Args: Ranges, Args&: Content, Args&: Kind, Args&: Ty); |
959 | assert((AccessList.size() == AccIndex + 1) && |
960 | "New Access should have been at AccIndex" ); |
961 | LocalList.push_back(Elt: AccIndex); |
962 | AddToBins(AccessList[AccIndex].getRanges()); |
963 | return ChangeStatus::CHANGED; |
964 | } |
965 | |
966 | // Combine the new Access with the existing Access, and then update the |
967 | // mapping in the offset bins. |
968 | AAPointerInfo::Access Acc(&I, RemoteI, Ranges, Content, Kind, Ty); |
969 | auto &Current = AccessList[AccIndex]; |
970 | auto Before = Current; |
971 | Current &= Acc; |
972 | if (Current == Before) |
973 | return ChangeStatus::UNCHANGED; |
974 | |
975 | auto &ExistingRanges = Before.getRanges(); |
976 | auto &NewRanges = Current.getRanges(); |
977 | |
978 | // Ranges that are in the old access but not the new access need to be removed |
979 | // from the offset bins. |
980 | AAPointerInfo::RangeList ToRemove; |
981 | AAPointerInfo::RangeList::set_difference(L: ExistingRanges, R: NewRanges, D&: ToRemove); |
982 | LLVM_DEBUG(if (ToRemove.size()) dbgs() |
983 | << "[AAPointerInfo] Removing access from old offset bins\n" ;); |
984 | |
985 | for (auto Key : ToRemove) { |
986 | LLVM_DEBUG(dbgs() << " key " << Key << "\n" ); |
987 | assert(OffsetBins.count(Key) && "Existing Access must be in some bin." ); |
988 | auto &Bin = OffsetBins[Key]; |
989 | assert(Bin.count(AccIndex) && |
990 | "Expected bin to actually contain the Access." ); |
991 | Bin.erase(V: AccIndex); |
992 | } |
993 | |
994 | // Ranges that are in the new access but not the old access need to be added |
995 | // to the offset bins. |
996 | AAPointerInfo::RangeList ToAdd; |
997 | AAPointerInfo::RangeList::set_difference(L: NewRanges, R: ExistingRanges, D&: ToAdd); |
998 | AddToBins(ToAdd); |
999 | return ChangeStatus::CHANGED; |
1000 | } |
1001 | |
1002 | namespace { |
1003 | |
1004 | /// A helper containing a list of offsets computed for a Use. Ideally this |
1005 | /// list should be strictly ascending, but we ensure that only when we |
1006 | /// actually translate the list of offsets to a RangeList. |
1007 | struct OffsetInfo { |
1008 | using VecTy = SmallVector<int64_t>; |
1009 | using const_iterator = VecTy::const_iterator; |
1010 | VecTy Offsets; |
1011 | |
1012 | const_iterator begin() const { return Offsets.begin(); } |
1013 | const_iterator end() const { return Offsets.end(); } |
1014 | |
1015 | bool operator==(const OffsetInfo &RHS) const { |
1016 | return Offsets == RHS.Offsets; |
1017 | } |
1018 | |
1019 | bool operator!=(const OffsetInfo &RHS) const { return !(*this == RHS); } |
1020 | |
1021 | void insert(int64_t Offset) { Offsets.push_back(Elt: Offset); } |
1022 | bool isUnassigned() const { return Offsets.size() == 0; } |
1023 | |
1024 | bool isUnknown() const { |
1025 | if (isUnassigned()) |
1026 | return false; |
1027 | if (Offsets.size() == 1) |
1028 | return Offsets.front() == AA::RangeTy::Unknown; |
1029 | return false; |
1030 | } |
1031 | |
1032 | void setUnknown() { |
1033 | Offsets.clear(); |
1034 | Offsets.push_back(Elt: AA::RangeTy::Unknown); |
1035 | } |
1036 | |
1037 | void addToAll(int64_t Inc) { |
1038 | for (auto &Offset : Offsets) { |
1039 | Offset += Inc; |
1040 | } |
1041 | } |
1042 | |
1043 | /// Copy offsets from \p R into the current list. |
1044 | /// |
1045 | /// Ideally all lists should be strictly ascending, but we defer that to the |
1046 | /// actual use of the list. So we just blindly append here. |
1047 | void merge(const OffsetInfo &R) { Offsets.append(RHS: R.Offsets); } |
1048 | }; |
1049 | |
1050 | #ifndef NDEBUG |
1051 | static raw_ostream &operator<<(raw_ostream &OS, const OffsetInfo &OI) { |
1052 | ListSeparator LS; |
1053 | OS << "[" ; |
1054 | for (auto Offset : OI) { |
1055 | OS << LS << Offset; |
1056 | } |
1057 | OS << "]" ; |
1058 | return OS; |
1059 | } |
1060 | #endif // NDEBUG |
1061 | |
1062 | struct AAPointerInfoImpl |
1063 | : public StateWrapper<AA::PointerInfo::State, AAPointerInfo> { |
1064 | using BaseTy = StateWrapper<AA::PointerInfo::State, AAPointerInfo>; |
1065 | AAPointerInfoImpl(const IRPosition &IRP, Attributor &A) : BaseTy(IRP) {} |
1066 | |
1067 | /// See AbstractAttribute::getAsStr(). |
1068 | const std::string getAsStr(Attributor *A) const override { |
1069 | return std::string("PointerInfo " ) + |
1070 | (isValidState() ? (std::string("#" ) + |
1071 | std::to_string(val: OffsetBins.size()) + " bins" ) |
1072 | : "<invalid>" ); |
1073 | } |
1074 | |
1075 | /// See AbstractAttribute::manifest(...). |
1076 | ChangeStatus manifest(Attributor &A) override { |
1077 | return AAPointerInfo::manifest(A); |
1078 | } |
1079 | |
1080 | virtual const_bin_iterator begin() const override { return State::begin(); } |
1081 | virtual const_bin_iterator end() const override { return State::end(); } |
1082 | virtual int64_t numOffsetBins() const override { |
1083 | return State::numOffsetBins(); |
1084 | } |
1085 | |
1086 | bool forallInterferingAccesses( |
1087 | AA::RangeTy Range, |
1088 | function_ref<bool(const AAPointerInfo::Access &, bool)> CB) |
1089 | const override { |
1090 | return State::forallInterferingAccesses(Range, CB); |
1091 | } |
1092 | |
1093 | bool forallInterferingAccesses( |
1094 | Attributor &A, const AbstractAttribute &QueryingAA, Instruction &I, |
1095 | bool FindInterferingWrites, bool FindInterferingReads, |
1096 | function_ref<bool(const Access &, bool)> UserCB, bool &HasBeenWrittenTo, |
1097 | AA::RangeTy &Range, |
1098 | function_ref<bool(const Access &)> SkipCB) const override { |
1099 | HasBeenWrittenTo = false; |
1100 | |
1101 | SmallPtrSet<const Access *, 8> DominatingWrites; |
1102 | SmallVector<std::pair<const Access *, bool>, 8> InterferingAccesses; |
1103 | |
1104 | Function &Scope = *I.getFunction(); |
1105 | bool IsKnownNoSync; |
1106 | bool IsAssumedNoSync = AA::hasAssumedIRAttr<Attribute::NoSync>( |
1107 | A, &QueryingAA, IRPosition::function(Scope), DepClassTy::OPTIONAL, |
1108 | IsKnownNoSync); |
1109 | const auto *ExecDomainAA = A.lookupAAFor<AAExecutionDomain>( |
1110 | IRP: IRPosition::function(F: Scope), QueryingAA: &QueryingAA, DepClass: DepClassTy::NONE); |
1111 | bool AllInSameNoSyncFn = IsAssumedNoSync; |
1112 | bool InstIsExecutedByInitialThreadOnly = |
1113 | ExecDomainAA && ExecDomainAA->isExecutedByInitialThreadOnly(I); |
1114 | |
1115 | // If the function is not ending in aligned barriers, we need the stores to |
1116 | // be in aligned barriers. The load being in one is not sufficient since the |
1117 | // store might be executed by a thread that disappears after, causing the |
1118 | // aligned barrier guarding the load to unblock and the load to read a value |
1119 | // that has no CFG path to the load. |
1120 | bool InstIsExecutedInAlignedRegion = |
1121 | FindInterferingReads && ExecDomainAA && |
1122 | ExecDomainAA->isExecutedInAlignedRegion(A, I); |
1123 | |
1124 | if (InstIsExecutedInAlignedRegion || InstIsExecutedByInitialThreadOnly) |
1125 | A.recordDependence(FromAA: *ExecDomainAA, ToAA: QueryingAA, DepClass: DepClassTy::OPTIONAL); |
1126 | |
1127 | InformationCache &InfoCache = A.getInfoCache(); |
1128 | bool IsThreadLocalObj = |
1129 | AA::isAssumedThreadLocalObject(A, Obj&: getAssociatedValue(), QueryingAA: *this); |
1130 | |
1131 | // Helper to determine if we need to consider threading, which we cannot |
1132 | // right now. However, if the function is (assumed) nosync or the thread |
1133 | // executing all instructions is the main thread only we can ignore |
1134 | // threading. Also, thread-local objects do not require threading reasoning. |
1135 | // Finally, we can ignore threading if either access is executed in an |
1136 | // aligned region. |
1137 | auto CanIgnoreThreadingForInst = [&](const Instruction &I) -> bool { |
1138 | if (IsThreadLocalObj || AllInSameNoSyncFn) |
1139 | return true; |
1140 | const auto *FnExecDomainAA = |
1141 | I.getFunction() == &Scope |
1142 | ? ExecDomainAA |
1143 | : A.lookupAAFor<AAExecutionDomain>( |
1144 | IRP: IRPosition::function(F: *I.getFunction()), QueryingAA: &QueryingAA, |
1145 | DepClass: DepClassTy::NONE); |
1146 | if (!FnExecDomainAA) |
1147 | return false; |
1148 | if (InstIsExecutedInAlignedRegion || |
1149 | (FindInterferingWrites && |
1150 | FnExecDomainAA->isExecutedInAlignedRegion(A, I))) { |
1151 | A.recordDependence(FromAA: *FnExecDomainAA, ToAA: QueryingAA, DepClass: DepClassTy::OPTIONAL); |
1152 | return true; |
1153 | } |
1154 | if (InstIsExecutedByInitialThreadOnly && |
1155 | FnExecDomainAA->isExecutedByInitialThreadOnly(I)) { |
1156 | A.recordDependence(FromAA: *FnExecDomainAA, ToAA: QueryingAA, DepClass: DepClassTy::OPTIONAL); |
1157 | return true; |
1158 | } |
1159 | return false; |
1160 | }; |
1161 | |
1162 | // Helper to determine if the access is executed by the same thread as the |
1163 | // given instruction, for now it is sufficient to avoid any potential |
1164 | // threading effects as we cannot deal with them anyway. |
1165 | auto CanIgnoreThreading = [&](const Access &Acc) -> bool { |
1166 | return CanIgnoreThreadingForInst(*Acc.getRemoteInst()) || |
1167 | (Acc.getRemoteInst() != Acc.getLocalInst() && |
1168 | CanIgnoreThreadingForInst(*Acc.getLocalInst())); |
1169 | }; |
1170 | |
1171 | // TODO: Use inter-procedural reachability and dominance. |
1172 | bool IsKnownNoRecurse; |
1173 | AA::hasAssumedIRAttr<Attribute::NoRecurse>( |
1174 | A, this, IRPosition::function(Scope), DepClassTy::OPTIONAL, |
1175 | IsKnownNoRecurse); |
1176 | |
1177 | // TODO: Use reaching kernels from AAKernelInfo (or move it to |
1178 | // AAExecutionDomain) such that we allow scopes other than kernels as long |
1179 | // as the reaching kernels are disjoint. |
1180 | bool InstInKernel = Scope.hasFnAttribute(Kind: "kernel" ); |
1181 | bool ObjHasKernelLifetime = false; |
1182 | const bool UseDominanceReasoning = |
1183 | FindInterferingWrites && IsKnownNoRecurse; |
1184 | const DominatorTree *DT = |
1185 | InfoCache.getAnalysisResultForFunction<DominatorTreeAnalysis>(F: Scope); |
1186 | |
1187 | // Helper to check if a value has "kernel lifetime", that is it will not |
1188 | // outlive a GPU kernel. This is true for shared, constant, and local |
1189 | // globals on AMD and NVIDIA GPUs. |
1190 | auto HasKernelLifetime = [&](Value *V, Module &M) { |
1191 | if (!AA::isGPU(M)) |
1192 | return false; |
1193 | switch (AA::GPUAddressSpace(V->getType()->getPointerAddressSpace())) { |
1194 | case AA::GPUAddressSpace::Shared: |
1195 | case AA::GPUAddressSpace::Constant: |
1196 | case AA::GPUAddressSpace::Local: |
1197 | return true; |
1198 | default: |
1199 | return false; |
1200 | }; |
1201 | }; |
1202 | |
1203 | // The IsLiveInCalleeCB will be used by the AA::isPotentiallyReachable query |
1204 | // to determine if we should look at reachability from the callee. For |
1205 | // certain pointers we know the lifetime and we do not have to step into the |
1206 | // callee to determine reachability as the pointer would be dead in the |
1207 | // callee. See the conditional initialization below. |
1208 | std::function<bool(const Function &)> IsLiveInCalleeCB; |
1209 | |
1210 | if (auto *AI = dyn_cast<AllocaInst>(Val: &getAssociatedValue())) { |
1211 | // If the alloca containing function is not recursive the alloca |
1212 | // must be dead in the callee. |
1213 | const Function *AIFn = AI->getFunction(); |
1214 | ObjHasKernelLifetime = AIFn->hasFnAttribute(Kind: "kernel" ); |
1215 | bool IsKnownNoRecurse; |
1216 | if (AA::hasAssumedIRAttr<Attribute::NoRecurse>( |
1217 | A, this, IRPosition::function(*AIFn), DepClassTy::OPTIONAL, |
1218 | IsKnownNoRecurse)) { |
1219 | IsLiveInCalleeCB = [AIFn](const Function &Fn) { return AIFn != &Fn; }; |
1220 | } |
1221 | } else if (auto *GV = dyn_cast<GlobalValue>(Val: &getAssociatedValue())) { |
1222 | // If the global has kernel lifetime we can stop if we reach a kernel |
1223 | // as it is "dead" in the (unknown) callees. |
1224 | ObjHasKernelLifetime = HasKernelLifetime(GV, *GV->getParent()); |
1225 | if (ObjHasKernelLifetime) |
1226 | IsLiveInCalleeCB = [](const Function &Fn) { |
1227 | return !Fn.hasFnAttribute(Kind: "kernel" ); |
1228 | }; |
1229 | } |
1230 | |
1231 | // Set of accesses/instructions that will overwrite the result and are |
1232 | // therefore blockers in the reachability traversal. |
1233 | AA::InstExclusionSetTy ExclusionSet; |
1234 | |
1235 | auto AccessCB = [&](const Access &Acc, bool Exact) { |
1236 | Function *AccScope = Acc.getRemoteInst()->getFunction(); |
1237 | bool AccInSameScope = AccScope == &Scope; |
1238 | |
1239 | // If the object has kernel lifetime we can ignore accesses only reachable |
1240 | // by other kernels. For now we only skip accesses *in* other kernels. |
1241 | if (InstInKernel && ObjHasKernelLifetime && !AccInSameScope && |
1242 | AccScope->hasFnAttribute(Kind: "kernel" )) |
1243 | return true; |
1244 | |
1245 | if (Exact && Acc.isMustAccess() && Acc.getRemoteInst() != &I) { |
1246 | if (Acc.isWrite() || (isa<LoadInst>(Val: I) && Acc.isWriteOrAssumption())) |
1247 | ExclusionSet.insert(Ptr: Acc.getRemoteInst()); |
1248 | } |
1249 | |
1250 | if ((!FindInterferingWrites || !Acc.isWriteOrAssumption()) && |
1251 | (!FindInterferingReads || !Acc.isRead())) |
1252 | return true; |
1253 | |
1254 | bool Dominates = FindInterferingWrites && DT && Exact && |
1255 | Acc.isMustAccess() && AccInSameScope && |
1256 | DT->dominates(Def: Acc.getRemoteInst(), User: &I); |
1257 | if (Dominates) |
1258 | DominatingWrites.insert(Ptr: &Acc); |
1259 | |
1260 | // Track if all interesting accesses are in the same `nosync` function as |
1261 | // the given instruction. |
1262 | AllInSameNoSyncFn &= Acc.getRemoteInst()->getFunction() == &Scope; |
1263 | |
1264 | InterferingAccesses.push_back(Elt: {&Acc, Exact}); |
1265 | return true; |
1266 | }; |
1267 | if (!State::forallInterferingAccesses(I, CB: AccessCB, Range)) |
1268 | return false; |
1269 | |
1270 | HasBeenWrittenTo = !DominatingWrites.empty(); |
1271 | |
1272 | // Dominating writes form a chain, find the least/lowest member. |
1273 | Instruction *LeastDominatingWriteInst = nullptr; |
1274 | for (const Access *Acc : DominatingWrites) { |
1275 | if (!LeastDominatingWriteInst) { |
1276 | LeastDominatingWriteInst = Acc->getRemoteInst(); |
1277 | } else if (DT->dominates(Def: LeastDominatingWriteInst, |
1278 | User: Acc->getRemoteInst())) { |
1279 | LeastDominatingWriteInst = Acc->getRemoteInst(); |
1280 | } |
1281 | } |
1282 | |
1283 | // Helper to determine if we can skip a specific write access. |
1284 | auto CanSkipAccess = [&](const Access &Acc, bool Exact) { |
1285 | if (SkipCB && SkipCB(Acc)) |
1286 | return true; |
1287 | if (!CanIgnoreThreading(Acc)) |
1288 | return false; |
1289 | |
1290 | // Check read (RAW) dependences and write (WAR) dependences as necessary. |
1291 | // If we successfully excluded all effects we are interested in, the |
1292 | // access can be skipped. |
1293 | bool ReadChecked = !FindInterferingReads; |
1294 | bool WriteChecked = !FindInterferingWrites; |
1295 | |
1296 | // If the instruction cannot reach the access, the former does not |
1297 | // interfere with what the access reads. |
1298 | if (!ReadChecked) { |
1299 | if (!AA::isPotentiallyReachable(A, FromI: I, ToI: *Acc.getRemoteInst(), QueryingAA, |
1300 | ExclusionSet: &ExclusionSet, GoBackwardsCB: IsLiveInCalleeCB)) |
1301 | ReadChecked = true; |
1302 | } |
1303 | // If the instruction cannot be reach from the access, the latter does not |
1304 | // interfere with what the instruction reads. |
1305 | if (!WriteChecked) { |
1306 | if (!AA::isPotentiallyReachable(A, FromI: *Acc.getRemoteInst(), ToI: I, QueryingAA, |
1307 | ExclusionSet: &ExclusionSet, GoBackwardsCB: IsLiveInCalleeCB)) |
1308 | WriteChecked = true; |
1309 | } |
1310 | |
1311 | // If we still might be affected by the write of the access but there are |
1312 | // dominating writes in the function of the instruction |
1313 | // (HasBeenWrittenTo), we can try to reason that the access is overwritten |
1314 | // by them. This would have happend above if they are all in the same |
1315 | // function, so we only check the inter-procedural case. Effectively, we |
1316 | // want to show that there is no call after the dominting write that might |
1317 | // reach the access, and when it returns reach the instruction with the |
1318 | // updated value. To this end, we iterate all call sites, check if they |
1319 | // might reach the instruction without going through another access |
1320 | // (ExclusionSet) and at the same time might reach the access. However, |
1321 | // that is all part of AAInterFnReachability. |
1322 | if (!WriteChecked && HasBeenWrittenTo && |
1323 | Acc.getRemoteInst()->getFunction() != &Scope) { |
1324 | |
1325 | const auto *FnReachabilityAA = A.getAAFor<AAInterFnReachability>( |
1326 | QueryingAA, IRP: IRPosition::function(F: Scope), DepClass: DepClassTy::OPTIONAL); |
1327 | |
1328 | // Without going backwards in the call tree, can we reach the access |
1329 | // from the least dominating write. Do not allow to pass the instruction |
1330 | // itself either. |
1331 | bool Inserted = ExclusionSet.insert(Ptr: &I).second; |
1332 | |
1333 | if (!FnReachabilityAA || |
1334 | !FnReachabilityAA->instructionCanReach( |
1335 | A, Inst: *LeastDominatingWriteInst, |
1336 | Fn: *Acc.getRemoteInst()->getFunction(), ExclusionSet: &ExclusionSet)) |
1337 | WriteChecked = true; |
1338 | |
1339 | if (Inserted) |
1340 | ExclusionSet.erase(Ptr: &I); |
1341 | } |
1342 | |
1343 | if (ReadChecked && WriteChecked) |
1344 | return true; |
1345 | |
1346 | if (!DT || !UseDominanceReasoning) |
1347 | return false; |
1348 | if (!DominatingWrites.count(Ptr: &Acc)) |
1349 | return false; |
1350 | return LeastDominatingWriteInst != Acc.getRemoteInst(); |
1351 | }; |
1352 | |
1353 | // Run the user callback on all accesses we cannot skip and return if |
1354 | // that succeeded for all or not. |
1355 | for (auto &It : InterferingAccesses) { |
1356 | if ((!AllInSameNoSyncFn && !IsThreadLocalObj && !ExecDomainAA) || |
1357 | !CanSkipAccess(*It.first, It.second)) { |
1358 | if (!UserCB(*It.first, It.second)) |
1359 | return false; |
1360 | } |
1361 | } |
1362 | return true; |
1363 | } |
1364 | |
1365 | ChangeStatus translateAndAddStateFromCallee(Attributor &A, |
1366 | const AAPointerInfo &OtherAA, |
1367 | CallBase &CB) { |
1368 | using namespace AA::PointerInfo; |
1369 | if (!OtherAA.getState().isValidState() || !isValidState()) |
1370 | return indicatePessimisticFixpoint(); |
1371 | |
1372 | const auto &OtherAAImpl = static_cast<const AAPointerInfoImpl &>(OtherAA); |
1373 | bool IsByval = OtherAAImpl.getAssociatedArgument()->hasByValAttr(); |
1374 | |
1375 | // Combine the accesses bin by bin. |
1376 | ChangeStatus Changed = ChangeStatus::UNCHANGED; |
1377 | const auto &State = OtherAAImpl.getState(); |
1378 | for (const auto &It : State) { |
1379 | for (auto Index : It.getSecond()) { |
1380 | const auto &RAcc = State.getAccess(Index); |
1381 | if (IsByval && !RAcc.isRead()) |
1382 | continue; |
1383 | bool UsedAssumedInformation = false; |
1384 | AccessKind AK = RAcc.getKind(); |
1385 | auto Content = A.translateArgumentToCallSiteContent( |
1386 | V: RAcc.getContent(), CB, AA: *this, UsedAssumedInformation); |
1387 | AK = AccessKind(AK & (IsByval ? AccessKind::AK_R : AccessKind::AK_RW)); |
1388 | AK = AccessKind(AK | (RAcc.isMayAccess() ? AK_MAY : AK_MUST)); |
1389 | |
1390 | Changed |= addAccess(A, Ranges: RAcc.getRanges(), I&: CB, Content, Kind: AK, |
1391 | Ty: RAcc.getType(), RemoteI: RAcc.getRemoteInst()); |
1392 | } |
1393 | } |
1394 | return Changed; |
1395 | } |
1396 | |
1397 | ChangeStatus translateAndAddState(Attributor &A, const AAPointerInfo &OtherAA, |
1398 | const OffsetInfo &Offsets, CallBase &CB) { |
1399 | using namespace AA::PointerInfo; |
1400 | if (!OtherAA.getState().isValidState() || !isValidState()) |
1401 | return indicatePessimisticFixpoint(); |
1402 | |
1403 | const auto &OtherAAImpl = static_cast<const AAPointerInfoImpl &>(OtherAA); |
1404 | |
1405 | // Combine the accesses bin by bin. |
1406 | ChangeStatus Changed = ChangeStatus::UNCHANGED; |
1407 | const auto &State = OtherAAImpl.getState(); |
1408 | for (const auto &It : State) { |
1409 | for (auto Index : It.getSecond()) { |
1410 | const auto &RAcc = State.getAccess(Index); |
1411 | for (auto Offset : Offsets) { |
1412 | auto NewRanges = Offset == AA::RangeTy::Unknown |
1413 | ? AA::RangeTy::getUnknown() |
1414 | : RAcc.getRanges(); |
1415 | if (!NewRanges.isUnknown()) { |
1416 | NewRanges.addToAllOffsets(Inc: Offset); |
1417 | } |
1418 | Changed |= |
1419 | addAccess(A, Ranges: NewRanges, I&: CB, Content: RAcc.getContent(), Kind: RAcc.getKind(), |
1420 | Ty: RAcc.getType(), RemoteI: RAcc.getRemoteInst()); |
1421 | } |
1422 | } |
1423 | } |
1424 | return Changed; |
1425 | } |
1426 | |
1427 | /// Statistic tracking for all AAPointerInfo implementations. |
1428 | /// See AbstractAttribute::trackStatistics(). |
1429 | void trackPointerInfoStatistics(const IRPosition &IRP) const {} |
1430 | |
1431 | /// Dump the state into \p O. |
1432 | void dumpState(raw_ostream &O) { |
1433 | for (auto &It : OffsetBins) { |
1434 | O << "[" << It.first.Offset << "-" << It.first.Offset + It.first.Size |
1435 | << "] : " << It.getSecond().size() << "\n" ; |
1436 | for (auto AccIndex : It.getSecond()) { |
1437 | auto &Acc = AccessList[AccIndex]; |
1438 | O << " - " << Acc.getKind() << " - " << *Acc.getLocalInst() << "\n" ; |
1439 | if (Acc.getLocalInst() != Acc.getRemoteInst()) |
1440 | O << " --> " << *Acc.getRemoteInst() |
1441 | << "\n" ; |
1442 | if (!Acc.isWrittenValueYetUndetermined()) { |
1443 | if (isa_and_nonnull<Function>(Val: Acc.getWrittenValue())) |
1444 | O << " - c: func " << Acc.getWrittenValue()->getName() |
1445 | << "\n" ; |
1446 | else if (Acc.getWrittenValue()) |
1447 | O << " - c: " << *Acc.getWrittenValue() << "\n" ; |
1448 | else |
1449 | O << " - c: <unknown>\n" ; |
1450 | } |
1451 | } |
1452 | } |
1453 | } |
1454 | }; |
1455 | |
1456 | struct AAPointerInfoFloating : public AAPointerInfoImpl { |
1457 | using AccessKind = AAPointerInfo::AccessKind; |
1458 | AAPointerInfoFloating(const IRPosition &IRP, Attributor &A) |
1459 | : AAPointerInfoImpl(IRP, A) {} |
1460 | |
1461 | /// Deal with an access and signal if it was handled successfully. |
1462 | bool handleAccess(Attributor &A, Instruction &I, |
1463 | std::optional<Value *> Content, AccessKind Kind, |
1464 | SmallVectorImpl<int64_t> &Offsets, ChangeStatus &Changed, |
1465 | Type &Ty) { |
1466 | using namespace AA::PointerInfo; |
1467 | auto Size = AA::RangeTy::Unknown; |
1468 | const DataLayout &DL = A.getDataLayout(); |
1469 | TypeSize AccessSize = DL.getTypeStoreSize(Ty: &Ty); |
1470 | if (!AccessSize.isScalable()) |
1471 | Size = AccessSize.getFixedValue(); |
1472 | |
1473 | // Make a strictly ascending list of offsets as required by addAccess() |
1474 | llvm::sort(C&: Offsets); |
1475 | auto *Last = std::unique(first: Offsets.begin(), last: Offsets.end()); |
1476 | Offsets.erase(CS: Last, CE: Offsets.end()); |
1477 | |
1478 | VectorType *VT = dyn_cast<VectorType>(Val: &Ty); |
1479 | if (!VT || VT->getElementCount().isScalable() || |
1480 | !Content.value_or(u: nullptr) || !isa<Constant>(Val: *Content) || |
1481 | (*Content)->getType() != VT || |
1482 | DL.getTypeStoreSize(Ty: VT->getElementType()).isScalable()) { |
1483 | Changed = Changed | addAccess(A, Ranges: {Offsets, Size}, I, Content, Kind, Ty: &Ty); |
1484 | } else { |
1485 | // Handle vector stores with constant content element-wise. |
1486 | // TODO: We could look for the elements or create instructions |
1487 | // representing them. |
1488 | // TODO: We need to push the Content into the range abstraction |
1489 | // (AA::RangeTy) to allow different content values for different |
1490 | // ranges. ranges. Hence, support vectors storing different values. |
1491 | Type *ElementType = VT->getElementType(); |
1492 | int64_t ElementSize = DL.getTypeStoreSize(Ty: ElementType).getFixedValue(); |
1493 | auto *ConstContent = cast<Constant>(Val: *Content); |
1494 | Type *Int32Ty = Type::getInt32Ty(C&: ElementType->getContext()); |
1495 | SmallVector<int64_t> ElementOffsets(Offsets.begin(), Offsets.end()); |
1496 | |
1497 | for (int i = 0, e = VT->getElementCount().getFixedValue(); i != e; ++i) { |
1498 | Value *ElementContent = ConstantExpr::getExtractElement( |
1499 | Vec: ConstContent, Idx: ConstantInt::get(Ty: Int32Ty, V: i)); |
1500 | |
1501 | // Add the element access. |
1502 | Changed = Changed | addAccess(A, Ranges: {ElementOffsets, ElementSize}, I, |
1503 | Content: ElementContent, Kind, Ty: ElementType); |
1504 | |
1505 | // Advance the offsets for the next element. |
1506 | for (auto &ElementOffset : ElementOffsets) |
1507 | ElementOffset += ElementSize; |
1508 | } |
1509 | } |
1510 | return true; |
1511 | }; |
1512 | |
1513 | /// See AbstractAttribute::updateImpl(...). |
1514 | ChangeStatus updateImpl(Attributor &A) override; |
1515 | |
1516 | /// If the indices to \p GEP can be traced to constants, incorporate all |
1517 | /// of these into \p UsrOI. |
1518 | /// |
1519 | /// \return true iff \p UsrOI is updated. |
1520 | bool collectConstantsForGEP(Attributor &A, const DataLayout &DL, |
1521 | OffsetInfo &UsrOI, const OffsetInfo &PtrOI, |
1522 | const GEPOperator *GEP); |
1523 | |
1524 | /// See AbstractAttribute::trackStatistics() |
1525 | void trackStatistics() const override { |
1526 | AAPointerInfoImpl::trackPointerInfoStatistics(IRP: getIRPosition()); |
1527 | } |
1528 | }; |
1529 | |
1530 | bool AAPointerInfoFloating::collectConstantsForGEP(Attributor &A, |
1531 | const DataLayout &DL, |
1532 | OffsetInfo &UsrOI, |
1533 | const OffsetInfo &PtrOI, |
1534 | const GEPOperator *GEP) { |
1535 | unsigned BitWidth = DL.getIndexTypeSizeInBits(Ty: GEP->getType()); |
1536 | MapVector<Value *, APInt> VariableOffsets; |
1537 | APInt ConstantOffset(BitWidth, 0); |
1538 | |
1539 | assert(!UsrOI.isUnknown() && !PtrOI.isUnknown() && |
1540 | "Don't look for constant values if the offset has already been " |
1541 | "determined to be unknown." ); |
1542 | |
1543 | if (!GEP->collectOffset(DL, BitWidth, VariableOffsets, ConstantOffset)) { |
1544 | UsrOI.setUnknown(); |
1545 | return true; |
1546 | } |
1547 | |
1548 | LLVM_DEBUG(dbgs() << "[AAPointerInfo] GEP offset is " |
1549 | << (VariableOffsets.empty() ? "" : "not" ) << " constant " |
1550 | << *GEP << "\n" ); |
1551 | |
1552 | auto Union = PtrOI; |
1553 | Union.addToAll(Inc: ConstantOffset.getSExtValue()); |
1554 | |
1555 | // Each VI in VariableOffsets has a set of potential constant values. Every |
1556 | // combination of elements, picked one each from these sets, is separately |
1557 | // added to the original set of offsets, thus resulting in more offsets. |
1558 | for (const auto &VI : VariableOffsets) { |
1559 | auto *PotentialConstantsAA = A.getAAFor<AAPotentialConstantValues>( |
1560 | QueryingAA: *this, IRP: IRPosition::value(V: *VI.first), DepClass: DepClassTy::OPTIONAL); |
1561 | if (!PotentialConstantsAA || !PotentialConstantsAA->isValidState()) { |
1562 | UsrOI.setUnknown(); |
1563 | return true; |
1564 | } |
1565 | |
1566 | // UndefValue is treated as a zero, which leaves Union as is. |
1567 | if (PotentialConstantsAA->undefIsContained()) |
1568 | continue; |
1569 | |
1570 | // We need at least one constant in every set to compute an actual offset. |
1571 | // Otherwise, we end up pessimizing AAPointerInfo by respecting offsets that |
1572 | // don't actually exist. In other words, the absence of constant values |
1573 | // implies that the operation can be assumed dead for now. |
1574 | auto &AssumedSet = PotentialConstantsAA->getAssumedSet(); |
1575 | if (AssumedSet.empty()) |
1576 | return false; |
1577 | |
1578 | OffsetInfo Product; |
1579 | for (const auto &ConstOffset : AssumedSet) { |
1580 | auto CopyPerOffset = Union; |
1581 | CopyPerOffset.addToAll(Inc: ConstOffset.getSExtValue() * |
1582 | VI.second.getZExtValue()); |
1583 | Product.merge(R: CopyPerOffset); |
1584 | } |
1585 | Union = Product; |
1586 | } |
1587 | |
1588 | UsrOI = std::move(Union); |
1589 | return true; |
1590 | } |
1591 | |
1592 | ChangeStatus AAPointerInfoFloating::updateImpl(Attributor &A) { |
1593 | using namespace AA::PointerInfo; |
1594 | ChangeStatus Changed = ChangeStatus::UNCHANGED; |
1595 | const DataLayout &DL = A.getDataLayout(); |
1596 | Value &AssociatedValue = getAssociatedValue(); |
1597 | |
1598 | DenseMap<Value *, OffsetInfo> OffsetInfoMap; |
1599 | OffsetInfoMap[&AssociatedValue].insert(Offset: 0); |
1600 | |
1601 | auto HandlePassthroughUser = [&](Value *Usr, Value *CurPtr, bool &Follow) { |
1602 | // One does not simply walk into a map and assign a reference to a possibly |
1603 | // new location. That can cause an invalidation before the assignment |
1604 | // happens, like so: |
1605 | // |
1606 | // OffsetInfoMap[Usr] = OffsetInfoMap[CurPtr]; /* bad idea! */ |
1607 | // |
1608 | // The RHS is a reference that may be invalidated by an insertion caused by |
1609 | // the LHS. So we ensure that the side-effect of the LHS happens first. |
1610 | auto &UsrOI = OffsetInfoMap[Usr]; |
1611 | auto &PtrOI = OffsetInfoMap[CurPtr]; |
1612 | assert(!PtrOI.isUnassigned() && |
1613 | "Cannot pass through if the input Ptr was not visited!" ); |
1614 | UsrOI = PtrOI; |
1615 | Follow = true; |
1616 | return true; |
1617 | }; |
1618 | |
1619 | const auto *F = getAnchorScope(); |
1620 | const auto *CI = |
1621 | F ? A.getInfoCache().getAnalysisResultForFunction<CycleAnalysis>(F: *F) |
1622 | : nullptr; |
1623 | const auto *TLI = |
1624 | F ? A.getInfoCache().getTargetLibraryInfoForFunction(F: *F) : nullptr; |
1625 | |
1626 | auto UsePred = [&](const Use &U, bool &Follow) -> bool { |
1627 | Value *CurPtr = U.get(); |
1628 | User *Usr = U.getUser(); |
1629 | LLVM_DEBUG(dbgs() << "[AAPointerInfo] Analyze " << *CurPtr << " in " << *Usr |
1630 | << "\n" ); |
1631 | assert(OffsetInfoMap.count(CurPtr) && |
1632 | "The current pointer offset should have been seeded!" ); |
1633 | |
1634 | if (ConstantExpr *CE = dyn_cast<ConstantExpr>(Val: Usr)) { |
1635 | if (CE->isCast()) |
1636 | return HandlePassthroughUser(Usr, CurPtr, Follow); |
1637 | if (CE->isCompare()) |
1638 | return true; |
1639 | if (!isa<GEPOperator>(Val: CE)) { |
1640 | LLVM_DEBUG(dbgs() << "[AAPointerInfo] Unhandled constant user " << *CE |
1641 | << "\n" ); |
1642 | return false; |
1643 | } |
1644 | } |
1645 | if (auto *GEP = dyn_cast<GEPOperator>(Val: Usr)) { |
1646 | // Note the order here, the Usr access might change the map, CurPtr is |
1647 | // already in it though. |
1648 | auto &UsrOI = OffsetInfoMap[Usr]; |
1649 | auto &PtrOI = OffsetInfoMap[CurPtr]; |
1650 | |
1651 | if (UsrOI.isUnknown()) |
1652 | return true; |
1653 | |
1654 | if (PtrOI.isUnknown()) { |
1655 | Follow = true; |
1656 | UsrOI.setUnknown(); |
1657 | return true; |
1658 | } |
1659 | |
1660 | Follow = collectConstantsForGEP(A, DL, UsrOI, PtrOI, GEP); |
1661 | return true; |
1662 | } |
1663 | if (isa<PtrToIntInst>(Val: Usr)) |
1664 | return false; |
1665 | if (isa<CastInst>(Val: Usr) || isa<SelectInst>(Val: Usr) || isa<ReturnInst>(Val: Usr)) |
1666 | return HandlePassthroughUser(Usr, CurPtr, Follow); |
1667 | |
1668 | // For PHIs we need to take care of the recurrence explicitly as the value |
1669 | // might change while we iterate through a loop. For now, we give up if |
1670 | // the PHI is not invariant. |
1671 | if (isa<PHINode>(Val: Usr)) { |
1672 | // Note the order here, the Usr access might change the map, CurPtr is |
1673 | // already in it though. |
1674 | bool IsFirstPHIUser = !OffsetInfoMap.count(Val: Usr); |
1675 | auto &UsrOI = OffsetInfoMap[Usr]; |
1676 | auto &PtrOI = OffsetInfoMap[CurPtr]; |
1677 | |
1678 | // Check if the PHI operand has already an unknown offset as we can't |
1679 | // improve on that anymore. |
1680 | if (PtrOI.isUnknown()) { |
1681 | LLVM_DEBUG(dbgs() << "[AAPointerInfo] PHI operand offset unknown " |
1682 | << *CurPtr << " in " << *Usr << "\n" ); |
1683 | Follow = !UsrOI.isUnknown(); |
1684 | UsrOI.setUnknown(); |
1685 | return true; |
1686 | } |
1687 | |
1688 | // Check if the PHI is invariant (so far). |
1689 | if (UsrOI == PtrOI) { |
1690 | assert(!PtrOI.isUnassigned() && |
1691 | "Cannot assign if the current Ptr was not visited!" ); |
1692 | LLVM_DEBUG(dbgs() << "[AAPointerInfo] PHI is invariant (so far)" ); |
1693 | return true; |
1694 | } |
1695 | |
1696 | // Check if the PHI operand can be traced back to AssociatedValue. |
1697 | APInt Offset( |
1698 | DL.getIndexSizeInBits(AS: CurPtr->getType()->getPointerAddressSpace()), |
1699 | 0); |
1700 | Value *CurPtrBase = CurPtr->stripAndAccumulateConstantOffsets( |
1701 | DL, Offset, /* AllowNonInbounds */ true); |
1702 | auto It = OffsetInfoMap.find(Val: CurPtrBase); |
1703 | if (It == OffsetInfoMap.end()) { |
1704 | LLVM_DEBUG(dbgs() << "[AAPointerInfo] PHI operand is too complex " |
1705 | << *CurPtr << " in " << *Usr << "\n" ); |
1706 | UsrOI.setUnknown(); |
1707 | Follow = true; |
1708 | return true; |
1709 | } |
1710 | |
1711 | // Check if the PHI operand is not dependent on the PHI itself. Every |
1712 | // recurrence is a cyclic net of PHIs in the data flow, and has an |
1713 | // equivalent Cycle in the control flow. One of those PHIs must be in the |
1714 | // header of that control flow Cycle. This is independent of the choice of |
1715 | // Cycles reported by CycleInfo. It is sufficient to check the PHIs in |
1716 | // every Cycle header; if such a node is marked unknown, this will |
1717 | // eventually propagate through the whole net of PHIs in the recurrence. |
1718 | if (mayBeInCycle(CI, I: cast<Instruction>(Val: Usr), /* HeaderOnly */ true)) { |
1719 | auto BaseOI = It->getSecond(); |
1720 | BaseOI.addToAll(Inc: Offset.getZExtValue()); |
1721 | if (IsFirstPHIUser || BaseOI == UsrOI) { |
1722 | LLVM_DEBUG(dbgs() << "[AAPointerInfo] PHI is invariant " << *CurPtr |
1723 | << " in " << *Usr << "\n" ); |
1724 | return HandlePassthroughUser(Usr, CurPtr, Follow); |
1725 | } |
1726 | |
1727 | LLVM_DEBUG( |
1728 | dbgs() << "[AAPointerInfo] PHI operand pointer offset mismatch " |
1729 | << *CurPtr << " in " << *Usr << "\n" ); |
1730 | UsrOI.setUnknown(); |
1731 | Follow = true; |
1732 | return true; |
1733 | } |
1734 | |
1735 | UsrOI.merge(R: PtrOI); |
1736 | Follow = true; |
1737 | return true; |
1738 | } |
1739 | |
1740 | if (auto *LoadI = dyn_cast<LoadInst>(Val: Usr)) { |
1741 | // If the access is to a pointer that may or may not be the associated |
1742 | // value, e.g. due to a PHI, we cannot assume it will be read. |
1743 | AccessKind AK = AccessKind::AK_R; |
1744 | if (getUnderlyingObject(V: CurPtr) == &AssociatedValue) |
1745 | AK = AccessKind(AK | AccessKind::AK_MUST); |
1746 | else |
1747 | AK = AccessKind(AK | AccessKind::AK_MAY); |
1748 | if (!handleAccess(A, I&: *LoadI, /* Content */ nullptr, Kind: AK, |
1749 | Offsets&: OffsetInfoMap[CurPtr].Offsets, Changed, |
1750 | Ty&: *LoadI->getType())) |
1751 | return false; |
1752 | |
1753 | auto IsAssumption = [](Instruction &I) { |
1754 | if (auto *II = dyn_cast<IntrinsicInst>(Val: &I)) |
1755 | return II->isAssumeLikeIntrinsic(); |
1756 | return false; |
1757 | }; |
1758 | |
1759 | auto IsImpactedInRange = [&](Instruction *FromI, Instruction *ToI) { |
1760 | // Check if the assumption and the load are executed together without |
1761 | // memory modification. |
1762 | do { |
1763 | if (FromI->mayWriteToMemory() && !IsAssumption(*FromI)) |
1764 | return true; |
1765 | FromI = FromI->getNextNonDebugInstruction(); |
1766 | } while (FromI && FromI != ToI); |
1767 | return false; |
1768 | }; |
1769 | |
1770 | BasicBlock *BB = LoadI->getParent(); |
1771 | auto IsValidAssume = [&](IntrinsicInst &IntrI) { |
1772 | if (IntrI.getIntrinsicID() != Intrinsic::assume) |
1773 | return false; |
1774 | BasicBlock *IntrBB = IntrI.getParent(); |
1775 | if (IntrI.getParent() == BB) { |
1776 | if (IsImpactedInRange(LoadI->getNextNonDebugInstruction(), &IntrI)) |
1777 | return false; |
1778 | } else { |
1779 | auto PredIt = pred_begin(BB: IntrBB); |
1780 | if (PredIt == pred_end(BB: IntrBB)) |
1781 | return false; |
1782 | if ((*PredIt) != BB) |
1783 | return false; |
1784 | if (++PredIt != pred_end(BB: IntrBB)) |
1785 | return false; |
1786 | for (auto *SuccBB : successors(BB)) { |
1787 | if (SuccBB == IntrBB) |
1788 | continue; |
1789 | if (isa<UnreachableInst>(Val: SuccBB->getTerminator())) |
1790 | continue; |
1791 | return false; |
1792 | } |
1793 | if (IsImpactedInRange(LoadI->getNextNonDebugInstruction(), |
1794 | BB->getTerminator())) |
1795 | return false; |
1796 | if (IsImpactedInRange(&IntrBB->front(), &IntrI)) |
1797 | return false; |
1798 | } |
1799 | return true; |
1800 | }; |
1801 | |
1802 | std::pair<Value *, IntrinsicInst *> Assumption; |
1803 | for (const Use &LoadU : LoadI->uses()) { |
1804 | if (auto *CmpI = dyn_cast<CmpInst>(Val: LoadU.getUser())) { |
1805 | if (!CmpI->isEquality() || !CmpI->isTrueWhenEqual()) |
1806 | continue; |
1807 | for (const Use &CmpU : CmpI->uses()) { |
1808 | if (auto *IntrI = dyn_cast<IntrinsicInst>(Val: CmpU.getUser())) { |
1809 | if (!IsValidAssume(*IntrI)) |
1810 | continue; |
1811 | int Idx = CmpI->getOperandUse(i: 0) == LoadU; |
1812 | Assumption = {CmpI->getOperand(i_nocapture: Idx), IntrI}; |
1813 | break; |
1814 | } |
1815 | } |
1816 | } |
1817 | if (Assumption.first) |
1818 | break; |
1819 | } |
1820 | |
1821 | // Check if we found an assumption associated with this load. |
1822 | if (!Assumption.first || !Assumption.second) |
1823 | return true; |
1824 | |
1825 | LLVM_DEBUG(dbgs() << "[AAPointerInfo] Assumption found " |
1826 | << *Assumption.second << ": " << *LoadI |
1827 | << " == " << *Assumption.first << "\n" ); |
1828 | bool UsedAssumedInformation = false; |
1829 | std::optional<Value *> Content = nullptr; |
1830 | if (Assumption.first) |
1831 | Content = |
1832 | A.getAssumedSimplified(V: *Assumption.first, AA: *this, |
1833 | UsedAssumedInformation, S: AA::Interprocedural); |
1834 | return handleAccess( |
1835 | A, I&: *Assumption.second, Content, Kind: AccessKind::AK_ASSUMPTION, |
1836 | Offsets&: OffsetInfoMap[CurPtr].Offsets, Changed, Ty&: *LoadI->getType()); |
1837 | } |
1838 | |
1839 | auto HandleStoreLike = [&](Instruction &I, Value *ValueOp, Type &ValueTy, |
1840 | ArrayRef<Value *> OtherOps, AccessKind AK) { |
1841 | for (auto *OtherOp : OtherOps) { |
1842 | if (OtherOp == CurPtr) { |
1843 | LLVM_DEBUG( |
1844 | dbgs() |
1845 | << "[AAPointerInfo] Escaping use in store like instruction " << I |
1846 | << "\n" ); |
1847 | return false; |
1848 | } |
1849 | } |
1850 | |
1851 | // If the access is to a pointer that may or may not be the associated |
1852 | // value, e.g. due to a PHI, we cannot assume it will be written. |
1853 | if (getUnderlyingObject(V: CurPtr) == &AssociatedValue) |
1854 | AK = AccessKind(AK | AccessKind::AK_MUST); |
1855 | else |
1856 | AK = AccessKind(AK | AccessKind::AK_MAY); |
1857 | bool UsedAssumedInformation = false; |
1858 | std::optional<Value *> Content = nullptr; |
1859 | if (ValueOp) |
1860 | Content = A.getAssumedSimplified( |
1861 | V: *ValueOp, AA: *this, UsedAssumedInformation, S: AA::Interprocedural); |
1862 | return handleAccess(A, I, Content, Kind: AK, Offsets&: OffsetInfoMap[CurPtr].Offsets, |
1863 | Changed, Ty&: ValueTy); |
1864 | }; |
1865 | |
1866 | if (auto *StoreI = dyn_cast<StoreInst>(Val: Usr)) |
1867 | return HandleStoreLike(*StoreI, StoreI->getValueOperand(), |
1868 | *StoreI->getValueOperand()->getType(), |
1869 | {StoreI->getValueOperand()}, AccessKind::AK_W); |
1870 | if (auto *RMWI = dyn_cast<AtomicRMWInst>(Val: Usr)) |
1871 | return HandleStoreLike(*RMWI, nullptr, *RMWI->getValOperand()->getType(), |
1872 | {RMWI->getValOperand()}, AccessKind::AK_RW); |
1873 | if (auto *CXI = dyn_cast<AtomicCmpXchgInst>(Val: Usr)) |
1874 | return HandleStoreLike( |
1875 | *CXI, nullptr, *CXI->getNewValOperand()->getType(), |
1876 | {CXI->getCompareOperand(), CXI->getNewValOperand()}, |
1877 | AccessKind::AK_RW); |
1878 | |
1879 | if (auto *CB = dyn_cast<CallBase>(Val: Usr)) { |
1880 | if (CB->isLifetimeStartOrEnd()) |
1881 | return true; |
1882 | if (getFreedOperand(CB, TLI) == U) |
1883 | return true; |
1884 | if (CB->isArgOperand(U: &U)) { |
1885 | unsigned ArgNo = CB->getArgOperandNo(U: &U); |
1886 | const auto *CSArgPI = A.getAAFor<AAPointerInfo>( |
1887 | QueryingAA: *this, IRP: IRPosition::callsite_argument(CB: *CB, ArgNo), |
1888 | DepClass: DepClassTy::REQUIRED); |
1889 | if (!CSArgPI) |
1890 | return false; |
1891 | Changed = |
1892 | translateAndAddState(A, OtherAA: *CSArgPI, Offsets: OffsetInfoMap[CurPtr], CB&: *CB) | |
1893 | Changed; |
1894 | return isValidState(); |
1895 | } |
1896 | LLVM_DEBUG(dbgs() << "[AAPointerInfo] Call user not handled " << *CB |
1897 | << "\n" ); |
1898 | // TODO: Allow some call uses |
1899 | return false; |
1900 | } |
1901 | |
1902 | LLVM_DEBUG(dbgs() << "[AAPointerInfo] User not handled " << *Usr << "\n" ); |
1903 | return false; |
1904 | }; |
1905 | auto EquivalentUseCB = [&](const Use &OldU, const Use &NewU) { |
1906 | assert(OffsetInfoMap.count(OldU) && "Old use should be known already!" ); |
1907 | if (OffsetInfoMap.count(Val: NewU)) { |
1908 | LLVM_DEBUG({ |
1909 | if (!(OffsetInfoMap[NewU] == OffsetInfoMap[OldU])) { |
1910 | dbgs() << "[AAPointerInfo] Equivalent use callback failed: " |
1911 | << OffsetInfoMap[NewU] << " vs " << OffsetInfoMap[OldU] |
1912 | << "\n" ; |
1913 | } |
1914 | }); |
1915 | return OffsetInfoMap[NewU] == OffsetInfoMap[OldU]; |
1916 | } |
1917 | OffsetInfoMap[NewU] = OffsetInfoMap[OldU]; |
1918 | return true; |
1919 | }; |
1920 | if (!A.checkForAllUses(Pred: UsePred, QueryingAA: *this, V: AssociatedValue, |
1921 | /* CheckBBLivenessOnly */ true, LivenessDepClass: DepClassTy::OPTIONAL, |
1922 | /* IgnoreDroppableUses */ true, EquivalentUseCB)) { |
1923 | LLVM_DEBUG(dbgs() << "[AAPointerInfo] Check for all uses failed, abort!\n" ); |
1924 | return indicatePessimisticFixpoint(); |
1925 | } |
1926 | |
1927 | LLVM_DEBUG({ |
1928 | dbgs() << "Accesses by bin after update:\n" ; |
1929 | dumpState(dbgs()); |
1930 | }); |
1931 | |
1932 | return Changed; |
1933 | } |
1934 | |
1935 | struct AAPointerInfoReturned final : AAPointerInfoImpl { |
1936 | AAPointerInfoReturned(const IRPosition &IRP, Attributor &A) |
1937 | : AAPointerInfoImpl(IRP, A) {} |
1938 | |
1939 | /// See AbstractAttribute::updateImpl(...). |
1940 | ChangeStatus updateImpl(Attributor &A) override { |
1941 | return indicatePessimisticFixpoint(); |
1942 | } |
1943 | |
1944 | /// See AbstractAttribute::trackStatistics() |
1945 | void trackStatistics() const override { |
1946 | AAPointerInfoImpl::trackPointerInfoStatistics(IRP: getIRPosition()); |
1947 | } |
1948 | }; |
1949 | |
1950 | struct AAPointerInfoArgument final : AAPointerInfoFloating { |
1951 | AAPointerInfoArgument(const IRPosition &IRP, Attributor &A) |
1952 | : AAPointerInfoFloating(IRP, A) {} |
1953 | |
1954 | /// See AbstractAttribute::trackStatistics() |
1955 | void trackStatistics() const override { |
1956 | AAPointerInfoImpl::trackPointerInfoStatistics(IRP: getIRPosition()); |
1957 | } |
1958 | }; |
1959 | |
1960 | struct AAPointerInfoCallSiteArgument final : AAPointerInfoFloating { |
1961 | AAPointerInfoCallSiteArgument(const IRPosition &IRP, Attributor &A) |
1962 | : AAPointerInfoFloating(IRP, A) {} |
1963 | |
1964 | /// See AbstractAttribute::updateImpl(...). |
1965 | ChangeStatus updateImpl(Attributor &A) override { |
1966 | using namespace AA::PointerInfo; |
1967 | // We handle memory intrinsics explicitly, at least the first (= |
1968 | // destination) and second (=source) arguments as we know how they are |
1969 | // accessed. |
1970 | if (auto *MI = dyn_cast_or_null<MemIntrinsic>(Val: getCtxI())) { |
1971 | ConstantInt *Length = dyn_cast<ConstantInt>(Val: MI->getLength()); |
1972 | int64_t LengthVal = AA::RangeTy::Unknown; |
1973 | if (Length) |
1974 | LengthVal = Length->getSExtValue(); |
1975 | unsigned ArgNo = getIRPosition().getCallSiteArgNo(); |
1976 | ChangeStatus Changed = ChangeStatus::UNCHANGED; |
1977 | if (ArgNo > 1) { |
1978 | LLVM_DEBUG(dbgs() << "[AAPointerInfo] Unhandled memory intrinsic " |
1979 | << *MI << "\n" ); |
1980 | return indicatePessimisticFixpoint(); |
1981 | } else { |
1982 | auto Kind = |
1983 | ArgNo == 0 ? AccessKind::AK_MUST_WRITE : AccessKind::AK_MUST_READ; |
1984 | Changed = |
1985 | Changed | addAccess(A, Ranges: {0, LengthVal}, I&: *MI, Content: nullptr, Kind, Ty: nullptr); |
1986 | } |
1987 | LLVM_DEBUG({ |
1988 | dbgs() << "Accesses by bin after update:\n" ; |
1989 | dumpState(dbgs()); |
1990 | }); |
1991 | |
1992 | return Changed; |
1993 | } |
1994 | |
1995 | // TODO: Once we have call site specific value information we can provide |
1996 | // call site specific liveness information and then it makes |
1997 | // sense to specialize attributes for call sites arguments instead of |
1998 | // redirecting requests to the callee argument. |
1999 | Argument *Arg = getAssociatedArgument(); |
2000 | if (Arg) { |
2001 | const IRPosition &ArgPos = IRPosition::argument(Arg: *Arg); |
2002 | auto *ArgAA = |
2003 | A.getAAFor<AAPointerInfo>(QueryingAA: *this, IRP: ArgPos, DepClass: DepClassTy::REQUIRED); |
2004 | if (ArgAA && ArgAA->getState().isValidState()) |
2005 | return translateAndAddStateFromCallee(A, OtherAA: *ArgAA, |
2006 | CB&: *cast<CallBase>(Val: getCtxI())); |
2007 | if (!Arg->getParent()->isDeclaration()) |
2008 | return indicatePessimisticFixpoint(); |
2009 | } |
2010 | |
2011 | bool IsKnownNoCapture; |
2012 | if (!AA::hasAssumedIRAttr<Attribute::NoCapture>( |
2013 | A, this, getIRPosition(), DepClassTy::OPTIONAL, IsKnownNoCapture)) |
2014 | return indicatePessimisticFixpoint(); |
2015 | |
2016 | bool IsKnown = false; |
2017 | if (AA::isAssumedReadNone(A, IRP: getIRPosition(), QueryingAA: *this, IsKnown)) |
2018 | return ChangeStatus::UNCHANGED; |
2019 | bool ReadOnly = AA::isAssumedReadOnly(A, IRP: getIRPosition(), QueryingAA: *this, IsKnown); |
2020 | auto Kind = |
2021 | ReadOnly ? AccessKind::AK_MAY_READ : AccessKind::AK_MAY_READ_WRITE; |
2022 | return addAccess(A, Ranges: AA::RangeTy::getUnknown(), I&: *getCtxI(), Content: nullptr, Kind, |
2023 | Ty: nullptr); |
2024 | } |
2025 | |
2026 | /// See AbstractAttribute::trackStatistics() |
2027 | void trackStatistics() const override { |
2028 | AAPointerInfoImpl::trackPointerInfoStatistics(IRP: getIRPosition()); |
2029 | } |
2030 | }; |
2031 | |
2032 | struct AAPointerInfoCallSiteReturned final : AAPointerInfoFloating { |
2033 | AAPointerInfoCallSiteReturned(const IRPosition &IRP, Attributor &A) |
2034 | : AAPointerInfoFloating(IRP, A) {} |
2035 | |
2036 | /// See AbstractAttribute::trackStatistics() |
2037 | void trackStatistics() const override { |
2038 | AAPointerInfoImpl::trackPointerInfoStatistics(IRP: getIRPosition()); |
2039 | } |
2040 | }; |
2041 | } // namespace |
2042 | |
2043 | /// -----------------------NoUnwind Function Attribute-------------------------- |
2044 | |
2045 | namespace { |
2046 | struct AANoUnwindImpl : AANoUnwind { |
2047 | AANoUnwindImpl(const IRPosition &IRP, Attributor &A) : AANoUnwind(IRP, A) {} |
2048 | |
2049 | /// See AbstractAttribute::initialize(...). |
2050 | void initialize(Attributor &A) override { |
2051 | bool IsKnown; |
2052 | assert(!AA::hasAssumedIRAttr<Attribute::NoUnwind>( |
2053 | A, nullptr, getIRPosition(), DepClassTy::NONE, IsKnown)); |
2054 | (void)IsKnown; |
2055 | } |
2056 | |
2057 | const std::string getAsStr(Attributor *A) const override { |
2058 | return getAssumed() ? "nounwind" : "may-unwind" ; |
2059 | } |
2060 | |
2061 | /// See AbstractAttribute::updateImpl(...). |
2062 | ChangeStatus updateImpl(Attributor &A) override { |
2063 | auto Opcodes = { |
2064 | (unsigned)Instruction::Invoke, (unsigned)Instruction::CallBr, |
2065 | (unsigned)Instruction::Call, (unsigned)Instruction::CleanupRet, |
2066 | (unsigned)Instruction::CatchSwitch, (unsigned)Instruction::Resume}; |
2067 | |
2068 | auto CheckForNoUnwind = [&](Instruction &I) { |
2069 | if (!I.mayThrow(/* IncludePhaseOneUnwind */ true)) |
2070 | return true; |
2071 | |
2072 | if (const auto *CB = dyn_cast<CallBase>(Val: &I)) { |
2073 | bool IsKnownNoUnwind; |
2074 | return AA::hasAssumedIRAttr<Attribute::NoUnwind>( |
2075 | A, this, IRPosition::callsite_function(*CB), DepClassTy::REQUIRED, |
2076 | IsKnownNoUnwind); |
2077 | } |
2078 | return false; |
2079 | }; |
2080 | |
2081 | bool UsedAssumedInformation = false; |
2082 | if (!A.checkForAllInstructions(CheckForNoUnwind, *this, Opcodes, |
2083 | UsedAssumedInformation)) |
2084 | return indicatePessimisticFixpoint(); |
2085 | |
2086 | return ChangeStatus::UNCHANGED; |
2087 | } |
2088 | }; |
2089 | |
2090 | struct AANoUnwindFunction final : public AANoUnwindImpl { |
2091 | AANoUnwindFunction(const IRPosition &IRP, Attributor &A) |
2092 | : AANoUnwindImpl(IRP, A) {} |
2093 | |
2094 | /// See AbstractAttribute::trackStatistics() |
2095 | void trackStatistics() const override { STATS_DECLTRACK_FN_ATTR(nounwind) } |
2096 | }; |
2097 | |
2098 | /// NoUnwind attribute deduction for a call sites. |
2099 | struct AANoUnwindCallSite final |
2100 | : AACalleeToCallSite<AANoUnwind, AANoUnwindImpl> { |
2101 | AANoUnwindCallSite(const IRPosition &IRP, Attributor &A) |
2102 | : AACalleeToCallSite<AANoUnwind, AANoUnwindImpl>(IRP, A) {} |
2103 | |
2104 | /// See AbstractAttribute::trackStatistics() |
2105 | void trackStatistics() const override { STATS_DECLTRACK_CS_ATTR(nounwind); } |
2106 | }; |
2107 | } // namespace |
2108 | |
2109 | /// ------------------------ NoSync Function Attribute ------------------------- |
2110 | |
2111 | bool AANoSync::isAlignedBarrier(const CallBase &CB, bool ExecutedAligned) { |
2112 | switch (CB.getIntrinsicID()) { |
2113 | case Intrinsic::nvvm_barrier0: |
2114 | case Intrinsic::nvvm_barrier0_and: |
2115 | case Intrinsic::nvvm_barrier0_or: |
2116 | case Intrinsic::nvvm_barrier0_popc: |
2117 | return true; |
2118 | case Intrinsic::amdgcn_s_barrier: |
2119 | if (ExecutedAligned) |
2120 | return true; |
2121 | break; |
2122 | default: |
2123 | break; |
2124 | } |
2125 | return hasAssumption(CB, AssumptionStr: KnownAssumptionString("ompx_aligned_barrier" )); |
2126 | } |
2127 | |
2128 | bool AANoSync::isNonRelaxedAtomic(const Instruction *I) { |
2129 | if (!I->isAtomic()) |
2130 | return false; |
2131 | |
2132 | if (auto *FI = dyn_cast<FenceInst>(Val: I)) |
2133 | // All legal orderings for fence are stronger than monotonic. |
2134 | return FI->getSyncScopeID() != SyncScope::SingleThread; |
2135 | if (auto *AI = dyn_cast<AtomicCmpXchgInst>(Val: I)) { |
2136 | // Unordered is not a legal ordering for cmpxchg. |
2137 | return (AI->getSuccessOrdering() != AtomicOrdering::Monotonic || |
2138 | AI->getFailureOrdering() != AtomicOrdering::Monotonic); |
2139 | } |
2140 | |
2141 | AtomicOrdering Ordering; |
2142 | switch (I->getOpcode()) { |
2143 | case Instruction::AtomicRMW: |
2144 | Ordering = cast<AtomicRMWInst>(Val: I)->getOrdering(); |
2145 | break; |
2146 | case Instruction::Store: |
2147 | Ordering = cast<StoreInst>(Val: I)->getOrdering(); |
2148 | break; |
2149 | case Instruction::Load: |
2150 | Ordering = cast<LoadInst>(Val: I)->getOrdering(); |
2151 | break; |
2152 | default: |
2153 | llvm_unreachable( |
2154 | "New atomic operations need to be known in the attributor." ); |
2155 | } |
2156 | |
2157 | return (Ordering != AtomicOrdering::Unordered && |
2158 | Ordering != AtomicOrdering::Monotonic); |
2159 | } |
2160 | |
2161 | /// Return true if this intrinsic is nosync. This is only used for intrinsics |
2162 | /// which would be nosync except that they have a volatile flag. All other |
2163 | /// intrinsics are simply annotated with the nosync attribute in Intrinsics.td. |
2164 | bool AANoSync::isNoSyncIntrinsic(const Instruction *I) { |
2165 | if (auto *MI = dyn_cast<MemIntrinsic>(Val: I)) |
2166 | return !MI->isVolatile(); |
2167 | return false; |
2168 | } |
2169 | |
2170 | namespace { |
2171 | struct AANoSyncImpl : AANoSync { |
2172 | AANoSyncImpl(const IRPosition &IRP, Attributor &A) : AANoSync(IRP, A) {} |
2173 | |
2174 | /// See AbstractAttribute::initialize(...). |
2175 | void initialize(Attributor &A) override { |
2176 | bool IsKnown; |
2177 | assert(!AA::hasAssumedIRAttr<Attribute::NoSync>(A, nullptr, getIRPosition(), |
2178 | DepClassTy::NONE, IsKnown)); |
2179 | (void)IsKnown; |
2180 | } |
2181 | |
2182 | const std::string getAsStr(Attributor *A) const override { |
2183 | return getAssumed() ? "nosync" : "may-sync" ; |
2184 | } |
2185 | |
2186 | /// See AbstractAttribute::updateImpl(...). |
2187 | ChangeStatus updateImpl(Attributor &A) override; |
2188 | }; |
2189 | |
2190 | ChangeStatus AANoSyncImpl::updateImpl(Attributor &A) { |
2191 | |
2192 | auto CheckRWInstForNoSync = [&](Instruction &I) { |
2193 | return AA::isNoSyncInst(A, I, *this); |
2194 | }; |
2195 | |
2196 | auto CheckForNoSync = [&](Instruction &I) { |
2197 | // At this point we handled all read/write effects and they are all |
2198 | // nosync, so they can be skipped. |
2199 | if (I.mayReadOrWriteMemory()) |
2200 | return true; |
2201 | |
2202 | bool IsKnown; |
2203 | CallBase &CB = cast<CallBase>(Val&: I); |
2204 | if (AA::hasAssumedIRAttr<Attribute::NoSync>( |
2205 | A, this, IRPosition::callsite_function(CB), DepClassTy::OPTIONAL, |
2206 | IsKnown)) |
2207 | return true; |
2208 | |
2209 | // non-convergent and readnone imply nosync. |
2210 | return !CB.isConvergent(); |
2211 | }; |
2212 | |
2213 | bool UsedAssumedInformation = false; |
2214 | if (!A.checkForAllReadWriteInstructions(CheckRWInstForNoSync, *this, |
2215 | UsedAssumedInformation) || |
2216 | !A.checkForAllCallLikeInstructions(CheckForNoSync, *this, |
2217 | UsedAssumedInformation)) |
2218 | return indicatePessimisticFixpoint(); |
2219 | |
2220 | return ChangeStatus::UNCHANGED; |
2221 | } |
2222 | |
2223 | struct AANoSyncFunction final : public AANoSyncImpl { |
2224 | AANoSyncFunction(const IRPosition &IRP, Attributor &A) |
2225 | : AANoSyncImpl(IRP, A) {} |
2226 | |
2227 | /// See AbstractAttribute::trackStatistics() |
2228 | void trackStatistics() const override { STATS_DECLTRACK_FN_ATTR(nosync) } |
2229 | }; |
2230 | |
2231 | /// NoSync attribute deduction for a call sites. |
2232 | struct AANoSyncCallSite final : AACalleeToCallSite<AANoSync, AANoSyncImpl> { |
2233 | AANoSyncCallSite(const IRPosition &IRP, Attributor &A) |
2234 | : AACalleeToCallSite<AANoSync, AANoSyncImpl>(IRP, A) {} |
2235 | |
2236 | /// See AbstractAttribute::trackStatistics() |
2237 | void trackStatistics() const override { STATS_DECLTRACK_CS_ATTR(nosync); } |
2238 | }; |
2239 | } // namespace |
2240 | |
2241 | /// ------------------------ No-Free Attributes ---------------------------- |
2242 | |
2243 | namespace { |
2244 | struct AANoFreeImpl : public AANoFree { |
2245 | AANoFreeImpl(const IRPosition &IRP, Attributor &A) : AANoFree(IRP, A) {} |
2246 | |
2247 | /// See AbstractAttribute::initialize(...). |
2248 | void initialize(Attributor &A) override { |
2249 | bool IsKnown; |
2250 | assert(!AA::hasAssumedIRAttr<Attribute::NoFree>(A, nullptr, getIRPosition(), |
2251 | DepClassTy::NONE, IsKnown)); |
2252 | (void)IsKnown; |
2253 | } |
2254 | |
2255 | /// See AbstractAttribute::updateImpl(...). |
2256 | ChangeStatus updateImpl(Attributor &A) override { |
2257 | auto CheckForNoFree = [&](Instruction &I) { |
2258 | bool IsKnown; |
2259 | return AA::hasAssumedIRAttr<Attribute::NoFree>( |
2260 | A, this, IRPosition::callsite_function(cast<CallBase>(I)), |
2261 | DepClassTy::REQUIRED, IsKnown); |
2262 | }; |
2263 | |
2264 | bool UsedAssumedInformation = false; |
2265 | if (!A.checkForAllCallLikeInstructions(CheckForNoFree, *this, |
2266 | UsedAssumedInformation)) |
2267 | return indicatePessimisticFixpoint(); |
2268 | return ChangeStatus::UNCHANGED; |
2269 | } |
2270 | |
2271 | /// See AbstractAttribute::getAsStr(). |
2272 | const std::string getAsStr(Attributor *A) const override { |
2273 | return getAssumed() ? "nofree" : "may-free" ; |
2274 | } |
2275 | }; |
2276 | |
2277 | struct AANoFreeFunction final : public AANoFreeImpl { |
2278 | AANoFreeFunction(const IRPosition &IRP, Attributor &A) |
2279 | : AANoFreeImpl(IRP, A) {} |
2280 | |
2281 | /// See AbstractAttribute::trackStatistics() |
2282 | void trackStatistics() const override { STATS_DECLTRACK_FN_ATTR(nofree) } |
2283 | }; |
2284 | |
2285 | /// NoFree attribute deduction for a call sites. |
2286 | struct AANoFreeCallSite final : AACalleeToCallSite<AANoFree, AANoFreeImpl> { |
2287 | AANoFreeCallSite(const IRPosition &IRP, Attributor &A) |
2288 | : AACalleeToCallSite<AANoFree, AANoFreeImpl>(IRP, A) {} |
2289 | |
2290 | /// See AbstractAttribute::trackStatistics() |
2291 | void trackStatistics() const override { STATS_DECLTRACK_CS_ATTR(nofree); } |
2292 | }; |
2293 | |
2294 | /// NoFree attribute for floating values. |
2295 | struct AANoFreeFloating : AANoFreeImpl { |
2296 | AANoFreeFloating(const IRPosition &IRP, Attributor &A) |
2297 | : AANoFreeImpl(IRP, A) {} |
2298 | |
2299 | /// See AbstractAttribute::trackStatistics() |
2300 | void trackStatistics() const override{STATS_DECLTRACK_FLOATING_ATTR(nofree)} |
2301 | |
2302 | /// See Abstract Attribute::updateImpl(...). |
2303 | ChangeStatus updateImpl(Attributor &A) override { |
2304 | const IRPosition &IRP = getIRPosition(); |
2305 | |
2306 | bool IsKnown; |
2307 | if (AA::hasAssumedIRAttr<Attribute::NoFree>(A, this, |
2308 | IRPosition::function_scope(IRP), |
2309 | DepClassTy::OPTIONAL, IsKnown)) |
2310 | return ChangeStatus::UNCHANGED; |
2311 | |
2312 | Value &AssociatedValue = getIRPosition().getAssociatedValue(); |
2313 | auto Pred = [&](const Use &U, bool &Follow) -> bool { |
2314 | Instruction *UserI = cast<Instruction>(Val: U.getUser()); |
2315 | if (auto *CB = dyn_cast<CallBase>(Val: UserI)) { |
2316 | if (CB->isBundleOperand(U: &U)) |
2317 | return false; |
2318 | if (!CB->isArgOperand(U: &U)) |
2319 | return true; |
2320 | unsigned ArgNo = CB->getArgOperandNo(U: &U); |
2321 | |
2322 | bool IsKnown; |
2323 | return AA::hasAssumedIRAttr<Attribute::NoFree>( |
2324 | A, this, IRPosition::callsite_argument(*CB, ArgNo), |
2325 | DepClassTy::REQUIRED, IsKnown); |
2326 | } |
2327 | |
2328 | if (isa<GetElementPtrInst>(Val: UserI) || isa<BitCastInst>(Val: UserI) || |
2329 | isa<PHINode>(Val: UserI) || isa<SelectInst>(Val: UserI)) { |
2330 | Follow = true; |
2331 | return true; |
2332 | } |
2333 | if (isa<StoreInst>(Val: UserI) || isa<LoadInst>(Val: UserI) || |
2334 | isa<ReturnInst>(Val: UserI)) |
2335 | return true; |
2336 | |
2337 | // Unknown user. |
2338 | return false; |
2339 | }; |
2340 | if (!A.checkForAllUses(Pred, *this, AssociatedValue)) |
2341 | return indicatePessimisticFixpoint(); |
2342 | |
2343 | return ChangeStatus::UNCHANGED; |
2344 | } |
2345 | }; |
2346 | |
2347 | /// NoFree attribute for a call site argument. |
2348 | struct AANoFreeArgument final : AANoFreeFloating { |
2349 | AANoFreeArgument(const IRPosition &IRP, Attributor &A) |
2350 | : AANoFreeFloating(IRP, A) {} |
2351 | |
2352 | /// See AbstractAttribute::trackStatistics() |
2353 | void trackStatistics() const override { STATS_DECLTRACK_ARG_ATTR(nofree) } |
2354 | }; |
2355 | |
2356 | /// NoFree attribute for call site arguments. |
2357 | struct AANoFreeCallSiteArgument final : AANoFreeFloating { |
2358 | AANoFreeCallSiteArgument(const IRPosition &IRP, Attributor &A) |
2359 | : AANoFreeFloating(IRP, A) {} |
2360 | |
2361 | /// See AbstractAttribute::updateImpl(...). |
2362 | ChangeStatus updateImpl(Attributor &A) override { |
2363 | // TODO: Once we have call site specific value information we can provide |
2364 | // call site specific liveness information and then it makes |
2365 | // sense to specialize attributes for call sites arguments instead of |
2366 | // redirecting requests to the callee argument. |
2367 | Argument *Arg = getAssociatedArgument(); |
2368 | if (!Arg) |
2369 | return indicatePessimisticFixpoint(); |
2370 | const IRPosition &ArgPos = IRPosition::argument(Arg: *Arg); |
2371 | bool IsKnown; |
2372 | if (AA::hasAssumedIRAttr<Attribute::NoFree>(A, this, ArgPos, |
2373 | DepClassTy::REQUIRED, IsKnown)) |
2374 | return ChangeStatus::UNCHANGED; |
2375 | return indicatePessimisticFixpoint(); |
2376 | } |
2377 | |
2378 | /// See AbstractAttribute::trackStatistics() |
2379 | void trackStatistics() const override{STATS_DECLTRACK_CSARG_ATTR(nofree)}; |
2380 | }; |
2381 | |
2382 | /// NoFree attribute for function return value. |
2383 | struct AANoFreeReturned final : AANoFreeFloating { |
2384 | AANoFreeReturned(const IRPosition &IRP, Attributor &A) |
2385 | : AANoFreeFloating(IRP, A) { |
2386 | llvm_unreachable("NoFree is not applicable to function returns!" ); |
2387 | } |
2388 | |
2389 | /// See AbstractAttribute::initialize(...). |
2390 | void initialize(Attributor &A) override { |
2391 | llvm_unreachable("NoFree is not applicable to function returns!" ); |
2392 | } |
2393 | |
2394 | /// See AbstractAttribute::updateImpl(...). |
2395 | ChangeStatus updateImpl(Attributor &A) override { |
2396 | llvm_unreachable("NoFree is not applicable to function returns!" ); |
2397 | } |
2398 | |
2399 | /// See AbstractAttribute::trackStatistics() |
2400 | void trackStatistics() const override {} |
2401 | }; |
2402 | |
2403 | /// NoFree attribute deduction for a call site return value. |
2404 | struct AANoFreeCallSiteReturned final : AANoFreeFloating { |
2405 | AANoFreeCallSiteReturned(const IRPosition &IRP, Attributor &A) |
2406 | : AANoFreeFloating(IRP, A) {} |
2407 | |
2408 | ChangeStatus manifest(Attributor &A) override { |
2409 | return ChangeStatus::UNCHANGED; |
2410 | } |
2411 | /// See AbstractAttribute::trackStatistics() |
2412 | void trackStatistics() const override { STATS_DECLTRACK_CSRET_ATTR(nofree) } |
2413 | }; |
2414 | } // namespace |
2415 | |
2416 | /// ------------------------ NonNull Argument Attribute ------------------------ |
2417 | |
2418 | bool AANonNull::isImpliedByIR(Attributor &A, const IRPosition &IRP, |
2419 | Attribute::AttrKind ImpliedAttributeKind, |
2420 | bool IgnoreSubsumingPositions) { |
2421 | SmallVector<Attribute::AttrKind, 2> AttrKinds; |
2422 | AttrKinds.push_back(Attribute::NonNull); |
2423 | if (!NullPointerIsDefined(IRP.getAnchorScope(), |
2424 | IRP.getAssociatedType()->getPointerAddressSpace())) |
2425 | AttrKinds.push_back(Attribute::Dereferenceable); |
2426 | if (A.hasAttr(IRP, AttrKinds, IgnoreSubsumingPositions, Attribute::NonNull)) |
2427 | return true; |
2428 | |
2429 | DominatorTree *DT = nullptr; |
2430 | AssumptionCache *AC = nullptr; |
2431 | InformationCache &InfoCache = A.getInfoCache(); |
2432 | if (const Function *Fn = IRP.getAnchorScope()) { |
2433 | if (!Fn->isDeclaration()) { |
2434 | DT = InfoCache.getAnalysisResultForFunction<DominatorTreeAnalysis>(F: *Fn); |
2435 | AC = InfoCache.getAnalysisResultForFunction<AssumptionAnalysis>(F: *Fn); |
2436 | } |
2437 | } |
2438 | |
2439 | SmallVector<AA::ValueAndContext> Worklist; |
2440 | if (IRP.getPositionKind() != IRP_RETURNED) { |
2441 | Worklist.push_back(Elt: {IRP.getAssociatedValue(), IRP.getCtxI()}); |
2442 | } else { |
2443 | bool UsedAssumedInformation = false; |
2444 | if (!A.checkForAllInstructions( |
2445 | Pred: [&](Instruction &I) { |
2446 | Worklist.push_back(Elt: {*cast<ReturnInst>(Val&: I).getReturnValue(), &I}); |
2447 | return true; |
2448 | }, |
2449 | Fn: IRP.getAssociatedFunction(), QueryingAA: nullptr, Opcodes: {Instruction::Ret}, |
2450 | UsedAssumedInformation)) |
2451 | return false; |
2452 | } |
2453 | |
2454 | if (llvm::any_of(Range&: Worklist, P: [&](AA::ValueAndContext VAC) { |
2455 | return !isKnownNonZero( |
2456 | V: VAC.getValue(), |
2457 | Q: SimplifyQuery(A.getDataLayout(), DT, AC, VAC.getCtxI())); |
2458 | })) |
2459 | return false; |
2460 | |
2461 | A.manifestAttrs(IRP, {Attribute::get(IRP.getAnchorValue().getContext(), |
2462 | Attribute::NonNull)}); |
2463 | return true; |
2464 | } |
2465 | |
2466 | namespace { |
2467 | static int64_t getKnownNonNullAndDerefBytesForUse( |
2468 | Attributor &A, const AbstractAttribute &QueryingAA, Value &AssociatedValue, |
2469 | const Use *U, const Instruction *I, bool &IsNonNull, bool &TrackUse) { |
2470 | TrackUse = false; |
2471 | |
2472 | const Value *UseV = U->get(); |
2473 | if (!UseV->getType()->isPointerTy()) |
2474 | return 0; |
2475 | |
2476 | // We need to follow common pointer manipulation uses to the accesses they |
2477 | // feed into. We can try to be smart to avoid looking through things we do not |
2478 | // like for now, e.g., non-inbounds GEPs. |
2479 | if (isa<CastInst>(Val: I)) { |
2480 | TrackUse = true; |
2481 | return 0; |
2482 | } |
2483 | |
2484 | if (isa<GetElementPtrInst>(Val: I)) { |
2485 | TrackUse = true; |
2486 | return 0; |
2487 | } |
2488 | |
2489 | Type *PtrTy = UseV->getType(); |
2490 | const Function *F = I->getFunction(); |
2491 | bool NullPointerIsDefined = |
2492 | F ? llvm::NullPointerIsDefined(F, AS: PtrTy->getPointerAddressSpace()) : true; |
2493 | const DataLayout &DL = A.getInfoCache().getDL(); |
2494 | if (const auto *CB = dyn_cast<CallBase>(Val: I)) { |
2495 | if (CB->isBundleOperand(U)) { |
2496 | if (RetainedKnowledge RK = getKnowledgeFromUse( |
2497 | U, {Attribute::NonNull, Attribute::Dereferenceable})) { |
2498 | IsNonNull |= |
2499 | (RK.AttrKind == Attribute::NonNull || !NullPointerIsDefined); |
2500 | return RK.ArgValue; |
2501 | } |
2502 | return 0; |
2503 | } |
2504 | |
2505 | if (CB->isCallee(U)) { |
2506 | IsNonNull |= !NullPointerIsDefined; |
2507 | return 0; |
2508 | } |
2509 | |
2510 | unsigned ArgNo = CB->getArgOperandNo(U); |
2511 | IRPosition IRP = IRPosition::callsite_argument(CB: *CB, ArgNo); |
2512 | // As long as we only use known information there is no need to track |
2513 | // dependences here. |
2514 | bool IsKnownNonNull; |
2515 | AA::hasAssumedIRAttr<Attribute::NonNull>(A, &QueryingAA, IRP, |
2516 | DepClassTy::NONE, IsKnownNonNull); |
2517 | IsNonNull |= IsKnownNonNull; |
2518 | auto *DerefAA = |
2519 | A.getAAFor<AADereferenceable>(QueryingAA, IRP, DepClass: DepClassTy::NONE); |
2520 | return DerefAA ? DerefAA->getKnownDereferenceableBytes() : 0; |
2521 | } |
2522 | |
2523 | std::optional<MemoryLocation> Loc = MemoryLocation::getOrNone(Inst: I); |
2524 | if (!Loc || Loc->Ptr != UseV || !Loc->Size.isPrecise() || |
2525 | Loc->Size.isScalable() || I->isVolatile()) |
2526 | return 0; |
2527 | |
2528 | int64_t Offset; |
2529 | const Value *Base = |
2530 | getMinimalBaseOfPointer(A, QueryingAA, Ptr: Loc->Ptr, BytesOffset&: Offset, DL); |
2531 | if (Base && Base == &AssociatedValue) { |
2532 | int64_t DerefBytes = Loc->Size.getValue() + Offset; |
2533 | IsNonNull |= !NullPointerIsDefined; |
2534 | return std::max(a: int64_t(0), b: DerefBytes); |
2535 | } |
2536 | |
2537 | /// Corner case when an offset is 0. |
2538 | Base = GetPointerBaseWithConstantOffset(Ptr: Loc->Ptr, Offset, DL, |
2539 | /*AllowNonInbounds*/ true); |
2540 | if (Base && Base == &AssociatedValue && Offset == 0) { |
2541 | int64_t DerefBytes = Loc->Size.getValue(); |
2542 | IsNonNull |= !NullPointerIsDefined; |
2543 | return std::max(a: int64_t(0), b: DerefBytes); |
2544 | } |
2545 | |
2546 | return 0; |
2547 | } |
2548 | |
2549 | struct AANonNullImpl : AANonNull { |
2550 | AANonNullImpl(const IRPosition &IRP, Attributor &A) : AANonNull(IRP, A) {} |
2551 | |
2552 | /// See AbstractAttribute::initialize(...). |
2553 | void initialize(Attributor &A) override { |
2554 | Value &V = *getAssociatedValue().stripPointerCasts(); |
2555 | if (isa<ConstantPointerNull>(Val: V)) { |
2556 | indicatePessimisticFixpoint(); |
2557 | return; |
2558 | } |
2559 | |
2560 | if (Instruction *CtxI = getCtxI()) |
2561 | followUsesInMBEC(*this, A, getState(), *CtxI); |
2562 | } |
2563 | |
2564 | /// See followUsesInMBEC |
2565 | bool followUseInMBEC(Attributor &A, const Use *U, const Instruction *I, |
2566 | AANonNull::StateType &State) { |
2567 | bool IsNonNull = false; |
2568 | bool TrackUse = false; |
2569 | getKnownNonNullAndDerefBytesForUse(A, *this, getAssociatedValue(), U, I, |
2570 | IsNonNull, TrackUse); |
2571 | State.setKnown(IsNonNull); |
2572 | return TrackUse; |
2573 | } |
2574 | |
2575 | /// See AbstractAttribute::getAsStr(). |
2576 | const std::string getAsStr(Attributor *A) const override { |
2577 | return getAssumed() ? "nonnull" : "may-null" ; |
2578 | } |
2579 | }; |
2580 | |
2581 | /// NonNull attribute for a floating value. |
2582 | struct AANonNullFloating : public AANonNullImpl { |
2583 | AANonNullFloating(const IRPosition &IRP, Attributor &A) |
2584 | : AANonNullImpl(IRP, A) {} |
2585 | |
2586 | /// See AbstractAttribute::updateImpl(...). |
2587 | ChangeStatus updateImpl(Attributor &A) override { |
2588 | auto CheckIRP = [&](const IRPosition &IRP) { |
2589 | bool IsKnownNonNull; |
2590 | return AA::hasAssumedIRAttr<Attribute::NonNull>( |
2591 | A, *this, IRP, DepClassTy::OPTIONAL, IsKnownNonNull); |
2592 | }; |
2593 | |
2594 | bool Stripped; |
2595 | bool UsedAssumedInformation = false; |
2596 | Value *AssociatedValue = &getAssociatedValue(); |
2597 | SmallVector<AA::ValueAndContext> Values; |
2598 | if (!A.getAssumedSimplifiedValues(IRP: getIRPosition(), AA: *this, Values, |
2599 | S: AA::AnyScope, UsedAssumedInformation)) |
2600 | Stripped = false; |
2601 | else |
2602 | Stripped = |
2603 | Values.size() != 1 || Values.front().getValue() != AssociatedValue; |
2604 | |
2605 | if (!Stripped) { |
2606 | bool IsKnown; |
2607 | if (auto *PHI = dyn_cast<PHINode>(AssociatedValue)) |
2608 | if (llvm::all_of(PHI->incoming_values(), [&](Value *Op) { |
2609 | return AA::hasAssumedIRAttr<Attribute::NonNull>( |
2610 | A, this, IRPosition::value(*Op), DepClassTy::OPTIONAL, |
2611 | IsKnown); |
2612 | })) |
2613 | return ChangeStatus::UNCHANGED; |
2614 | if (auto *Select = dyn_cast<SelectInst>(AssociatedValue)) |
2615 | if (AA::hasAssumedIRAttr<Attribute::NonNull>( |
2616 | A, this, IRPosition::value(*Select->getFalseValue()), |
2617 | DepClassTy::OPTIONAL, IsKnown) && |
2618 | AA::hasAssumedIRAttr<Attribute::NonNull>( |
2619 | A, this, IRPosition::value(*Select->getTrueValue()), |
2620 | DepClassTy::OPTIONAL, IsKnown)) |
2621 | return ChangeStatus::UNCHANGED; |
2622 | |
2623 | // If we haven't stripped anything we might still be able to use a |
2624 | // different AA, but only if the IRP changes. Effectively when we |
2625 | // interpret this not as a call site value but as a floating/argument |
2626 | // value. |
2627 | const IRPosition AVIRP = IRPosition::value(V: *AssociatedValue); |
2628 | if (AVIRP == getIRPosition() || !CheckIRP(AVIRP)) |
2629 | return indicatePessimisticFixpoint(); |
2630 | return ChangeStatus::UNCHANGED; |
2631 | } |
2632 | |
2633 | for (const auto &VAC : Values) |
2634 | if (!CheckIRP(IRPosition::value(V: *VAC.getValue()))) |
2635 | return indicatePessimisticFixpoint(); |
2636 | |
2637 | return ChangeStatus::UNCHANGED; |
2638 | } |
2639 | |
2640 | /// See AbstractAttribute::trackStatistics() |
2641 | void trackStatistics() const override { STATS_DECLTRACK_FNRET_ATTR(nonnull) } |
2642 | }; |
2643 | |
2644 | /// NonNull attribute for function return value. |
2645 | struct AANonNullReturned final |
2646 | : AAReturnedFromReturnedValues<AANonNull, AANonNull, AANonNull::StateType, |
2647 | false, AANonNull::IRAttributeKind, false> { |
2648 | AANonNullReturned(const IRPosition &IRP, Attributor &A) |
2649 | : AAReturnedFromReturnedValues<AANonNull, AANonNull, AANonNull::StateType, |
2650 | false, Attribute::NonNull, false>(IRP, A) { |
2651 | } |
2652 | |
2653 | /// See AbstractAttribute::getAsStr(). |
2654 | const std::string getAsStr(Attributor *A) const override { |
2655 | return getAssumed() ? "nonnull" : "may-null" ; |
2656 | } |
2657 | |
2658 | /// See AbstractAttribute::trackStatistics() |
2659 | void trackStatistics() const override { STATS_DECLTRACK_FNRET_ATTR(nonnull) } |
2660 | }; |
2661 | |
2662 | /// NonNull attribute for function argument. |
2663 | struct AANonNullArgument final |
2664 | : AAArgumentFromCallSiteArguments<AANonNull, AANonNullImpl> { |
2665 | AANonNullArgument(const IRPosition &IRP, Attributor &A) |
2666 | : AAArgumentFromCallSiteArguments<AANonNull, AANonNullImpl>(IRP, A) {} |
2667 | |
2668 | /// See AbstractAttribute::trackStatistics() |
2669 | void trackStatistics() const override { STATS_DECLTRACK_ARG_ATTR(nonnull) } |
2670 | }; |
2671 | |
2672 | struct AANonNullCallSiteArgument final : AANonNullFloating { |
2673 | AANonNullCallSiteArgument(const IRPosition &IRP, Attributor &A) |
2674 | : AANonNullFloating(IRP, A) {} |
2675 | |
2676 | /// See AbstractAttribute::trackStatistics() |
2677 | void trackStatistics() const override { STATS_DECLTRACK_CSARG_ATTR(nonnull) } |
2678 | }; |
2679 | |
2680 | /// NonNull attribute for a call site return position. |
2681 | struct AANonNullCallSiteReturned final |
2682 | : AACalleeToCallSite<AANonNull, AANonNullImpl> { |
2683 | AANonNullCallSiteReturned(const IRPosition &IRP, Attributor &A) |
2684 | : AACalleeToCallSite<AANonNull, AANonNullImpl>(IRP, A) {} |
2685 | |
2686 | /// See AbstractAttribute::trackStatistics() |
2687 | void trackStatistics() const override { STATS_DECLTRACK_CSRET_ATTR(nonnull) } |
2688 | }; |
2689 | } // namespace |
2690 | |
2691 | /// ------------------------ Must-Progress Attributes -------------------------- |
2692 | namespace { |
2693 | struct AAMustProgressImpl : public AAMustProgress { |
2694 | AAMustProgressImpl(const IRPosition &IRP, Attributor &A) |
2695 | : AAMustProgress(IRP, A) {} |
2696 | |
2697 | /// See AbstractAttribute::initialize(...). |
2698 | void initialize(Attributor &A) override { |
2699 | bool IsKnown; |
2700 | assert(!AA::hasAssumedIRAttr<Attribute::MustProgress>( |
2701 | A, nullptr, getIRPosition(), DepClassTy::NONE, IsKnown)); |
2702 | (void)IsKnown; |
2703 | } |
2704 | |
2705 | /// See AbstractAttribute::getAsStr() |
2706 | const std::string getAsStr(Attributor *A) const override { |
2707 | return getAssumed() ? "mustprogress" : "may-not-progress" ; |
2708 | } |
2709 | }; |
2710 | |
2711 | struct AAMustProgressFunction final : AAMustProgressImpl { |
2712 | AAMustProgressFunction(const IRPosition &IRP, Attributor &A) |
2713 | : AAMustProgressImpl(IRP, A) {} |
2714 | |
2715 | /// See AbstractAttribute::updateImpl(...). |
2716 | ChangeStatus updateImpl(Attributor &A) override { |
2717 | bool IsKnown; |
2718 | if (AA::hasAssumedIRAttr<Attribute::WillReturn>( |
2719 | A, this, getIRPosition(), DepClassTy::OPTIONAL, IsKnown)) { |
2720 | if (IsKnown) |
2721 | return indicateOptimisticFixpoint(); |
2722 | return ChangeStatus::UNCHANGED; |
2723 | } |
2724 | |
2725 | auto CheckForMustProgress = [&](AbstractCallSite ACS) { |
2726 | IRPosition IPos = IRPosition::callsite_function(CB: *ACS.getInstruction()); |
2727 | bool IsKnownMustProgress; |
2728 | return AA::hasAssumedIRAttr<Attribute::MustProgress>( |
2729 | A, this, IPos, DepClassTy::REQUIRED, IsKnownMustProgress, |
2730 | /* IgnoreSubsumingPositions */ true); |
2731 | }; |
2732 | |
2733 | bool AllCallSitesKnown = true; |
2734 | if (!A.checkForAllCallSites(CheckForMustProgress, *this, |
2735 | /* RequireAllCallSites */ true, |
2736 | AllCallSitesKnown)) |
2737 | return indicatePessimisticFixpoint(); |
2738 | |
2739 | return ChangeStatus::UNCHANGED; |
2740 | } |
2741 | |
2742 | /// See AbstractAttribute::trackStatistics() |
2743 | void trackStatistics() const override { |
2744 | STATS_DECLTRACK_FN_ATTR(mustprogress) |
2745 | } |
2746 | }; |
2747 | |
2748 | /// MustProgress attribute deduction for a call sites. |
2749 | struct AAMustProgressCallSite final : AAMustProgressImpl { |
2750 | AAMustProgressCallSite(const IRPosition &IRP, Attributor &A) |
2751 | : AAMustProgressImpl(IRP, A) {} |
2752 | |
2753 | /// See AbstractAttribute::updateImpl(...). |
2754 | ChangeStatus updateImpl(Attributor &A) override { |
2755 | // TODO: Once we have call site specific value information we can provide |
2756 | // call site specific liveness information and then it makes |
2757 | // sense to specialize attributes for call sites arguments instead of |
2758 | // redirecting requests to the callee argument. |
2759 | const IRPosition &FnPos = IRPosition::function(F: *getAnchorScope()); |
2760 | bool IsKnownMustProgress; |
2761 | if (!AA::hasAssumedIRAttr<Attribute::MustProgress>( |
2762 | A, this, FnPos, DepClassTy::REQUIRED, IsKnownMustProgress)) |
2763 | return indicatePessimisticFixpoint(); |
2764 | return ChangeStatus::UNCHANGED; |
2765 | } |
2766 | |
2767 | /// See AbstractAttribute::trackStatistics() |
2768 | void trackStatistics() const override { |
2769 | STATS_DECLTRACK_CS_ATTR(mustprogress); |
2770 | } |
2771 | }; |
2772 | } // namespace |
2773 | |
2774 | /// ------------------------ No-Recurse Attributes ---------------------------- |
2775 | |
2776 | namespace { |
2777 | struct AANoRecurseImpl : public AANoRecurse { |
2778 | AANoRecurseImpl(const IRPosition &IRP, Attributor &A) : AANoRecurse(IRP, A) {} |
2779 | |
2780 | /// See AbstractAttribute::initialize(...). |
2781 | void initialize(Attributor &A) override { |
2782 | bool IsKnown; |
2783 | assert(!AA::hasAssumedIRAttr<Attribute::NoRecurse>( |
2784 | A, nullptr, getIRPosition(), DepClassTy::NONE, IsKnown)); |
2785 | (void)IsKnown; |
2786 | } |
2787 | |
2788 | /// See AbstractAttribute::getAsStr() |
2789 | const std::string getAsStr(Attributor *A) const override { |
2790 | return getAssumed() ? "norecurse" : "may-recurse" ; |
2791 | } |
2792 | }; |
2793 | |
2794 | struct AANoRecurseFunction final : AANoRecurseImpl { |
2795 | AANoRecurseFunction(const IRPosition &IRP, Attributor &A) |
2796 | : AANoRecurseImpl(IRP, A) {} |
2797 | |
2798 | /// See AbstractAttribute::updateImpl(...). |
2799 | ChangeStatus updateImpl(Attributor &A) override { |
2800 | |
2801 | // If all live call sites are known to be no-recurse, we are as well. |
2802 | auto CallSitePred = [&](AbstractCallSite ACS) { |
2803 | bool IsKnownNoRecurse; |
2804 | if (!AA::hasAssumedIRAttr<Attribute::NoRecurse>( |
2805 | A, this, |
2806 | IRPosition::function(*ACS.getInstruction()->getFunction()), |
2807 | DepClassTy::NONE, IsKnownNoRecurse)) |
2808 | return false; |
2809 | return IsKnownNoRecurse; |
2810 | }; |
2811 | bool UsedAssumedInformation = false; |
2812 | if (A.checkForAllCallSites(CallSitePred, *this, true, |
2813 | UsedAssumedInformation)) { |
2814 | // If we know all call sites and all are known no-recurse, we are done. |
2815 | // If all known call sites, which might not be all that exist, are known |
2816 | // to be no-recurse, we are not done but we can continue to assume |
2817 | // no-recurse. If one of the call sites we have not visited will become |
2818 | // live, another update is triggered. |
2819 | if (!UsedAssumedInformation) |
2820 | indicateOptimisticFixpoint(); |
2821 | return ChangeStatus::UNCHANGED; |
2822 | } |
2823 | |
2824 | const AAInterFnReachability *EdgeReachability = |
2825 | A.getAAFor<AAInterFnReachability>(*this, getIRPosition(), |
2826 | DepClassTy::REQUIRED); |
2827 | if (EdgeReachability && EdgeReachability->canReach(A, Fn: *getAnchorScope())) |
2828 | return indicatePessimisticFixpoint(); |
2829 | return ChangeStatus::UNCHANGED; |
2830 | } |
2831 | |
2832 | void trackStatistics() const override { STATS_DECLTRACK_FN_ATTR(norecurse) } |
2833 | }; |
2834 | |
2835 | /// NoRecurse attribute deduction for a call sites. |
2836 | struct AANoRecurseCallSite final |
2837 | : AACalleeToCallSite<AANoRecurse, AANoRecurseImpl> { |
2838 | AANoRecurseCallSite(const IRPosition &IRP, Attributor &A) |
2839 | : AACalleeToCallSite<AANoRecurse, AANoRecurseImpl>(IRP, A) {} |
2840 | |
2841 | /// See AbstractAttribute::trackStatistics() |
2842 | void trackStatistics() const override { STATS_DECLTRACK_CS_ATTR(norecurse); } |
2843 | }; |
2844 | } // namespace |
2845 | |
2846 | /// ------------------------ No-Convergent Attribute -------------------------- |
2847 | |
2848 | namespace { |
2849 | struct AANonConvergentImpl : public AANonConvergent { |
2850 | AANonConvergentImpl(const IRPosition &IRP, Attributor &A) |
2851 | : AANonConvergent(IRP, A) {} |
2852 | |
2853 | /// See AbstractAttribute::getAsStr() |
2854 | const std::string getAsStr(Attributor *A) const override { |
2855 | return getAssumed() ? "non-convergent" : "may-be-convergent" ; |
2856 | } |
2857 | }; |
2858 | |
2859 | struct AANonConvergentFunction final : AANonConvergentImpl { |
2860 | AANonConvergentFunction(const IRPosition &IRP, Attributor &A) |
2861 | : AANonConvergentImpl(IRP, A) {} |
2862 | |
2863 | /// See AbstractAttribute::updateImpl(...). |
2864 | ChangeStatus updateImpl(Attributor &A) override { |
2865 | // If all function calls are known to not be convergent, we are not |
2866 | // convergent. |
2867 | auto CalleeIsNotConvergent = [&](Instruction &Inst) { |
2868 | CallBase &CB = cast<CallBase>(Val&: Inst); |
2869 | auto *Callee = dyn_cast_if_present<Function>(Val: CB.getCalledOperand()); |
2870 | if (!Callee || Callee->isIntrinsic()) { |
2871 | return false; |
2872 | } |
2873 | if (Callee->isDeclaration()) { |
2874 | return !Callee->hasFnAttribute(Attribute::Convergent); |
2875 | } |
2876 | const auto *ConvergentAA = A.getAAFor<AANonConvergent>( |
2877 | QueryingAA: *this, IRP: IRPosition::function(F: *Callee), DepClass: DepClassTy::REQUIRED); |
2878 | return ConvergentAA && ConvergentAA->isAssumedNotConvergent(); |
2879 | }; |
2880 | |
2881 | bool UsedAssumedInformation = false; |
2882 | if (!A.checkForAllCallLikeInstructions(Pred: CalleeIsNotConvergent, QueryingAA: *this, |
2883 | UsedAssumedInformation)) { |
2884 | return indicatePessimisticFixpoint(); |
2885 | } |
2886 | return ChangeStatus::UNCHANGED; |
2887 | } |
2888 | |
2889 | ChangeStatus manifest(Attributor &A) override { |
2890 | if (isKnownNotConvergent() && |
2891 | A.hasAttr(getIRPosition(), Attribute::Convergent)) { |
2892 | A.removeAttrs(getIRPosition(), {Attribute::Convergent}); |
2893 | return ChangeStatus::CHANGED; |
2894 | } |
2895 | return ChangeStatus::UNCHANGED; |
2896 | } |
2897 | |
2898 | void trackStatistics() const override { STATS_DECLTRACK_FN_ATTR(convergent) } |
2899 | }; |
2900 | } // namespace |
2901 | |
2902 | /// -------------------- Undefined-Behavior Attributes ------------------------ |
2903 | |
2904 | namespace { |
2905 | struct AAUndefinedBehaviorImpl : public AAUndefinedBehavior { |
2906 | AAUndefinedBehaviorImpl(const IRPosition &IRP, Attributor &A) |
2907 | : AAUndefinedBehavior(IRP, A) {} |
2908 | |
2909 | /// See AbstractAttribute::updateImpl(...). |
2910 | // through a pointer (i.e. also branches etc.) |
2911 | ChangeStatus updateImpl(Attributor &A) override { |
2912 | const size_t UBPrevSize = KnownUBInsts.size(); |
2913 | const size_t NoUBPrevSize = AssumedNoUBInsts.size(); |
2914 | |
2915 | auto InspectMemAccessInstForUB = [&](Instruction &I) { |
2916 | // Lang ref now states volatile store is not UB, let's skip them. |
2917 | if (I.isVolatile() && I.mayWriteToMemory()) |
2918 | return true; |
2919 | |
2920 | // Skip instructions that are already saved. |
2921 | if (AssumedNoUBInsts.count(Ptr: &I) || KnownUBInsts.count(Ptr: &I)) |
2922 | return true; |
2923 | |
2924 | // If we reach here, we know we have an instruction |
2925 | // that accesses memory through a pointer operand, |
2926 | // for which getPointerOperand() should give it to us. |
2927 | Value *PtrOp = |
2928 | const_cast<Value *>(getPointerOperand(I: &I, /* AllowVolatile */ true)); |
2929 | assert(PtrOp && |
2930 | "Expected pointer operand of memory accessing instruction" ); |
2931 | |
2932 | // Either we stopped and the appropriate action was taken, |
2933 | // or we got back a simplified value to continue. |
2934 | std::optional<Value *> SimplifiedPtrOp = |
2935 | stopOnUndefOrAssumed(A, V: PtrOp, I: &I); |
2936 | if (!SimplifiedPtrOp || !*SimplifiedPtrOp) |
2937 | return true; |
2938 | const Value *PtrOpVal = *SimplifiedPtrOp; |
2939 | |
2940 | // A memory access through a pointer is considered UB |
2941 | // only if the pointer has constant null value. |
2942 | // TODO: Expand it to not only check constant values. |
2943 | if (!isa<ConstantPointerNull>(Val: PtrOpVal)) { |
2944 | AssumedNoUBInsts.insert(Ptr: &I); |
2945 | return true; |
2946 | } |
2947 | const Type *PtrTy = PtrOpVal->getType(); |
2948 | |
2949 | // Because we only consider instructions inside functions, |
2950 | // assume that a parent function exists. |
2951 | const Function *F = I.getFunction(); |
2952 | |
2953 | // A memory access using constant null pointer is only considered UB |
2954 | // if null pointer is _not_ defined for the target platform. |
2955 | if (llvm::NullPointerIsDefined(F, AS: PtrTy->getPointerAddressSpace())) |
2956 | AssumedNoUBInsts.insert(Ptr: &I); |
2957 | else |
2958 | KnownUBInsts.insert(Ptr: &I); |
2959 | return true; |
2960 | }; |
2961 | |
2962 | auto InspectBrInstForUB = [&](Instruction &I) { |
2963 | // A conditional branch instruction is considered UB if it has `undef` |
2964 | // condition. |
2965 | |
2966 | // Skip instructions that are already saved. |
2967 | if (AssumedNoUBInsts.count(Ptr: &I) || KnownUBInsts.count(Ptr: &I)) |
2968 | return true; |
2969 | |
2970 | // We know we have a branch instruction. |
2971 | auto *BrInst = cast<BranchInst>(Val: &I); |
2972 | |
2973 | // Unconditional branches are never considered UB. |
2974 | if (BrInst->isUnconditional()) |
2975 | return true; |
2976 | |
2977 | // Either we stopped and the appropriate action was taken, |
2978 | // or we got back a simplified value to continue. |
2979 | std::optional<Value *> SimplifiedCond = |
2980 | stopOnUndefOrAssumed(A, V: BrInst->getCondition(), I: BrInst); |
2981 | if (!SimplifiedCond || !*SimplifiedCond) |
2982 | return true; |
2983 | AssumedNoUBInsts.insert(Ptr: &I); |
2984 | return true; |
2985 | }; |
2986 | |
2987 | auto InspectCallSiteForUB = [&](Instruction &I) { |
2988 | // Check whether a callsite always cause UB or not |
2989 | |
2990 | // Skip instructions that are already saved. |
2991 | if (AssumedNoUBInsts.count(Ptr: &I) || KnownUBInsts.count(Ptr: &I)) |
2992 | return true; |
2993 | |
2994 | // Check nonnull and noundef argument attribute violation for each |
2995 | // callsite. |
2996 | CallBase &CB = cast<CallBase>(Val&: I); |
2997 | auto *Callee = dyn_cast_if_present<Function>(Val: CB.getCalledOperand()); |
2998 | if (!Callee) |
2999 | return true; |
3000 | for (unsigned idx = 0; idx < CB.arg_size(); idx++) { |
3001 | // If current argument is known to be simplified to null pointer and the |
3002 | // corresponding argument position is known to have nonnull attribute, |
3003 | // the argument is poison. Furthermore, if the argument is poison and |
3004 | // the position is known to have noundef attriubte, this callsite is |
3005 | // considered UB. |
3006 | if (idx >= Callee->arg_size()) |
3007 | break; |
3008 | Value *ArgVal = CB.getArgOperand(i: idx); |
3009 | if (!ArgVal) |
3010 | continue; |
3011 | // Here, we handle three cases. |
3012 | // (1) Not having a value means it is dead. (we can replace the value |
3013 | // with undef) |
3014 | // (2) Simplified to undef. The argument violate noundef attriubte. |
3015 | // (3) Simplified to null pointer where known to be nonnull. |
3016 | // The argument is a poison value and violate noundef attribute. |
3017 | IRPosition CalleeArgumentIRP = IRPosition::callsite_argument(CB, ArgNo: idx); |
3018 | bool IsKnownNoUndef; |
3019 | AA::hasAssumedIRAttr<Attribute::NoUndef>( |
3020 | A, this, CalleeArgumentIRP, DepClassTy::NONE, IsKnownNoUndef); |
3021 | if (!IsKnownNoUndef) |
3022 | continue; |
3023 | bool UsedAssumedInformation = false; |
3024 | std::optional<Value *> SimplifiedVal = |
3025 | A.getAssumedSimplified(IRP: IRPosition::value(V: *ArgVal), AA: *this, |
3026 | UsedAssumedInformation, S: AA::Interprocedural); |
3027 | if (UsedAssumedInformation) |
3028 | continue; |
3029 | if (SimplifiedVal && !*SimplifiedVal) |
3030 | return true; |
3031 | if (!SimplifiedVal || isa<UndefValue>(Val: **SimplifiedVal)) { |
3032 | KnownUBInsts.insert(Ptr: &I); |
3033 | continue; |
3034 | } |
3035 | if (!ArgVal->getType()->isPointerTy() || |
3036 | !isa<ConstantPointerNull>(Val: **SimplifiedVal)) |
3037 | continue; |
3038 | bool IsKnownNonNull; |
3039 | AA::hasAssumedIRAttr<Attribute::NonNull>( |
3040 | A, this, CalleeArgumentIRP, DepClassTy::NONE, IsKnownNonNull); |
3041 | if (IsKnownNonNull) |
3042 | KnownUBInsts.insert(Ptr: &I); |
3043 | } |
3044 | return true; |
3045 | }; |
3046 | |
3047 | auto InspectReturnInstForUB = [&](Instruction &I) { |
3048 | auto &RI = cast<ReturnInst>(Val&: I); |
3049 | // Either we stopped and the appropriate action was taken, |
3050 | // or we got back a simplified return value to continue. |
3051 | std::optional<Value *> SimplifiedRetValue = |
3052 | stopOnUndefOrAssumed(A, V: RI.getReturnValue(), I: &I); |
3053 | if (!SimplifiedRetValue || !*SimplifiedRetValue) |
3054 | return true; |
3055 | |
3056 | // Check if a return instruction always cause UB or not |
3057 | // Note: It is guaranteed that the returned position of the anchor |
3058 | // scope has noundef attribute when this is called. |
3059 | // We also ensure the return position is not "assumed dead" |
3060 | // because the returned value was then potentially simplified to |
3061 | // `undef` in AAReturnedValues without removing the `noundef` |
3062 | // attribute yet. |
3063 | |
3064 | // When the returned position has noundef attriubte, UB occurs in the |
3065 | // following cases. |
3066 | // (1) Returned value is known to be undef. |
3067 | // (2) The value is known to be a null pointer and the returned |
3068 | // position has nonnull attribute (because the returned value is |
3069 | // poison). |
3070 | if (isa<ConstantPointerNull>(Val: *SimplifiedRetValue)) { |
3071 | bool IsKnownNonNull; |
3072 | AA::hasAssumedIRAttr<Attribute::NonNull>( |
3073 | A, this, IRPosition::returned(*getAnchorScope()), DepClassTy::NONE, |
3074 | IsKnownNonNull); |
3075 | if (IsKnownNonNull) |
3076 | KnownUBInsts.insert(Ptr: &I); |
3077 | } |
3078 | |
3079 | return true; |
3080 | }; |
3081 | |
3082 | bool UsedAssumedInformation = false; |
3083 | A.checkForAllInstructions(Pred: InspectMemAccessInstForUB, QueryingAA: *this, |
3084 | Opcodes: {Instruction::Load, Instruction::Store, |
3085 | Instruction::AtomicCmpXchg, |
3086 | Instruction::AtomicRMW}, |
3087 | UsedAssumedInformation, |
3088 | /* CheckBBLivenessOnly */ true); |
3089 | A.checkForAllInstructions(Pred: InspectBrInstForUB, QueryingAA: *this, Opcodes: {Instruction::Br}, |
3090 | UsedAssumedInformation, |
3091 | /* CheckBBLivenessOnly */ true); |
3092 | A.checkForAllCallLikeInstructions(Pred: InspectCallSiteForUB, QueryingAA: *this, |
3093 | UsedAssumedInformation); |
3094 | |
3095 | // If the returned position of the anchor scope has noundef attriubte, check |
3096 | // all returned instructions. |
3097 | if (!getAnchorScope()->getReturnType()->isVoidTy()) { |
3098 | const IRPosition &ReturnIRP = IRPosition::returned(F: *getAnchorScope()); |
3099 | if (!A.isAssumedDead(IRP: ReturnIRP, QueryingAA: this, FnLivenessAA: nullptr, UsedAssumedInformation)) { |
3100 | bool IsKnownNoUndef; |
3101 | AA::hasAssumedIRAttr<Attribute::NoUndef>( |
3102 | A, this, ReturnIRP, DepClassTy::NONE, IsKnownNoUndef); |
3103 | if (IsKnownNoUndef) |
3104 | A.checkForAllInstructions(Pred: InspectReturnInstForUB, QueryingAA: *this, |
3105 | Opcodes: {Instruction::Ret}, UsedAssumedInformation, |
3106 | /* CheckBBLivenessOnly */ true); |
3107 | } |
3108 | } |
3109 | |
3110 | if (NoUBPrevSize != AssumedNoUBInsts.size() || |
3111 | UBPrevSize != KnownUBInsts.size()) |
3112 | return ChangeStatus::CHANGED; |
3113 | return ChangeStatus::UNCHANGED; |
3114 | } |
3115 | |
3116 | bool isKnownToCauseUB(Instruction *I) const override { |
3117 | return KnownUBInsts.count(Ptr: I); |
3118 | } |
3119 | |
3120 | bool isAssumedToCauseUB(Instruction *I) const override { |
3121 | // In simple words, if an instruction is not in the assumed to _not_ |
3122 | // cause UB, then it is assumed UB (that includes those |
3123 | // in the KnownUBInsts set). The rest is boilerplate |
3124 | // is to ensure that it is one of the instructions we test |
3125 | // for UB. |
3126 | |
3127 | switch (I->getOpcode()) { |
3128 | case Instruction::Load: |
3129 | case Instruction::Store: |
3130 | case Instruction::AtomicCmpXchg: |
3131 | case Instruction::AtomicRMW: |
3132 | return !AssumedNoUBInsts.count(Ptr: I); |
3133 | case Instruction::Br: { |
3134 | auto *BrInst = cast<BranchInst>(Val: I); |
3135 | if (BrInst->isUnconditional()) |
3136 | return false; |
3137 | return !AssumedNoUBInsts.count(Ptr: I); |
3138 | } break; |
3139 | default: |
3140 | return false; |
3141 | } |
3142 | return false; |
3143 | } |
3144 | |
3145 | ChangeStatus manifest(Attributor &A) override { |
3146 | if (KnownUBInsts.empty()) |
3147 | return ChangeStatus::UNCHANGED; |
3148 | for (Instruction *I : KnownUBInsts) |
3149 | A.changeToUnreachableAfterManifest(I); |
3150 | return ChangeStatus::CHANGED; |
3151 | } |
3152 | |
3153 | /// See AbstractAttribute::getAsStr() |
3154 | const std::string getAsStr(Attributor *A) const override { |
3155 | return getAssumed() ? "undefined-behavior" : "no-ub" ; |
3156 | } |
3157 | |
3158 | /// Note: The correctness of this analysis depends on the fact that the |
3159 | /// following 2 sets will stop changing after some point. |
3160 | /// "Change" here means that their size changes. |
3161 | /// The size of each set is monotonically increasing |
3162 | /// (we only add items to them) and it is upper bounded by the number of |
3163 | /// instructions in the processed function (we can never save more |
3164 | /// elements in either set than this number). Hence, at some point, |
3165 | /// they will stop increasing. |
3166 | /// Consequently, at some point, both sets will have stopped |
3167 | /// changing, effectively making the analysis reach a fixpoint. |
3168 | |
3169 | /// Note: These 2 sets are disjoint and an instruction can be considered |
3170 | /// one of 3 things: |
3171 | /// 1) Known to cause UB (AAUndefinedBehavior could prove it) and put it in |
3172 | /// the KnownUBInsts set. |
3173 | /// 2) Assumed to cause UB (in every updateImpl, AAUndefinedBehavior |
3174 | /// has a reason to assume it). |
3175 | /// 3) Assumed to not cause UB. very other instruction - AAUndefinedBehavior |
3176 | /// could not find a reason to assume or prove that it can cause UB, |
3177 | /// hence it assumes it doesn't. We have a set for these instructions |
3178 | /// so that we don't reprocess them in every update. |
3179 | /// Note however that instructions in this set may cause UB. |
3180 | |
3181 | protected: |
3182 | /// A set of all live instructions _known_ to cause UB. |
3183 | SmallPtrSet<Instruction *, 8> KnownUBInsts; |
3184 | |
3185 | private: |
3186 | /// A set of all the (live) instructions that are assumed to _not_ cause UB. |
3187 | SmallPtrSet<Instruction *, 8> AssumedNoUBInsts; |
3188 | |
3189 | // Should be called on updates in which if we're processing an instruction |
3190 | // \p I that depends on a value \p V, one of the following has to happen: |
3191 | // - If the value is assumed, then stop. |
3192 | // - If the value is known but undef, then consider it UB. |
3193 | // - Otherwise, do specific processing with the simplified value. |
3194 | // We return std::nullopt in the first 2 cases to signify that an appropriate |
3195 | // action was taken and the caller should stop. |
3196 | // Otherwise, we return the simplified value that the caller should |
3197 | // use for specific processing. |
3198 | std::optional<Value *> stopOnUndefOrAssumed(Attributor &A, Value *V, |
3199 | Instruction *I) { |
3200 | bool UsedAssumedInformation = false; |
3201 | std::optional<Value *> SimplifiedV = |
3202 | A.getAssumedSimplified(IRP: IRPosition::value(V: *V), AA: *this, |
3203 | UsedAssumedInformation, S: AA::Interprocedural); |
3204 | if (!UsedAssumedInformation) { |
3205 | // Don't depend on assumed values. |
3206 | if (!SimplifiedV) { |
3207 | // If it is known (which we tested above) but it doesn't have a value, |
3208 | // then we can assume `undef` and hence the instruction is UB. |
3209 | KnownUBInsts.insert(Ptr: I); |
3210 | return std::nullopt; |
3211 | } |
3212 | if (!*SimplifiedV) |
3213 | return nullptr; |
3214 | V = *SimplifiedV; |
3215 | } |
3216 | if (isa<UndefValue>(Val: V)) { |
3217 | KnownUBInsts.insert(Ptr: I); |
3218 | return std::nullopt; |
3219 | } |
3220 | return V; |
3221 | } |
3222 | }; |
3223 | |
3224 | struct AAUndefinedBehaviorFunction final : AAUndefinedBehaviorImpl { |
3225 | AAUndefinedBehaviorFunction(const IRPosition &IRP, Attributor &A) |
3226 | : AAUndefinedBehaviorImpl(IRP, A) {} |
3227 | |
3228 | /// See AbstractAttribute::trackStatistics() |
3229 | void trackStatistics() const override { |
3230 | STATS_DECL(UndefinedBehaviorInstruction, Instruction, |
3231 | "Number of instructions known to have UB" ); |
3232 | BUILD_STAT_NAME(UndefinedBehaviorInstruction, Instruction) += |
3233 | KnownUBInsts.size(); |
3234 | } |
3235 | }; |
3236 | } // namespace |
3237 | |
3238 | /// ------------------------ Will-Return Attributes ---------------------------- |
3239 | |
3240 | namespace { |
3241 | // Helper function that checks whether a function has any cycle which we don't |
3242 | // know if it is bounded or not. |
3243 | // Loops with maximum trip count are considered bounded, any other cycle not. |
3244 | static bool mayContainUnboundedCycle(Function &F, Attributor &A) { |
3245 | ScalarEvolution *SE = |
3246 | A.getInfoCache().getAnalysisResultForFunction<ScalarEvolutionAnalysis>(F); |
3247 | LoopInfo *LI = A.getInfoCache().getAnalysisResultForFunction<LoopAnalysis>(F); |
3248 | // If either SCEV or LoopInfo is not available for the function then we assume |
3249 | // any cycle to be unbounded cycle. |
3250 | // We use scc_iterator which uses Tarjan algorithm to find all the maximal |
3251 | // SCCs.To detect if there's a cycle, we only need to find the maximal ones. |
3252 | if (!SE || !LI) { |
3253 | for (scc_iterator<Function *> SCCI = scc_begin(G: &F); !SCCI.isAtEnd(); ++SCCI) |
3254 | if (SCCI.hasCycle()) |
3255 | return true; |
3256 | return false; |
3257 | } |
3258 | |
3259 | // If there's irreducible control, the function may contain non-loop cycles. |
3260 | if (mayContainIrreducibleControl(F, LI)) |
3261 | return true; |
3262 | |
3263 | // Any loop that does not have a max trip count is considered unbounded cycle. |
3264 | for (auto *L : LI->getLoopsInPreorder()) { |
3265 | if (!SE->getSmallConstantMaxTripCount(L)) |
3266 | return true; |
3267 | } |
3268 | return false; |
3269 | } |
3270 | |
3271 | struct AAWillReturnImpl : public AAWillReturn { |
3272 | AAWillReturnImpl(const IRPosition &IRP, Attributor &A) |
3273 | : AAWillReturn(IRP, A) {} |
3274 | |
3275 | /// See AbstractAttribute::initialize(...). |
3276 | void initialize(Attributor &A) override { |
3277 | bool IsKnown; |
3278 | assert(!AA::hasAssumedIRAttr<Attribute::WillReturn>( |
3279 | A, nullptr, getIRPosition(), DepClassTy::NONE, IsKnown)); |
3280 | (void)IsKnown; |
3281 | } |
3282 | |
3283 | /// Check for `mustprogress` and `readonly` as they imply `willreturn`. |
3284 | bool isImpliedByMustprogressAndReadonly(Attributor &A, bool KnownOnly) { |
3285 | if (!A.hasAttr(getIRPosition(), {Attribute::MustProgress})) |
3286 | return false; |
3287 | |
3288 | bool IsKnown; |
3289 | if (AA::isAssumedReadOnly(A, IRP: getIRPosition(), QueryingAA: *this, IsKnown)) |
3290 | return IsKnown || !KnownOnly; |
3291 | return false; |
3292 | } |
3293 | |
3294 | /// See AbstractAttribute::updateImpl(...). |
3295 | ChangeStatus updateImpl(Attributor &A) override { |
3296 | if (isImpliedByMustprogressAndReadonly(A, /* KnownOnly */ false)) |
3297 | return ChangeStatus::UNCHANGED; |
3298 | |
3299 | auto CheckForWillReturn = [&](Instruction &I) { |
3300 | IRPosition IPos = IRPosition::callsite_function(CB: cast<CallBase>(Val&: I)); |
3301 | bool IsKnown; |
3302 | if (AA::hasAssumedIRAttr<Attribute::WillReturn>( |
3303 | A, this, IPos, DepClassTy::REQUIRED, IsKnown)) { |
3304 | if (IsKnown) |
3305 | return true; |
3306 | } else { |
3307 | return false; |
3308 | } |
3309 | bool IsKnownNoRecurse; |
3310 | return AA::hasAssumedIRAttr<Attribute::NoRecurse>( |
3311 | A, this, IPos, DepClassTy::REQUIRED, IsKnownNoRecurse); |
3312 | }; |
3313 | |
3314 | bool UsedAssumedInformation = false; |
3315 | if (!A.checkForAllCallLikeInstructions(CheckForWillReturn, *this, |
3316 | UsedAssumedInformation)) |
3317 | return indicatePessimisticFixpoint(); |
3318 | |
3319 | return ChangeStatus::UNCHANGED; |
3320 | } |
3321 | |
3322 | /// See AbstractAttribute::getAsStr() |
3323 | const std::string getAsStr(Attributor *A) const override { |
3324 | return getAssumed() ? "willreturn" : "may-noreturn" ; |
3325 | } |
3326 | }; |
3327 | |
3328 | struct AAWillReturnFunction final : AAWillReturnImpl { |
3329 | AAWillReturnFunction(const IRPosition &IRP, Attributor &A) |
3330 | : AAWillReturnImpl(IRP, A) {} |
3331 | |
3332 | /// See AbstractAttribute::initialize(...). |
3333 | void initialize(Attributor &A) override { |
3334 | AAWillReturnImpl::initialize(A); |
3335 | |
3336 | Function *F = getAnchorScope(); |
3337 | assert(F && "Did expect an anchor function" ); |
3338 | if (F->isDeclaration() || mayContainUnboundedCycle(F&: *F, A)) |
3339 | indicatePessimisticFixpoint(); |
3340 | } |
3341 | |
3342 | /// See AbstractAttribute::trackStatistics() |
3343 | void trackStatistics() const override { STATS_DECLTRACK_FN_ATTR(willreturn) } |
3344 | }; |
3345 | |
3346 | /// WillReturn attribute deduction for a call sites. |
3347 | struct AAWillReturnCallSite final |
3348 | : AACalleeToCallSite<AAWillReturn, AAWillReturnImpl> { |
3349 | AAWillReturnCallSite(const IRPosition &IRP, Attributor &A) |
3350 | : AACalleeToCallSite<AAWillReturn, AAWillReturnImpl>(IRP, A) {} |
3351 | |
3352 | /// See AbstractAttribute::updateImpl(...). |
3353 | ChangeStatus updateImpl(Attributor &A) override { |
3354 | if (isImpliedByMustprogressAndReadonly(A, /* KnownOnly */ false)) |
3355 | return ChangeStatus::UNCHANGED; |
3356 | |
3357 | return AACalleeToCallSite::updateImpl(A); |
3358 | } |
3359 | |
3360 | /// See AbstractAttribute::trackStatistics() |
3361 | void trackStatistics() const override { STATS_DECLTRACK_CS_ATTR(willreturn); } |
3362 | }; |
3363 | } // namespace |
3364 | |
3365 | /// -------------------AAIntraFnReachability Attribute-------------------------- |
3366 | |
3367 | /// All information associated with a reachability query. This boilerplate code |
3368 | /// is used by both AAIntraFnReachability and AAInterFnReachability, with |
3369 | /// different \p ToTy values. |
3370 | template <typename ToTy> struct ReachabilityQueryInfo { |
3371 | enum class Reachable { |
3372 | No, |
3373 | Yes, |
3374 | }; |
3375 | |
3376 | /// Start here, |
3377 | const Instruction *From = nullptr; |
3378 | /// reach this place, |
3379 | const ToTy *To = nullptr; |
3380 | /// without going through any of these instructions, |
3381 | const AA::InstExclusionSetTy *ExclusionSet = nullptr; |
3382 | /// and remember if it worked: |
3383 | Reachable Result = Reachable::No; |
3384 | |
3385 | /// Precomputed hash for this RQI. |
3386 | unsigned Hash = 0; |
3387 | |
3388 | unsigned computeHashValue() const { |
3389 | assert(Hash == 0 && "Computed hash twice!" ); |
3390 | using InstSetDMI = DenseMapInfo<const AA::InstExclusionSetTy *>; |
3391 | using PairDMI = DenseMapInfo<std::pair<const Instruction *, const ToTy *>>; |
3392 | return const_cast<ReachabilityQueryInfo<ToTy> *>(this)->Hash = |
3393 | detail::combineHashValue(a: PairDMI ::getHashValue({From, To}), |
3394 | b: InstSetDMI::getHashValue(BES: ExclusionSet)); |
3395 | } |
3396 | |
3397 | ReachabilityQueryInfo(const Instruction *From, const ToTy *To) |
3398 | : From(From), To(To) {} |
3399 | |
3400 | /// Constructor replacement to ensure unique and stable sets are used for the |
3401 | /// cache. |
3402 | ReachabilityQueryInfo(Attributor &A, const Instruction &From, const ToTy &To, |
3403 | const AA::InstExclusionSetTy *ES, bool MakeUnique) |
3404 | : From(&From), To(&To), ExclusionSet(ES) { |
3405 | |
3406 | if (!ES || ES->empty()) { |
3407 | ExclusionSet = nullptr; |
3408 | } else if (MakeUnique) { |
3409 | ExclusionSet = A.getInfoCache().getOrCreateUniqueBlockExecutionSet(BES: ES); |
3410 | } |
3411 | } |
3412 | |
3413 | ReachabilityQueryInfo(const ReachabilityQueryInfo &RQI) |
3414 | : From(RQI.From), To(RQI.To), ExclusionSet(RQI.ExclusionSet) {} |
3415 | }; |
3416 | |
3417 | namespace llvm { |
3418 | template <typename ToTy> struct DenseMapInfo<ReachabilityQueryInfo<ToTy> *> { |
3419 | using InstSetDMI = DenseMapInfo<const AA::InstExclusionSetTy *>; |
3420 | using PairDMI = DenseMapInfo<std::pair<const Instruction *, const ToTy *>>; |
3421 | |
3422 | static ReachabilityQueryInfo<ToTy> EmptyKey; |
3423 | static ReachabilityQueryInfo<ToTy> TombstoneKey; |
3424 | |
3425 | static inline ReachabilityQueryInfo<ToTy> *getEmptyKey() { return &EmptyKey; } |
3426 | static inline ReachabilityQueryInfo<ToTy> *getTombstoneKey() { |
3427 | return &TombstoneKey; |
3428 | } |
3429 | static unsigned getHashValue(const ReachabilityQueryInfo<ToTy> *RQI) { |
3430 | return RQI->Hash ? RQI->Hash : RQI->computeHashValue(); |
3431 | } |
3432 | static bool isEqual(const ReachabilityQueryInfo<ToTy> *LHS, |
3433 | const ReachabilityQueryInfo<ToTy> *RHS) { |
3434 | if (!PairDMI::isEqual({LHS->From, LHS->To}, {RHS->From, RHS->To})) |
3435 | return false; |
3436 | return InstSetDMI::isEqual(LHS: LHS->ExclusionSet, RHS: RHS->ExclusionSet); |
3437 | } |
3438 | }; |
3439 | |
3440 | #define DefineKeys(ToTy) \ |
3441 | template <> \ |
3442 | ReachabilityQueryInfo<ToTy> \ |
3443 | DenseMapInfo<ReachabilityQueryInfo<ToTy> *>::EmptyKey = \ |
3444 | ReachabilityQueryInfo<ToTy>( \ |
3445 | DenseMapInfo<const Instruction *>::getEmptyKey(), \ |
3446 | DenseMapInfo<const ToTy *>::getEmptyKey()); \ |
3447 | template <> \ |
3448 | ReachabilityQueryInfo<ToTy> \ |
3449 | DenseMapInfo<ReachabilityQueryInfo<ToTy> *>::TombstoneKey = \ |
3450 | ReachabilityQueryInfo<ToTy>( \ |
3451 | DenseMapInfo<const Instruction *>::getTombstoneKey(), \ |
3452 | DenseMapInfo<const ToTy *>::getTombstoneKey()); |
3453 | |
3454 | DefineKeys(Instruction) DefineKeys(Function) |
3455 | #undef DefineKeys |
3456 | |
3457 | } // namespace llvm |
3458 | |
3459 | namespace { |
3460 | |
3461 | template <typename BaseTy, typename ToTy> |
3462 | struct CachedReachabilityAA : public BaseTy { |
3463 | using RQITy = ReachabilityQueryInfo<ToTy>; |
3464 | |
3465 | CachedReachabilityAA(const IRPosition &IRP, Attributor &A) : BaseTy(IRP, A) {} |
3466 | |
3467 | /// See AbstractAttribute::isQueryAA. |
3468 | bool isQueryAA() const override { return true; } |
3469 | |
3470 | /// See AbstractAttribute::updateImpl(...). |
3471 | ChangeStatus updateImpl(Attributor &A) override { |
3472 | ChangeStatus Changed = ChangeStatus::UNCHANGED; |
3473 | for (unsigned u = 0, e = QueryVector.size(); u < e; ++u) { |
3474 | RQITy *RQI = QueryVector[u]; |
3475 | if (RQI->Result == RQITy::Reachable::No && |
3476 | isReachableImpl(A, RQI&: *RQI, /*IsTemporaryRQI=*/false)) |
3477 | Changed = ChangeStatus::CHANGED; |
3478 | } |
3479 | return Changed; |
3480 | } |
3481 | |
3482 | virtual bool isReachableImpl(Attributor &A, RQITy &RQI, |
3483 | bool IsTemporaryRQI) = 0; |
3484 | |
3485 | bool rememberResult(Attributor &A, typename RQITy::Reachable Result, |
3486 | RQITy &RQI, bool UsedExclusionSet, bool IsTemporaryRQI) { |
3487 | RQI.Result = Result; |
3488 | |
3489 | // Remove the temporary RQI from the cache. |
3490 | if (IsTemporaryRQI) |
3491 | QueryCache.erase(&RQI); |
3492 | |
3493 | // Insert a plain RQI (w/o exclusion set) if that makes sense. Two options: |
3494 | // 1) If it is reachable, it doesn't matter if we have an exclusion set for |
3495 | // this query. 2) We did not use the exclusion set, potentially because |
3496 | // there is none. |
3497 | if (Result == RQITy::Reachable::Yes || !UsedExclusionSet) { |
3498 | RQITy PlainRQI(RQI.From, RQI.To); |
3499 | if (!QueryCache.count(&PlainRQI)) { |
3500 | RQITy *RQIPtr = new (A.Allocator) RQITy(RQI.From, RQI.To); |
3501 | RQIPtr->Result = Result; |
3502 | QueryVector.push_back(RQIPtr); |
3503 | QueryCache.insert(RQIPtr); |
3504 | } |
3505 | } |
3506 | |
3507 | // Check if we need to insert a new permanent RQI with the exclusion set. |
3508 | if (IsTemporaryRQI && Result != RQITy::Reachable::Yes && UsedExclusionSet) { |
3509 | assert((!RQI.ExclusionSet || !RQI.ExclusionSet->empty()) && |
3510 | "Did not expect empty set!" ); |
3511 | RQITy *RQIPtr = new (A.Allocator) |
3512 | RQITy(A, *RQI.From, *RQI.To, RQI.ExclusionSet, true); |
3513 | assert(RQIPtr->Result == RQITy::Reachable::No && "Already reachable?" ); |
3514 | RQIPtr->Result = Result; |
3515 | assert(!QueryCache.count(RQIPtr)); |
3516 | QueryVector.push_back(RQIPtr); |
3517 | QueryCache.insert(RQIPtr); |
3518 | } |
3519 | |
3520 | if (Result == RQITy::Reachable::No && IsTemporaryRQI) |
3521 | A.registerForUpdate(AA&: *this); |
3522 | return Result == RQITy::Reachable::Yes; |
3523 | } |
3524 | |
3525 | const std::string getAsStr(Attributor *A) const override { |
3526 | // TODO: Return the number of reachable queries. |
3527 | return "#queries(" + std::to_string(QueryVector.size()) + ")" ; |
3528 | } |
3529 | |
3530 | bool checkQueryCache(Attributor &A, RQITy &StackRQI, |
3531 | typename RQITy::Reachable &Result) { |
3532 | if (!this->getState().isValidState()) { |
3533 | Result = RQITy::Reachable::Yes; |
3534 | return true; |
3535 | } |
3536 | |
3537 | // If we have an exclusion set we might be able to find our answer by |
3538 | // ignoring it first. |
3539 | if (StackRQI.ExclusionSet) { |
3540 | RQITy PlainRQI(StackRQI.From, StackRQI.To); |
3541 | auto It = QueryCache.find(&PlainRQI); |
3542 | if (It != QueryCache.end() && (*It)->Result == RQITy::Reachable::No) { |
3543 | Result = RQITy::Reachable::No; |
3544 | return true; |
3545 | } |
3546 | } |
3547 | |
3548 | auto It = QueryCache.find(&StackRQI); |
3549 | if (It != QueryCache.end()) { |
3550 | Result = (*It)->Result; |
3551 | return true; |
3552 | } |
3553 | |
3554 | // Insert a temporary for recursive queries. We will replace it with a |
3555 | // permanent entry later. |
3556 | QueryCache.insert(&StackRQI); |
3557 | return false; |
3558 | } |
3559 | |
3560 | private: |
3561 | SmallVector<RQITy *> QueryVector; |
3562 | DenseSet<RQITy *> QueryCache; |
3563 | }; |
3564 | |
3565 | struct AAIntraFnReachabilityFunction final |
3566 | : public CachedReachabilityAA<AAIntraFnReachability, Instruction> { |
3567 | using Base = CachedReachabilityAA<AAIntraFnReachability, Instruction>; |
3568 | AAIntraFnReachabilityFunction(const IRPosition &IRP, Attributor &A) |
3569 | : Base(IRP, A) { |
3570 | DT = A.getInfoCache().getAnalysisResultForFunction<DominatorTreeAnalysis>( |
3571 | F: *IRP.getAssociatedFunction()); |
3572 | } |
3573 | |
3574 | bool isAssumedReachable( |
3575 | Attributor &A, const Instruction &From, const Instruction &To, |
3576 | const AA::InstExclusionSetTy *ExclusionSet) const override { |
3577 | auto *NonConstThis = const_cast<AAIntraFnReachabilityFunction *>(this); |
3578 | if (&From == &To) |
3579 | return true; |
3580 | |
3581 | RQITy StackRQI(A, From, To, ExclusionSet, false); |
3582 | typename RQITy::Reachable Result; |
3583 | if (!NonConstThis->checkQueryCache(A, StackRQI, Result)) |
3584 | return NonConstThis->isReachableImpl(A, RQI&: StackRQI, |
3585 | /*IsTemporaryRQI=*/true); |
3586 | return Result == RQITy::Reachable::Yes; |
3587 | } |
3588 | |
3589 | ChangeStatus updateImpl(Attributor &A) override { |
3590 | // We only depend on liveness. DeadEdges is all we care about, check if any |
3591 | // of them changed. |
3592 | auto *LivenessAA = |
3593 | A.getAAFor<AAIsDead>(QueryingAA: *this, IRP: getIRPosition(), DepClass: DepClassTy::OPTIONAL); |
3594 | if (LivenessAA && |
3595 | llvm::all_of(Range&: DeadEdges, |
3596 | P: [&](const auto &DeadEdge) { |
3597 | return LivenessAA->isEdgeDead(From: DeadEdge.first, |
3598 | To: DeadEdge.second); |
3599 | }) && |
3600 | llvm::all_of(Range&: DeadBlocks, P: [&](const BasicBlock *BB) { |
3601 | return LivenessAA->isAssumedDead(BB); |
3602 | })) { |
3603 | return ChangeStatus::UNCHANGED; |
3604 | } |
3605 | DeadEdges.clear(); |
3606 | DeadBlocks.clear(); |
3607 | return Base::updateImpl(A); |
3608 | } |
3609 | |
3610 | bool isReachableImpl(Attributor &A, RQITy &RQI, |
3611 | bool IsTemporaryRQI) override { |
3612 | const Instruction *Origin = RQI.From; |
3613 | bool UsedExclusionSet = false; |
3614 | |
3615 | auto WillReachInBlock = [&](const Instruction &From, const Instruction &To, |
3616 | const AA::InstExclusionSetTy *ExclusionSet) { |
3617 | const Instruction *IP = &From; |
3618 | while (IP && IP != &To) { |
3619 | if (ExclusionSet && IP != Origin && ExclusionSet->count(Ptr: IP)) { |
3620 | UsedExclusionSet = true; |
3621 | break; |
3622 | } |
3623 | IP = IP->getNextNode(); |
3624 | } |
3625 | return IP == &To; |
3626 | }; |
3627 | |
3628 | const BasicBlock *FromBB = RQI.From->getParent(); |
3629 | const BasicBlock *ToBB = RQI.To->getParent(); |
3630 | assert(FromBB->getParent() == ToBB->getParent() && |
3631 | "Not an intra-procedural query!" ); |
3632 | |
3633 | // Check intra-block reachability, however, other reaching paths are still |
3634 | // possible. |
3635 | if (FromBB == ToBB && |
3636 | WillReachInBlock(*RQI.From, *RQI.To, RQI.ExclusionSet)) |
3637 | return rememberResult(A, Result: RQITy::Reachable::Yes, RQI, UsedExclusionSet, |
3638 | IsTemporaryRQI); |
3639 | |
3640 | // Check if reaching the ToBB block is sufficient or if even that would not |
3641 | // ensure reaching the target. In the latter case we are done. |
3642 | if (!WillReachInBlock(ToBB->front(), *RQI.To, RQI.ExclusionSet)) |
3643 | return rememberResult(A, Result: RQITy::Reachable::No, RQI, UsedExclusionSet, |
3644 | IsTemporaryRQI); |
3645 | |
3646 | const Function *Fn = FromBB->getParent(); |
3647 | SmallPtrSet<const BasicBlock *, 16> ExclusionBlocks; |
3648 | if (RQI.ExclusionSet) |
3649 | for (auto *I : *RQI.ExclusionSet) |
3650 | if (I->getFunction() == Fn) |
3651 | ExclusionBlocks.insert(Ptr: I->getParent()); |
3652 | |
3653 | // Check if we make it out of the FromBB block at all. |
3654 | if (ExclusionBlocks.count(Ptr: FromBB) && |
3655 | !WillReachInBlock(*RQI.From, *FromBB->getTerminator(), |
3656 | RQI.ExclusionSet)) |
3657 | return rememberResult(A, Result: RQITy::Reachable::No, RQI, UsedExclusionSet: true, IsTemporaryRQI); |
3658 | |
3659 | auto *LivenessAA = |
3660 | A.getAAFor<AAIsDead>(QueryingAA: *this, IRP: getIRPosition(), DepClass: DepClassTy::OPTIONAL); |
3661 | if (LivenessAA && LivenessAA->isAssumedDead(BB: ToBB)) { |
3662 | DeadBlocks.insert(V: ToBB); |
3663 | return rememberResult(A, Result: RQITy::Reachable::No, RQI, UsedExclusionSet, |
3664 | IsTemporaryRQI); |
3665 | } |
3666 | |
3667 | SmallPtrSet<const BasicBlock *, 16> Visited; |
3668 | SmallVector<const BasicBlock *, 16> Worklist; |
3669 | Worklist.push_back(Elt: FromBB); |
3670 | |
3671 | DenseSet<std::pair<const BasicBlock *, const BasicBlock *>> LocalDeadEdges; |
3672 | while (!Worklist.empty()) { |
3673 | const BasicBlock *BB = Worklist.pop_back_val(); |
3674 | if (!Visited.insert(Ptr: BB).second) |
3675 | continue; |
3676 | for (const BasicBlock *SuccBB : successors(BB)) { |
3677 | if (LivenessAA && LivenessAA->isEdgeDead(From: BB, To: SuccBB)) { |
3678 | LocalDeadEdges.insert(V: {BB, SuccBB}); |
3679 | continue; |
3680 | } |
3681 | // We checked before if we just need to reach the ToBB block. |
3682 | if (SuccBB == ToBB) |
3683 | return rememberResult(A, Result: RQITy::Reachable::Yes, RQI, UsedExclusionSet, |
3684 | IsTemporaryRQI); |
3685 | if (DT && ExclusionBlocks.empty() && DT->dominates(A: BB, B: ToBB)) |
3686 | return rememberResult(A, Result: RQITy::Reachable::Yes, RQI, UsedExclusionSet, |
3687 | IsTemporaryRQI); |
3688 | |
3689 | if (ExclusionBlocks.count(Ptr: SuccBB)) { |
3690 | UsedExclusionSet = true; |
3691 | continue; |
3692 | } |
3693 | Worklist.push_back(Elt: SuccBB); |
3694 | } |
3695 | } |
3696 | |
3697 | DeadEdges.insert(I: LocalDeadEdges.begin(), E: LocalDeadEdges.end()); |
3698 | return rememberResult(A, Result: RQITy::Reachable::No, RQI, UsedExclusionSet, |
3699 | IsTemporaryRQI); |
3700 | } |
3701 | |
3702 | /// See AbstractAttribute::trackStatistics() |
3703 | void trackStatistics() const override {} |
3704 | |
3705 | private: |
3706 | // Set of assumed dead blocks we used in the last query. If any changes we |
3707 | // update the state. |
3708 | DenseSet<const BasicBlock *> DeadBlocks; |
3709 | |
3710 | // Set of assumed dead edges we used in the last query. If any changes we |
3711 | // update the state. |
3712 | DenseSet<std::pair<const BasicBlock *, const BasicBlock *>> DeadEdges; |
3713 | |
3714 | /// The dominator tree of the function to short-circuit reasoning. |
3715 | const DominatorTree *DT = nullptr; |
3716 | }; |
3717 | } // namespace |
3718 | |
3719 | /// ------------------------ NoAlias Argument Attribute ------------------------ |
3720 | |
3721 | bool AANoAlias::isImpliedByIR(Attributor &A, const IRPosition &IRP, |
3722 | Attribute::AttrKind ImpliedAttributeKind, |
3723 | bool IgnoreSubsumingPositions) { |
3724 | assert(ImpliedAttributeKind == Attribute::NoAlias && |
3725 | "Unexpected attribute kind" ); |
3726 | Value *Val = &IRP.getAssociatedValue(); |
3727 | if (IRP.getPositionKind() != IRP_CALL_SITE_ARGUMENT) { |
3728 | if (isa<AllocaInst>(Val)) |
3729 | return true; |
3730 | } else { |
3731 | IgnoreSubsumingPositions = true; |
3732 | } |
3733 | |
3734 | if (isa<UndefValue>(Val)) |
3735 | return true; |
3736 | |
3737 | if (isa<ConstantPointerNull>(Val) && |
3738 | !NullPointerIsDefined(F: IRP.getAnchorScope(), |
3739 | AS: Val->getType()->getPointerAddressSpace())) |
3740 | return true; |
3741 | |
3742 | if (A.hasAttr(IRP, {Attribute::ByVal, Attribute::NoAlias}, |
3743 | IgnoreSubsumingPositions, Attribute::NoAlias)) |
3744 | return true; |
3745 | |
3746 | return false; |
3747 | } |
3748 | |
3749 | namespace { |
3750 | struct AANoAliasImpl : AANoAlias { |
3751 | AANoAliasImpl(const IRPosition &IRP, Attributor &A) : AANoAlias(IRP, A) { |
3752 | assert(getAssociatedType()->isPointerTy() && |
3753 | "Noalias is a pointer attribute" ); |
3754 | } |
3755 | |
3756 | const std::string getAsStr(Attributor *A) const override { |
3757 | return getAssumed() ? "noalias" : "may-alias" ; |
3758 | } |
3759 | }; |
3760 | |
3761 | /// NoAlias attribute for a floating value. |
3762 | struct AANoAliasFloating final : AANoAliasImpl { |
3763 | AANoAliasFloating(const IRPosition &IRP, Attributor &A) |
3764 | : AANoAliasImpl(IRP, A) {} |
3765 | |
3766 | /// See AbstractAttribute::updateImpl(...). |
3767 | ChangeStatus updateImpl(Attributor &A) override { |
3768 | // TODO: Implement this. |
3769 | return indicatePessimisticFixpoint(); |
3770 | } |
3771 | |
3772 | /// See AbstractAttribute::trackStatistics() |
3773 | void trackStatistics() const override { |
3774 | STATS_DECLTRACK_FLOATING_ATTR(noalias) |
3775 | } |
3776 | }; |
3777 | |
3778 | /// NoAlias attribute for an argument. |
3779 | struct AANoAliasArgument final |
3780 | : AAArgumentFromCallSiteArguments<AANoAlias, AANoAliasImpl> { |
3781 | using Base = AAArgumentFromCallSiteArguments<AANoAlias, AANoAliasImpl>; |
3782 | AANoAliasArgument(const IRPosition &IRP, Attributor &A) : Base(IRP, A) {} |
3783 | |
3784 | /// See AbstractAttribute::update(...). |
3785 | ChangeStatus updateImpl(Attributor &A) override { |
3786 | // We have to make sure no-alias on the argument does not break |
3787 | // synchronization when this is a callback argument, see also [1] below. |
3788 | // If synchronization cannot be affected, we delegate to the base updateImpl |
3789 | // function, otherwise we give up for now. |
3790 | |
3791 | // If the function is no-sync, no-alias cannot break synchronization. |
3792 | bool IsKnownNoSycn; |
3793 | if (AA::hasAssumedIRAttr<Attribute::NoSync>( |
3794 | A, this, IRPosition::function_scope(getIRPosition()), |
3795 | DepClassTy::OPTIONAL, IsKnownNoSycn)) |
3796 | return Base::updateImpl(A); |
3797 | |
3798 | // If the argument is read-only, no-alias cannot break synchronization. |
3799 | bool IsKnown; |
3800 | if (AA::isAssumedReadOnly(A, getIRPosition(), *this, IsKnown)) |
3801 | return Base::updateImpl(A); |
3802 | |
3803 | // If the argument is never passed through callbacks, no-alias cannot break |
3804 | // synchronization. |
3805 | bool UsedAssumedInformation = false; |
3806 | if (A.checkForAllCallSites( |
3807 | [](AbstractCallSite ACS) { return !ACS.isCallbackCall(); }, *this, |
3808 | true, UsedAssumedInformation)) |
3809 | return Base::updateImpl(A); |
3810 | |
3811 | // TODO: add no-alias but make sure it doesn't break synchronization by |
3812 | // introducing fake uses. See: |
3813 | // [1] Compiler Optimizations for OpenMP, J. Doerfert and H. Finkel, |
3814 | // International Workshop on OpenMP 2018, |
3815 | // http://compilers.cs.uni-saarland.de/people/doerfert/par_opt18.pdf |
3816 | |
3817 | return indicatePessimisticFixpoint(); |
3818 | } |
3819 | |
3820 | /// See AbstractAttribute::trackStatistics() |
3821 | void trackStatistics() const override { STATS_DECLTRACK_ARG_ATTR(noalias) } |
3822 | }; |
3823 | |
3824 | struct AANoAliasCallSiteArgument final : AANoAliasImpl { |
3825 | AANoAliasCallSiteArgument(const IRPosition &IRP, Attributor &A) |
3826 | : AANoAliasImpl(IRP, A) {} |
3827 | |
3828 | /// Determine if the underlying value may alias with the call site argument |
3829 | /// \p OtherArgNo of \p ICS (= the underlying call site). |
3830 | bool mayAliasWithArgument(Attributor &A, AAResults *&AAR, |
3831 | const AAMemoryBehavior &MemBehaviorAA, |
3832 | const CallBase &CB, unsigned OtherArgNo) { |
3833 | // We do not need to worry about aliasing with the underlying IRP. |
3834 | if (this->getCalleeArgNo() == (int)OtherArgNo) |
3835 | return false; |
3836 | |
3837 | // If it is not a pointer or pointer vector we do not alias. |
3838 | const Value *ArgOp = CB.getArgOperand(i: OtherArgNo); |
3839 | if (!ArgOp->getType()->isPtrOrPtrVectorTy()) |
3840 | return false; |
3841 | |
3842 | auto *CBArgMemBehaviorAA = A.getAAFor<AAMemoryBehavior>( |
3843 | *this, IRPosition::callsite_argument(CB, ArgNo: OtherArgNo), DepClassTy::NONE); |
3844 | |
3845 | // If the argument is readnone, there is no read-write aliasing. |
3846 | if (CBArgMemBehaviorAA && CBArgMemBehaviorAA->isAssumedReadNone()) { |
3847 | A.recordDependence(FromAA: *CBArgMemBehaviorAA, ToAA: *this, DepClass: DepClassTy::OPTIONAL); |
3848 | return false; |
3849 | } |
3850 | |
3851 | // If the argument is readonly and the underlying value is readonly, there |
3852 | // is no read-write aliasing. |
3853 | bool IsReadOnly = MemBehaviorAA.isAssumedReadOnly(); |
3854 | if (CBArgMemBehaviorAA && CBArgMemBehaviorAA->isAssumedReadOnly() && |
3855 | IsReadOnly) { |
3856 | A.recordDependence(MemBehaviorAA, *this, DepClassTy::OPTIONAL); |
3857 | A.recordDependence(FromAA: *CBArgMemBehaviorAA, ToAA: *this, DepClass: DepClassTy::OPTIONAL); |
3858 | return false; |
3859 | } |
3860 | |
3861 | // We have to utilize actual alias analysis queries so we need the object. |
3862 | if (!AAR) |
3863 | AAR = A.getInfoCache().getAnalysisResultForFunction<AAManager>( |
3864 | *getAnchorScope()); |
3865 | |
3866 | // Try to rule it out at the call site. |
3867 | bool IsAliasing = !AAR || !AAR->isNoAlias(&getAssociatedValue(), ArgOp); |
3868 | LLVM_DEBUG(dbgs() << "[NoAliasCSArg] Check alias between " |
3869 | "callsite arguments: " |
3870 | << getAssociatedValue() << " " << *ArgOp << " => " |
3871 | << (IsAliasing ? "" : "no-" ) << "alias \n" ); |
3872 | |
3873 | return IsAliasing; |
3874 | } |
3875 | |
3876 | bool isKnownNoAliasDueToNoAliasPreservation( |
3877 | Attributor &A, AAResults *&AAR, const AAMemoryBehavior &MemBehaviorAA) { |
3878 | // We can deduce "noalias" if the following conditions hold. |
3879 | // (i) Associated value is assumed to be noalias in the definition. |
3880 | // (ii) Associated value is assumed to be no-capture in all the uses |
3881 | // possibly executed before this callsite. |
3882 | // (iii) There is no other pointer argument which could alias with the |
3883 | // value. |
3884 | |
3885 | auto IsDereferenceableOrNull = [&](Value *O, const DataLayout &DL) { |
3886 | const auto *DerefAA = A.getAAFor<AADereferenceable>( |
3887 | *this, IRPosition::value(V: *O), DepClassTy::OPTIONAL); |
3888 | return DerefAA ? DerefAA->getAssumedDereferenceableBytes() : 0; |
3889 | }; |
3890 | |
3891 | const IRPosition &VIRP = IRPosition::value(V: getAssociatedValue()); |
3892 | const Function *ScopeFn = VIRP.getAnchorScope(); |
3893 | // Check whether the value is captured in the scope using AANoCapture. |
3894 | // Look at CFG and check only uses possibly executed before this |
3895 | // callsite. |
3896 | auto UsePred = [&](const Use &U, bool &Follow) -> bool { |
3897 | Instruction *UserI = cast<Instruction>(Val: U.getUser()); |
3898 | |
3899 | // If UserI is the curr instruction and there is a single potential use of |
3900 | // the value in UserI we allow the use. |
3901 | // TODO: We should inspect the operands and allow those that cannot alias |
3902 | // with the value. |
3903 | if (UserI == getCtxI() && UserI->getNumOperands() == 1) |
3904 | return true; |
3905 | |
3906 | if (ScopeFn) { |
3907 | if (auto *CB = dyn_cast<CallBase>(Val: UserI)) { |
3908 | if (CB->isArgOperand(U: &U)) { |
3909 | |
3910 | unsigned ArgNo = CB->getArgOperandNo(U: &U); |
3911 | |
3912 | bool IsKnownNoCapture; |
3913 | if (AA::hasAssumedIRAttr<Attribute::NoCapture>( |
3914 | A, this, IRPosition::callsite_argument(*CB, ArgNo), |
3915 | DepClassTy::OPTIONAL, IsKnownNoCapture)) |
3916 | return true; |
3917 | } |
3918 | } |
3919 | |
3920 | if (!AA::isPotentiallyReachable( |
3921 | A, *UserI, *getCtxI(), *this, /* ExclusionSet */ nullptr, |
3922 | [ScopeFn](const Function &Fn) { return &Fn != ScopeFn; })) |
3923 | return true; |
3924 | } |
3925 | |
3926 | // TODO: We should track the capturing uses in AANoCapture but the problem |
3927 | // is CGSCC runs. For those we would need to "allow" AANoCapture for |
3928 | // a value in the module slice. |
3929 | switch (DetermineUseCaptureKind(U, IsDereferenceableOrNull)) { |
3930 | case UseCaptureKind::NO_CAPTURE: |
3931 | return true; |
3932 | case UseCaptureKind::MAY_CAPTURE: |
3933 | LLVM_DEBUG(dbgs() << "[AANoAliasCSArg] Unknown user: " << *UserI |
3934 | << "\n" ); |
3935 | return false; |
3936 | case UseCaptureKind::PASSTHROUGH: |
3937 | Follow = true; |
3938 | return true; |
3939 | } |
3940 | llvm_unreachable("unknown UseCaptureKind" ); |
3941 | }; |
3942 | |
3943 | bool IsKnownNoCapture; |
3944 | const AANoCapture *NoCaptureAA = nullptr; |
3945 | bool IsAssumedNoCapture = AA::hasAssumedIRAttr<Attribute::NoCapture>( |
3946 | A, this, VIRP, DepClassTy::NONE, IsKnownNoCapture, false, &NoCaptureAA); |
3947 | if (!IsAssumedNoCapture && |
3948 | (!NoCaptureAA || !NoCaptureAA->isAssumedNoCaptureMaybeReturned())) { |
3949 | if (!A.checkForAllUses(Pred: UsePred, QueryingAA: *this, V: getAssociatedValue())) { |
3950 | LLVM_DEBUG( |
3951 | dbgs() << "[AANoAliasCSArg] " << getAssociatedValue() |
3952 | << " cannot be noalias as it is potentially captured\n" ); |
3953 | return false; |
3954 | } |
3955 | } |
3956 | if (NoCaptureAA) |
3957 | A.recordDependence(*NoCaptureAA, *this, DepClassTy::OPTIONAL); |
3958 | |
3959 | // Check there is no other pointer argument which could alias with the |
3960 | // value passed at this call site. |
3961 | // TODO: AbstractCallSite |
3962 | const auto &CB = cast<CallBase>(getAnchorValue()); |
3963 | for (unsigned OtherArgNo = 0; OtherArgNo < CB.arg_size(); OtherArgNo++) |
3964 | if (mayAliasWithArgument(A, AAR, MemBehaviorAA, CB: CB, OtherArgNo)) |
3965 | return false; |
3966 | |
3967 | return true; |
3968 | } |
3969 | |
3970 | /// See AbstractAttribute::updateImpl(...). |
3971 | ChangeStatus updateImpl(Attributor &A) override { |
3972 | // If the argument is readnone we are done as there are no accesses via the |
3973 | // argument. |
3974 | auto *MemBehaviorAA = |
3975 | A.getAAFor<AAMemoryBehavior>(*this, getIRPosition(), DepClassTy::NONE); |
3976 | if (MemBehaviorAA && MemBehaviorAA->isAssumedReadNone()) { |
3977 | A.recordDependence(FromAA: *MemBehaviorAA, ToAA: *this, DepClass: DepClassTy::OPTIONAL); |
3978 | return ChangeStatus::UNCHANGED; |
3979 | } |
3980 | |
3981 | bool IsKnownNoAlias; |
3982 | const IRPosition &VIRP = IRPosition::value(V: getAssociatedValue()); |
3983 | if (!AA::hasAssumedIRAttr<Attribute::NoAlias>( |
3984 | A, this, VIRP, DepClassTy::REQUIRED, IsKnownNoAlias)) { |
3985 | LLVM_DEBUG(dbgs() << "[AANoAlias] " << getAssociatedValue() |
3986 | << " is not no-alias at the definition\n" ); |
3987 | return indicatePessimisticFixpoint(); |
3988 | } |
3989 | |
3990 | AAResults *AAR = nullptr; |
3991 | if (MemBehaviorAA && |
3992 | isKnownNoAliasDueToNoAliasPreservation(A, AAR, MemBehaviorAA: *MemBehaviorAA)) { |
3993 | LLVM_DEBUG( |
3994 | dbgs() << "[AANoAlias] No-Alias deduced via no-alias preservation\n" ); |
3995 | return ChangeStatus::UNCHANGED; |
3996 | } |
3997 | |
3998 | return indicatePessimisticFixpoint(); |
3999 | } |
4000 | |
4001 | /// See AbstractAttribute::trackStatistics() |
4002 | void trackStatistics() const override { STATS_DECLTRACK_CSARG_ATTR(noalias) } |
4003 | }; |
4004 | |
4005 | /// NoAlias attribute for function return value. |
4006 | struct AANoAliasReturned final : AANoAliasImpl { |
4007 | AANoAliasReturned(const IRPosition &IRP, Attributor &A) |
4008 | : AANoAliasImpl(IRP, A) {} |
4009 | |
4010 | /// See AbstractAttribute::updateImpl(...). |
4011 | ChangeStatus updateImpl(Attributor &A) override { |
4012 | |
4013 | auto CheckReturnValue = [&](Value &RV) -> bool { |
4014 | if (Constant *C = dyn_cast<Constant>(Val: &RV)) |
4015 | if (C->isNullValue() || isa<UndefValue>(Val: C)) |
4016 | return true; |
4017 | |
4018 | /// For now, we can only deduce noalias if we have call sites. |
4019 | /// FIXME: add more support. |
4020 | if (!isa<CallBase>(Val: &RV)) |
4021 | return false; |
4022 | |
4023 | const IRPosition &RVPos = IRPosition::value(V: RV); |
4024 | bool IsKnownNoAlias; |
4025 | if (!AA::hasAssumedIRAttr<Attribute::NoAlias>( |
4026 | A, this, RVPos, DepClassTy::REQUIRED, IsKnownNoAlias)) |
4027 | return false; |
4028 | |
4029 | bool IsKnownNoCapture; |
4030 | const AANoCapture *NoCaptureAA = nullptr; |
4031 | bool IsAssumedNoCapture = AA::hasAssumedIRAttr<Attribute::NoCapture>( |
4032 | A, this, RVPos, DepClassTy::REQUIRED, IsKnownNoCapture, false, |
4033 | &NoCaptureAA); |
4034 | return IsAssumedNoCapture || |
4035 | (NoCaptureAA && NoCaptureAA->isAssumedNoCaptureMaybeReturned()); |
4036 | }; |
4037 | |
4038 | if (!A.checkForAllReturnedValues(CheckReturnValue, *this)) |
4039 | return indicatePessimisticFixpoint(); |
4040 | |
4041 | return ChangeStatus::UNCHANGED; |
4042 | } |
4043 | |
4044 | /// See AbstractAttribute::trackStatistics() |
4045 | void trackStatistics() const override { STATS_DECLTRACK_FNRET_ATTR(noalias) } |
4046 | }; |
4047 | |
4048 | /// NoAlias attribute deduction for a call site return value. |
4049 | struct AANoAliasCallSiteReturned final |
4050 | : AACalleeToCallSite<AANoAlias, AANoAliasImpl> { |
4051 | AANoAliasCallSiteReturned(const IRPosition &IRP, Attributor &A) |
4052 | : AACalleeToCallSite<AANoAlias, AANoAliasImpl>(IRP, A) {} |
4053 | |
4054 | /// See AbstractAttribute::trackStatistics() |
4055 | void trackStatistics() const override { STATS_DECLTRACK_CSRET_ATTR(noalias); } |
4056 | }; |
4057 | } // namespace |
4058 | |
4059 | /// -------------------AAIsDead Function Attribute----------------------- |
4060 | |
4061 | namespace { |
4062 | struct AAIsDeadValueImpl : public AAIsDead { |
4063 | AAIsDeadValueImpl(const IRPosition &IRP, Attributor &A) : AAIsDead(IRP, A) {} |
4064 | |
4065 | /// See AAIsDead::isAssumedDead(). |
4066 | bool isAssumedDead() const override { return isAssumed(BitsEncoding: IS_DEAD); } |
4067 | |
4068 | /// See AAIsDead::isKnownDead(). |
4069 | bool isKnownDead() const override { return isKnown(BitsEncoding: IS_DEAD); } |
4070 | |
4071 | /// See AAIsDead::isAssumedDead(BasicBlock *). |
4072 | bool isAssumedDead(const BasicBlock *BB) const override { return false; } |
4073 | |
4074 | /// See AAIsDead::isKnownDead(BasicBlock *). |
4075 | bool isKnownDead(const BasicBlock *BB) const override { return false; } |
4076 | |
4077 | /// See AAIsDead::isAssumedDead(Instruction *I). |
4078 | bool isAssumedDead(const Instruction *I) const override { |
4079 | return I == getCtxI() && isAssumedDead(); |
4080 | } |
4081 | |
4082 | /// See AAIsDead::isKnownDead(Instruction *I). |
4083 | bool isKnownDead(const Instruction *I) const override { |
4084 | return isAssumedDead(I) && isKnownDead(); |
4085 | } |
4086 | |
4087 | /// See AbstractAttribute::getAsStr(). |
4088 | const std::string getAsStr(Attributor *A) const override { |
4089 | return isAssumedDead() ? "assumed-dead" : "assumed-live" ; |
4090 | } |
4091 | |
4092 | /// Check if all uses are assumed dead. |
4093 | bool areAllUsesAssumedDead(Attributor &A, Value &V) { |
4094 | // Callers might not check the type, void has no uses. |
4095 | if (V.getType()->isVoidTy() || V.use_empty()) |
4096 | return true; |
4097 | |
4098 | // If we replace a value with a constant there are no uses left afterwards. |
4099 | if (!isa<Constant>(Val: V)) { |
4100 | if (auto *I = dyn_cast<Instruction>(Val: &V)) |
4101 | if (!A.isRunOn(Fn&: *I->getFunction())) |
4102 | return false; |
4103 | bool UsedAssumedInformation = false; |
4104 | std::optional<Constant *> C = |
4105 | A.getAssumedConstant(V, AA: *this, UsedAssumedInformation); |
4106 | if (!C || *C) |
4107 | return true; |
4108 | } |
4109 | |
4110 | auto UsePred = [&](const Use &U, bool &Follow) { return false; }; |
4111 | // Explicitly set the dependence class to required because we want a long |
4112 | // chain of N dependent instructions to be considered live as soon as one is |
4113 | // without going through N update cycles. This is not required for |
4114 | // correctness. |
4115 | return A.checkForAllUses(Pred: UsePred, QueryingAA: *this, V, /* CheckBBLivenessOnly */ false, |
4116 | LivenessDepClass: DepClassTy::REQUIRED, |
4117 | /* IgnoreDroppableUses */ false); |
4118 | } |
4119 | |
4120 | /// Determine if \p I is assumed to be side-effect free. |
4121 | bool isAssumedSideEffectFree(Attributor &A, Instruction *I) { |
4122 | if (!I || wouldInstructionBeTriviallyDead(I)) |
4123 | return true; |
4124 | |
4125 | auto *CB = dyn_cast<CallBase>(Val: I); |
4126 | if (!CB || isa<IntrinsicInst>(Val: CB)) |
4127 | return false; |
4128 | |
4129 | const IRPosition &CallIRP = IRPosition::callsite_function(CB: *CB); |
4130 | |
4131 | bool IsKnownNoUnwind; |
4132 | if (!AA::hasAssumedIRAttr<Attribute::NoUnwind>( |
4133 | A, this, CallIRP, DepClassTy::OPTIONAL, IsKnownNoUnwind)) |
4134 | return false; |
4135 | |
4136 | bool IsKnown; |
4137 | return AA::isAssumedReadOnly(A, IRP: CallIRP, QueryingAA: *this, IsKnown); |
4138 | } |
4139 | }; |
4140 | |
4141 | struct AAIsDeadFloating : public AAIsDeadValueImpl { |
4142 | AAIsDeadFloating(const IRPosition &IRP, Attributor &A) |
4143 | : AAIsDeadValueImpl(IRP, A) {} |
4144 | |
4145 | /// See AbstractAttribute::initialize(...). |
4146 | void initialize(Attributor &A) override { |
4147 | AAIsDeadValueImpl::initialize(A); |
4148 | |
4149 | if (isa<UndefValue>(Val: getAssociatedValue())) { |
4150 | indicatePessimisticFixpoint(); |
4151 | return; |
4152 | } |
4153 | |
4154 | Instruction *I = dyn_cast<Instruction>(Val: &getAssociatedValue()); |
4155 | if (!isAssumedSideEffectFree(A, I)) { |
4156 | if (!isa_and_nonnull<StoreInst>(Val: I) && !isa_and_nonnull<FenceInst>(Val: I)) |
4157 | indicatePessimisticFixpoint(); |
4158 | else |
4159 | removeAssumedBits(BitsEncoding: HAS_NO_EFFECT); |
4160 | } |
4161 | } |
4162 | |
4163 | bool isDeadFence(Attributor &A, FenceInst &FI) { |
4164 | const auto *ExecDomainAA = A.lookupAAFor<AAExecutionDomain>( |
4165 | IRP: IRPosition::function(F: *FI.getFunction()), QueryingAA: *this, DepClass: DepClassTy::NONE); |
4166 | if (!ExecDomainAA || !ExecDomainAA->isNoOpFence(FI)) |
4167 | return false; |
4168 | A.recordDependence(FromAA: *ExecDomainAA, ToAA: *this, DepClass: DepClassTy::OPTIONAL); |
4169 | return true; |
4170 | } |
4171 | |
4172 | bool isDeadStore(Attributor &A, StoreInst &SI, |
4173 | SmallSetVector<Instruction *, 8> *AssumeOnlyInst = nullptr) { |
4174 | // Lang ref now states volatile store is not UB/dead, let's skip them. |
4175 | if (SI.isVolatile()) |
4176 | return false; |
4177 | |
4178 | // If we are collecting assumes to be deleted we are in the manifest stage. |
4179 | // It's problematic to collect the potential copies again now so we use the |
4180 | // cached ones. |
4181 | bool UsedAssumedInformation = false; |
4182 | if (!AssumeOnlyInst) { |
4183 | PotentialCopies.clear(); |
4184 | if (!AA::getPotentialCopiesOfStoredValue(A, SI, PotentialCopies, QueryingAA: *this, |
4185 | UsedAssumedInformation)) { |
4186 | LLVM_DEBUG( |
4187 | dbgs() |
4188 | << "[AAIsDead] Could not determine potential copies of store!\n" ); |
4189 | return false; |
4190 | } |
4191 | } |
4192 | LLVM_DEBUG(dbgs() << "[AAIsDead] Store has " << PotentialCopies.size() |
4193 | << " potential copies.\n" ); |
4194 | |
4195 | InformationCache &InfoCache = A.getInfoCache(); |
4196 | return llvm::all_of(Range&: PotentialCopies, P: [&](Value *V) { |
4197 | if (A.isAssumedDead(IRP: IRPosition::value(V: *V), QueryingAA: this, FnLivenessAA: nullptr, |
4198 | UsedAssumedInformation)) |
4199 | return true; |
4200 | if (auto *LI = dyn_cast<LoadInst>(Val: V)) { |
4201 | if (llvm::all_of(Range: LI->uses(), P: [&](const Use &U) { |
4202 | auto &UserI = cast<Instruction>(Val&: *U.getUser()); |
4203 | if (InfoCache.isOnlyUsedByAssume(I: UserI)) { |
4204 | if (AssumeOnlyInst) |
4205 | AssumeOnlyInst->insert(X: &UserI); |
4206 | return true; |
4207 | } |
4208 | return A.isAssumedDead(U, QueryingAA: this, FnLivenessAA: nullptr, UsedAssumedInformation); |
4209 | })) { |
4210 | return true; |
4211 | } |
4212 | } |
4213 | LLVM_DEBUG(dbgs() << "[AAIsDead] Potential copy " << *V |
4214 | << " is assumed live!\n" ); |
4215 | return false; |
4216 | }); |
4217 | } |
4218 | |
4219 | /// See AbstractAttribute::getAsStr(). |
4220 | const std::string getAsStr(Attributor *A) const override { |
4221 | Instruction *I = dyn_cast<Instruction>(Val: &getAssociatedValue()); |
4222 | if (isa_and_nonnull<StoreInst>(Val: I)) |
4223 | if (isValidState()) |
4224 | return "assumed-dead-store" ; |
4225 | if (isa_and_nonnull<FenceInst>(Val: I)) |
4226 | if (isValidState()) |
4227 | return "assumed-dead-fence" ; |
4228 | return AAIsDeadValueImpl::getAsStr(A); |
4229 | } |
4230 | |
4231 | /// See AbstractAttribute::updateImpl(...). |
4232 | ChangeStatus updateImpl(Attributor &A) override { |
4233 | Instruction *I = dyn_cast<Instruction>(Val: &getAssociatedValue()); |
4234 | if (auto *SI = dyn_cast_or_null<StoreInst>(Val: I)) { |
4235 | if (!isDeadStore(A, SI&: *SI)) |
4236 | return indicatePessimisticFixpoint(); |
4237 | } else if (auto *FI = dyn_cast_or_null<FenceInst>(Val: I)) { |
4238 | if (!isDeadFence(A, FI&: *FI)) |
4239 | return indicatePessimisticFixpoint(); |
4240 | } else { |
4241 | if (!isAssumedSideEffectFree(A, I)) |
4242 | return indicatePessimisticFixpoint(); |
4243 | if (!areAllUsesAssumedDead(A, V&: getAssociatedValue())) |
4244 | return indicatePessimisticFixpoint(); |
4245 | } |
4246 | return ChangeStatus::UNCHANGED; |
4247 | } |
4248 | |
4249 | bool isRemovableStore() const override { |
4250 | return isAssumed(BitsEncoding: IS_REMOVABLE) && isa<StoreInst>(Val: &getAssociatedValue()); |
4251 | } |
4252 | |
4253 | /// See AbstractAttribute::manifest(...). |
4254 | ChangeStatus manifest(Attributor &A) override { |
4255 | Value &V = getAssociatedValue(); |
4256 | if (auto *I = dyn_cast<Instruction>(Val: &V)) { |
4257 | // If we get here we basically know the users are all dead. We check if |
4258 | // isAssumedSideEffectFree returns true here again because it might not be |
4259 | // the case and only the users are dead but the instruction (=call) is |
4260 | // still needed. |
4261 | if (auto *SI = dyn_cast<StoreInst>(Val: I)) { |
4262 | SmallSetVector<Instruction *, 8> AssumeOnlyInst; |
4263 | bool IsDead = isDeadStore(A, SI&: *SI, AssumeOnlyInst: &AssumeOnlyInst); |
4264 | (void)IsDead; |
4265 | assert(IsDead && "Store was assumed to be dead!" ); |
4266 | A.deleteAfterManifest(I&: *I); |
4267 | for (size_t i = 0; i < AssumeOnlyInst.size(); ++i) { |
4268 | Instruction *AOI = AssumeOnlyInst[i]; |
4269 | for (auto *Usr : AOI->users()) |
4270 | AssumeOnlyInst.insert(X: cast<Instruction>(Val: Usr)); |
4271 | A.deleteAfterManifest(I&: *AOI); |
4272 | } |
4273 | return ChangeStatus::CHANGED; |
4274 | } |
4275 | if (auto *FI = dyn_cast<FenceInst>(Val: I)) { |
4276 | assert(isDeadFence(A, *FI)); |
4277 | A.deleteAfterManifest(I&: *FI); |
4278 | return ChangeStatus::CHANGED; |
4279 | } |
4280 | if (isAssumedSideEffectFree(A, I) && !isa<InvokeInst>(Val: I)) { |
4281 | A.deleteAfterManifest(I&: *I); |
4282 | return ChangeStatus::CHANGED; |
4283 | } |
4284 | } |
4285 | return ChangeStatus::UNCHANGED; |
4286 | } |
4287 | |
4288 | /// See AbstractAttribute::trackStatistics() |
4289 | void trackStatistics() const override { |
4290 | STATS_DECLTRACK_FLOATING_ATTR(IsDead) |
4291 | } |
4292 | |
4293 | private: |
4294 | // The potential copies of a dead store, used for deletion during manifest. |
4295 | SmallSetVector<Value *, 4> PotentialCopies; |
4296 | }; |
4297 | |
4298 | struct AAIsDeadArgument : public AAIsDeadFloating { |
4299 | AAIsDeadArgument(const IRPosition &IRP, Attributor &A) |
4300 | : AAIsDeadFloating(IRP, A) {} |
4301 | |
4302 | /// See AbstractAttribute::manifest(...). |
4303 | ChangeStatus manifest(Attributor &A) override { |
4304 | Argument &Arg = *getAssociatedArgument(); |
4305 | if (A.isValidFunctionSignatureRewrite(Arg, /* ReplacementTypes */ {})) |
4306 | if (A.registerFunctionSignatureRewrite( |
4307 | Arg, /* ReplacementTypes */ {}, |
4308 | CalleeRepairCB: Attributor::ArgumentReplacementInfo::CalleeRepairCBTy{}, |
4309 | ACSRepairCB: Attributor::ArgumentReplacementInfo::ACSRepairCBTy{})) { |
4310 | return ChangeStatus::CHANGED; |
4311 | } |
4312 | return ChangeStatus::UNCHANGED; |
4313 | } |
4314 | |
4315 | /// See AbstractAttribute::trackStatistics() |
4316 | void trackStatistics() const override { STATS_DECLTRACK_ARG_ATTR(IsDead) } |
4317 | }; |
4318 | |
4319 | struct AAIsDeadCallSiteArgument : public AAIsDeadValueImpl { |
4320 | AAIsDeadCallSiteArgument(const IRPosition &IRP, Attributor &A) |
4321 | : AAIsDeadValueImpl(IRP, A) {} |
4322 | |
4323 | /// See AbstractAttribute::initialize(...). |
4324 | void initialize(Attributor &A) override { |
4325 | AAIsDeadValueImpl::initialize(A); |
4326 | if (isa<UndefValue>(Val: getAssociatedValue())) |
4327 | indicatePessimisticFixpoint(); |
4328 | } |
4329 | |
4330 | /// See AbstractAttribute::updateImpl(...). |
4331 | ChangeStatus updateImpl(Attributor &A) override { |
4332 | // TODO: Once we have call site specific value information we can provide |
4333 | // call site specific liveness information and then it makes |
4334 | // sense to specialize attributes for call sites arguments instead of |
4335 | // redirecting requests to the callee argument. |
4336 | Argument *Arg = getAssociatedArgument(); |
4337 | if (!Arg) |
4338 | return indicatePessimisticFixpoint(); |
4339 | const IRPosition &ArgPos = IRPosition::argument(Arg: *Arg); |
4340 | auto *ArgAA = A.getAAFor<AAIsDead>(QueryingAA: *this, IRP: ArgPos, DepClass: DepClassTy::REQUIRED); |
4341 | if (!ArgAA) |
4342 | return indicatePessimisticFixpoint(); |
4343 | return clampStateAndIndicateChange(S&: getState(), R: ArgAA->getState()); |
4344 | } |
4345 | |
4346 | /// See AbstractAttribute::manifest(...). |
4347 | ChangeStatus manifest(Attributor &A) override { |
4348 | CallBase &CB = cast<CallBase>(Val&: getAnchorValue()); |
4349 | Use &U = CB.getArgOperandUse(i: getCallSiteArgNo()); |
4350 | assert(!isa<UndefValue>(U.get()) && |
4351 | "Expected undef values to be filtered out!" ); |
4352 | UndefValue &UV = *UndefValue::get(T: U->getType()); |
4353 | if (A.changeUseAfterManifest(U, NV&: UV)) |
4354 | return ChangeStatus::CHANGED; |
4355 | return ChangeStatus::UNCHANGED; |
4356 | } |
4357 | |
4358 | /// See AbstractAttribute::trackStatistics() |
4359 | void trackStatistics() const override { STATS_DECLTRACK_CSARG_ATTR(IsDead) } |
4360 | }; |
4361 | |
4362 | struct AAIsDeadCallSiteReturned : public AAIsDeadFloating { |
4363 | AAIsDeadCallSiteReturned(const IRPosition &IRP, Attributor &A) |
4364 | : AAIsDeadFloating(IRP, A) {} |
4365 | |
4366 | /// See AAIsDead::isAssumedDead(). |
4367 | bool isAssumedDead() const override { |
4368 | return AAIsDeadFloating::isAssumedDead() && IsAssumedSideEffectFree; |
4369 | } |
4370 | |
4371 | /// See AbstractAttribute::initialize(...). |
4372 | void initialize(Attributor &A) override { |
4373 | AAIsDeadFloating::initialize(A); |
4374 | if (isa<UndefValue>(Val: getAssociatedValue())) { |
4375 | indicatePessimisticFixpoint(); |
4376 | return; |
4377 | } |
4378 | |
4379 | // We track this separately as a secondary state. |
4380 | IsAssumedSideEffectFree = isAssumedSideEffectFree(A, I: getCtxI()); |
4381 | } |
4382 | |
4383 | /// See AbstractAttribute::updateImpl(...). |
4384 | ChangeStatus updateImpl(Attributor &A) override { |
4385 | ChangeStatus Changed = ChangeStatus::UNCHANGED; |
4386 | if (IsAssumedSideEffectFree && !isAssumedSideEffectFree(A, I: getCtxI())) { |
4387 | IsAssumedSideEffectFree = false; |
4388 | Changed = ChangeStatus::CHANGED; |
4389 | } |
4390 | if (!areAllUsesAssumedDead(A, V&: getAssociatedValue())) |
4391 | return indicatePessimisticFixpoint(); |
4392 | return Changed; |
4393 | } |
4394 | |
4395 | /// See AbstractAttribute::trackStatistics() |
4396 | void trackStatistics() const override { |
4397 | if (IsAssumedSideEffectFree) |
4398 | STATS_DECLTRACK_CSRET_ATTR(IsDead) |
4399 | else |
4400 | STATS_DECLTRACK_CSRET_ATTR(UnusedResult) |
4401 | } |
4402 | |
4403 | /// See AbstractAttribute::getAsStr(). |
4404 | const std::string getAsStr(Attributor *A) const override { |
4405 | return isAssumedDead() |
4406 | ? "assumed-dead" |
4407 | : (getAssumed() ? "assumed-dead-users" : "assumed-live" ); |
4408 | } |
4409 | |
4410 | private: |
4411 | bool IsAssumedSideEffectFree = true; |
4412 | }; |
4413 | |
4414 | struct AAIsDeadReturned : public AAIsDeadValueImpl { |
4415 | AAIsDeadReturned(const IRPosition &IRP, Attributor &A) |
4416 | : AAIsDeadValueImpl(IRP, A) {} |
4417 | |
4418 | /// See AbstractAttribute::updateImpl(...). |
4419 | ChangeStatus updateImpl(Attributor &A) override { |
4420 | |
4421 | bool UsedAssumedInformation = false; |
4422 | A.checkForAllInstructions(Pred: [](Instruction &) { return true; }, QueryingAA: *this, |
4423 | Opcodes: {Instruction::Ret}, UsedAssumedInformation); |
4424 | |
4425 | auto PredForCallSite = [&](AbstractCallSite ACS) { |
4426 | if (ACS.isCallbackCall() || !ACS.getInstruction()) |
4427 | return false; |
4428 | return areAllUsesAssumedDead(A, V&: *ACS.getInstruction()); |
4429 | }; |
4430 | |
4431 | if (!A.checkForAllCallSites(Pred: PredForCallSite, QueryingAA: *this, RequireAllCallSites: true, |
4432 | UsedAssumedInformation)) |
4433 | return indicatePessimisticFixpoint(); |
4434 | |
4435 | return ChangeStatus::UNCHANGED; |
4436 | } |
4437 | |
4438 | /// See AbstractAttribute::manifest(...). |
4439 | ChangeStatus manifest(Attributor &A) override { |
4440 | // TODO: Rewrite the signature to return void? |
4441 | bool AnyChange = false; |
4442 | UndefValue &UV = *UndefValue::get(T: getAssociatedFunction()->getReturnType()); |
4443 | auto RetInstPred = [&](Instruction &I) { |
4444 | ReturnInst &RI = cast<ReturnInst>(Val&: I); |
4445 | if (!isa<UndefValue>(Val: RI.getReturnValue())) |
4446 | AnyChange |= A.changeUseAfterManifest(U&: RI.getOperandUse(i: 0), NV&: UV); |
4447 | return true; |
4448 | }; |
4449 | bool UsedAssumedInformation = false; |
4450 | A.checkForAllInstructions(Pred: RetInstPred, QueryingAA: *this, Opcodes: {Instruction::Ret}, |
4451 | UsedAssumedInformation); |
4452 | return AnyChange ? ChangeStatus::CHANGED : ChangeStatus::UNCHANGED; |
4453 | } |
4454 | |
4455 | /// See AbstractAttribute::trackStatistics() |
4456 | void trackStatistics() const override { STATS_DECLTRACK_FNRET_ATTR(IsDead) } |
4457 | }; |
4458 | |
4459 | struct AAIsDeadFunction : public AAIsDead { |
4460 | AAIsDeadFunction(const IRPosition &IRP, Attributor &A) : AAIsDead(IRP, A) {} |
4461 | |
4462 | /// See AbstractAttribute::initialize(...). |
4463 | void initialize(Attributor &A) override { |
4464 | Function *F = getAnchorScope(); |
4465 | assert(F && "Did expect an anchor function" ); |
4466 | if (!isAssumedDeadInternalFunction(A)) { |
4467 | ToBeExploredFrom.insert(X: &F->getEntryBlock().front()); |
4468 | assumeLive(A, BB: F->getEntryBlock()); |
4469 | } |
4470 | } |
4471 | |
4472 | bool isAssumedDeadInternalFunction(Attributor &A) { |
4473 | if (!getAnchorScope()->hasLocalLinkage()) |
4474 | return false; |
4475 | bool UsedAssumedInformation = false; |
4476 | return A.checkForAllCallSites(Pred: [](AbstractCallSite) { return false; }, QueryingAA: *this, |
4477 | RequireAllCallSites: true, UsedAssumedInformation); |
4478 | } |
4479 | |
4480 | /// See AbstractAttribute::getAsStr(). |
4481 | const std::string getAsStr(Attributor *A) const override { |
4482 | return "Live[#BB " + std::to_string(val: AssumedLiveBlocks.size()) + "/" + |
4483 | std::to_string(val: getAnchorScope()->size()) + "][#TBEP " + |
4484 | std::to_string(val: ToBeExploredFrom.size()) + "][#KDE " + |
4485 | std::to_string(val: KnownDeadEnds.size()) + "]" ; |
4486 | } |
4487 | |
4488 | /// See AbstractAttribute::manifest(...). |
4489 | ChangeStatus manifest(Attributor &A) override { |
4490 | assert(getState().isValidState() && |
4491 | "Attempted to manifest an invalid state!" ); |
4492 | |
4493 | ChangeStatus HasChanged = ChangeStatus::UNCHANGED; |
4494 | Function &F = *getAnchorScope(); |
4495 | |
4496 | if (AssumedLiveBlocks.empty()) { |
4497 | A.deleteAfterManifest(F); |
4498 | return ChangeStatus::CHANGED; |
4499 | } |
4500 | |
4501 | // Flag to determine if we can change an invoke to a call assuming the |
4502 | // callee is nounwind. This is not possible if the personality of the |
4503 | // function allows to catch asynchronous exceptions. |
4504 | bool Invoke2CallAllowed = !mayCatchAsynchronousExceptions(F); |
4505 | |
4506 | KnownDeadEnds.set_union(ToBeExploredFrom); |
4507 | for (const Instruction *DeadEndI : KnownDeadEnds) { |
4508 | auto *CB = dyn_cast<CallBase>(Val: DeadEndI); |
4509 | if (!CB) |
4510 | continue; |
4511 | bool IsKnownNoReturn; |
4512 | bool MayReturn = !AA::hasAssumedIRAttr<Attribute::NoReturn>( |
4513 | A, this, IRPosition::callsite_function(*CB), DepClassTy::OPTIONAL, |
4514 | IsKnownNoReturn); |
4515 | if (MayReturn && (!Invoke2CallAllowed || !isa<InvokeInst>(Val: CB))) |
4516 | continue; |
4517 | |
4518 | if (auto *II = dyn_cast<InvokeInst>(Val: DeadEndI)) |
4519 | A.registerInvokeWithDeadSuccessor(II&: const_cast<InvokeInst &>(*II)); |
4520 | else |
4521 | A.changeToUnreachableAfterManifest( |
4522 | I: const_cast<Instruction *>(DeadEndI->getNextNode())); |
4523 | HasChanged = ChangeStatus::CHANGED; |
4524 | } |
4525 | |
4526 | STATS_DECL(AAIsDead, BasicBlock, "Number of dead basic blocks deleted." ); |
4527 | for (BasicBlock &BB : F) |
4528 | if (!AssumedLiveBlocks.count(V: &BB)) { |
4529 | A.deleteAfterManifest(BB); |
4530 | ++BUILD_STAT_NAME(AAIsDead, BasicBlock); |
4531 | HasChanged = ChangeStatus::CHANGED; |
4532 | } |
4533 | |
4534 | return HasChanged; |
4535 | } |
4536 | |
4537 | /// See AbstractAttribute::updateImpl(...). |
4538 | ChangeStatus updateImpl(Attributor &A) override; |
4539 | |
4540 | bool isEdgeDead(const BasicBlock *From, const BasicBlock *To) const override { |
4541 | assert(From->getParent() == getAnchorScope() && |
4542 | To->getParent() == getAnchorScope() && |
4543 | "Used AAIsDead of the wrong function" ); |
4544 | return isValidState() && !AssumedLiveEdges.count(V: std::make_pair(x&: From, y&: To)); |
4545 | } |
4546 | |
4547 | /// See AbstractAttribute::trackStatistics() |
4548 | void trackStatistics() const override {} |
4549 | |
4550 | /// Returns true if the function is assumed dead. |
4551 | bool isAssumedDead() const override { return false; } |
4552 | |
4553 | /// See AAIsDead::isKnownDead(). |
4554 | bool isKnownDead() const override { return false; } |
4555 | |
4556 | /// See AAIsDead::isAssumedDead(BasicBlock *). |
4557 | bool isAssumedDead(const BasicBlock *BB) const override { |
4558 | assert(BB->getParent() == getAnchorScope() && |
4559 | "BB must be in the same anchor scope function." ); |
4560 | |
4561 | if (!getAssumed()) |
4562 | return false; |
4563 | return !AssumedLiveBlocks.count(V: BB); |
4564 | } |
4565 | |
4566 | /// See AAIsDead::isKnownDead(BasicBlock *). |
4567 | bool isKnownDead(const BasicBlock *BB) const override { |
4568 | return getKnown() && isAssumedDead(BB); |
4569 | } |
4570 | |
4571 | /// See AAIsDead::isAssumed(Instruction *I). |
4572 | bool isAssumedDead(const Instruction *I) const override { |
4573 | assert(I->getParent()->getParent() == getAnchorScope() && |
4574 | "Instruction must be in the same anchor scope function." ); |
4575 | |
4576 | if (!getAssumed()) |
4577 | return false; |
4578 | |
4579 | // If it is not in AssumedLiveBlocks then it for sure dead. |
4580 | // Otherwise, it can still be after noreturn call in a live block. |
4581 | if (!AssumedLiveBlocks.count(V: I->getParent())) |
4582 | return true; |
4583 | |
4584 | // If it is not after a liveness barrier it is live. |
4585 | const Instruction *PrevI = I->getPrevNode(); |
4586 | while (PrevI) { |
4587 | if (KnownDeadEnds.count(key: PrevI) || ToBeExploredFrom.count(key: PrevI)) |
4588 | return true; |
4589 | PrevI = PrevI->getPrevNode(); |
4590 | } |
4591 | return false; |
4592 | } |
4593 | |
4594 | /// See AAIsDead::isKnownDead(Instruction *I). |
4595 | bool isKnownDead(const Instruction *I) const override { |
4596 | return getKnown() && isAssumedDead(I); |
4597 | } |
4598 | |
4599 | /// Assume \p BB is (partially) live now and indicate to the Attributor \p A |
4600 | /// that internal function called from \p BB should now be looked at. |
4601 | bool assumeLive(Attributor &A, const BasicBlock &BB) { |
4602 | if (!AssumedLiveBlocks.insert(V: &BB).second) |
4603 | return false; |
4604 | |
4605 | // We assume that all of BB is (probably) live now and if there are calls to |
4606 | // internal functions we will assume that those are now live as well. This |
4607 | // is a performance optimization for blocks with calls to a lot of internal |
4608 | // functions. It can however cause dead functions to be treated as live. |
4609 | for (const Instruction &I : BB) |
4610 | if (const auto *CB = dyn_cast<CallBase>(Val: &I)) |
4611 | if (auto *F = dyn_cast_if_present<Function>(Val: CB->getCalledOperand())) |
4612 | if (F->hasLocalLinkage()) |
4613 | A.markLiveInternalFunction(F: *F); |
4614 | return true; |
4615 | } |
4616 | |
4617 | /// Collection of instructions that need to be explored again, e.g., we |
4618 | /// did assume they do not transfer control to (one of their) successors. |
4619 | SmallSetVector<const Instruction *, 8> ToBeExploredFrom; |
4620 | |
4621 | /// Collection of instructions that are known to not transfer control. |
4622 | SmallSetVector<const Instruction *, 8> KnownDeadEnds; |
4623 | |
4624 | /// Collection of all assumed live edges |
4625 | DenseSet<std::pair<const BasicBlock *, const BasicBlock *>> AssumedLiveEdges; |
4626 | |
4627 | /// Collection of all assumed live BasicBlocks. |
4628 | DenseSet<const BasicBlock *> AssumedLiveBlocks; |
4629 | }; |
4630 | |
4631 | static bool |
4632 | identifyAliveSuccessors(Attributor &A, const CallBase &CB, |
4633 | AbstractAttribute &AA, |
4634 | SmallVectorImpl<const Instruction *> &AliveSuccessors) { |
4635 | const IRPosition &IPos = IRPosition::callsite_function(CB); |
4636 | |
4637 | bool IsKnownNoReturn; |
4638 | if (AA::hasAssumedIRAttr<Attribute::NoReturn>( |
4639 | A, &AA, IPos, DepClassTy::OPTIONAL, IsKnownNoReturn)) |
4640 | return !IsKnownNoReturn; |
4641 | if (CB.isTerminator()) |
4642 | AliveSuccessors.push_back(Elt: &CB.getSuccessor(Idx: 0)->front()); |
4643 | else |
4644 | AliveSuccessors.push_back(Elt: CB.getNextNode()); |
4645 | return false; |
4646 | } |
4647 | |
4648 | static bool |
4649 | identifyAliveSuccessors(Attributor &A, const InvokeInst &II, |
4650 | AbstractAttribute &AA, |
4651 | SmallVectorImpl<const Instruction *> &AliveSuccessors) { |
4652 | bool UsedAssumedInformation = |
4653 | identifyAliveSuccessors(A, CB: cast<CallBase>(Val: II), AA, AliveSuccessors); |
4654 | |
4655 | // First, determine if we can change an invoke to a call assuming the |
4656 | // callee is nounwind. This is not possible if the personality of the |
4657 | // function allows to catch asynchronous exceptions. |
4658 | if (AAIsDeadFunction::mayCatchAsynchronousExceptions(F: *II.getFunction())) { |
4659 | AliveSuccessors.push_back(Elt: &II.getUnwindDest()->front()); |
4660 | } else { |
4661 | const IRPosition &IPos = IRPosition::callsite_function(CB: II); |
4662 | |
4663 | bool IsKnownNoUnwind; |
4664 | if (AA::hasAssumedIRAttr<Attribute::NoUnwind>( |
4665 | A, &AA, IPos, DepClassTy::OPTIONAL, IsKnownNoUnwind)) { |
4666 | UsedAssumedInformation |= !IsKnownNoUnwind; |
4667 | } else { |
4668 | AliveSuccessors.push_back(Elt: &II.getUnwindDest()->front()); |
4669 | } |
4670 | } |
4671 | return UsedAssumedInformation; |
4672 | } |
4673 | |
4674 | static bool |
4675 | identifyAliveSuccessors(Attributor &A, const BranchInst &BI, |
4676 | AbstractAttribute &AA, |
4677 | SmallVectorImpl<const Instruction *> &AliveSuccessors) { |
4678 | bool UsedAssumedInformation = false; |
4679 | if (BI.getNumSuccessors() == 1) { |
4680 | AliveSuccessors.push_back(Elt: &BI.getSuccessor(i: 0)->front()); |
4681 | } else { |
4682 | std::optional<Constant *> C = |
4683 | A.getAssumedConstant(V: *BI.getCondition(), AA, UsedAssumedInformation); |
4684 | if (!C || isa_and_nonnull<UndefValue>(Val: *C)) { |
4685 | // No value yet, assume both edges are dead. |
4686 | } else if (isa_and_nonnull<ConstantInt>(Val: *C)) { |
4687 | const BasicBlock *SuccBB = |
4688 | BI.getSuccessor(i: 1 - cast<ConstantInt>(Val: *C)->getValue().getZExtValue()); |
4689 | AliveSuccessors.push_back(Elt: &SuccBB->front()); |
4690 | } else { |
4691 | AliveSuccessors.push_back(Elt: &BI.getSuccessor(i: 0)->front()); |
4692 | AliveSuccessors.push_back(Elt: &BI.getSuccessor(i: 1)->front()); |
4693 | UsedAssumedInformation = false; |
4694 | } |
4695 | } |
4696 | return UsedAssumedInformation; |
4697 | } |
4698 | |
4699 | static bool |
4700 | identifyAliveSuccessors(Attributor &A, const SwitchInst &SI, |
4701 | AbstractAttribute &AA, |
4702 | SmallVectorImpl<const Instruction *> &AliveSuccessors) { |
4703 | bool UsedAssumedInformation = false; |
4704 | SmallVector<AA::ValueAndContext> Values; |
4705 | if (!A.getAssumedSimplifiedValues(IRP: IRPosition::value(V: *SI.getCondition()), AA: &AA, |
4706 | Values, S: AA::AnyScope, |
4707 | UsedAssumedInformation)) { |
4708 | // Something went wrong, assume all successors are live. |
4709 | for (const BasicBlock *SuccBB : successors(BB: SI.getParent())) |
4710 | AliveSuccessors.push_back(Elt: &SuccBB->front()); |
4711 | return false; |
4712 | } |
4713 | |
4714 | if (Values.empty() || |
4715 | (Values.size() == 1 && |
4716 | isa_and_nonnull<UndefValue>(Val: Values.front().getValue()))) { |
4717 | // No valid value yet, assume all edges are dead. |
4718 | return UsedAssumedInformation; |
4719 | } |
4720 | |
4721 | Type &Ty = *SI.getCondition()->getType(); |
4722 | SmallPtrSet<ConstantInt *, 8> Constants; |
4723 | auto CheckForConstantInt = [&](Value *V) { |
4724 | if (auto *CI = dyn_cast_if_present<ConstantInt>(Val: AA::getWithType(V&: *V, Ty))) { |
4725 | Constants.insert(Ptr: CI); |
4726 | return true; |
4727 | } |
4728 | return false; |
4729 | }; |
4730 | |
4731 | if (!all_of(Range&: Values, P: [&](AA::ValueAndContext &VAC) { |
4732 | return CheckForConstantInt(VAC.getValue()); |
4733 | })) { |
4734 | for (const BasicBlock *SuccBB : successors(BB: SI.getParent())) |
4735 | AliveSuccessors.push_back(Elt: &SuccBB->front()); |
4736 | return UsedAssumedInformation; |
4737 | } |
4738 | |
4739 | unsigned MatchedCases = 0; |
4740 | for (const auto &CaseIt : SI.cases()) { |
4741 | if (Constants.count(Ptr: CaseIt.getCaseValue())) { |
4742 | ++MatchedCases; |
4743 | AliveSuccessors.push_back(Elt: &CaseIt.getCaseSuccessor()->front()); |
4744 | } |
4745 | } |
4746 | |
4747 | // If all potential values have been matched, we will not visit the default |
4748 | // case. |
4749 | if (MatchedCases < Constants.size()) |
4750 | AliveSuccessors.push_back(Elt: &SI.getDefaultDest()->front()); |
4751 | return UsedAssumedInformation; |
4752 | } |
4753 | |
4754 | ChangeStatus AAIsDeadFunction::updateImpl(Attributor &A) { |
4755 | ChangeStatus Change = ChangeStatus::UNCHANGED; |
4756 | |
4757 | if (AssumedLiveBlocks.empty()) { |
4758 | if (isAssumedDeadInternalFunction(A)) |
4759 | return ChangeStatus::UNCHANGED; |
4760 | |
4761 | Function *F = getAnchorScope(); |
4762 | ToBeExploredFrom.insert(X: &F->getEntryBlock().front()); |
4763 | assumeLive(A, BB: F->getEntryBlock()); |
4764 | Change = ChangeStatus::CHANGED; |
4765 | } |
4766 | |
4767 | LLVM_DEBUG(dbgs() << "[AAIsDead] Live [" << AssumedLiveBlocks.size() << "/" |
4768 | << getAnchorScope()->size() << "] BBs and " |
4769 | << ToBeExploredFrom.size() << " exploration points and " |
4770 | << KnownDeadEnds.size() << " known dead ends\n" ); |
4771 | |
4772 | // Copy and clear the list of instructions we need to explore from. It is |
4773 | // refilled with instructions the next update has to look at. |
4774 | SmallVector<const Instruction *, 8> Worklist(ToBeExploredFrom.begin(), |
4775 | ToBeExploredFrom.end()); |
4776 | decltype(ToBeExploredFrom) NewToBeExploredFrom; |
4777 | |
4778 | SmallVector<const Instruction *, 8> AliveSuccessors; |
4779 | while (!Worklist.empty()) { |
4780 | const Instruction *I = Worklist.pop_back_val(); |
4781 | LLVM_DEBUG(dbgs() << "[AAIsDead] Exploration inst: " << *I << "\n" ); |
4782 | |
4783 | // Fast forward for uninteresting instructions. We could look for UB here |
4784 | // though. |
4785 | while (!I->isTerminator() && !isa<CallBase>(Val: I)) |
4786 | I = I->getNextNode(); |
4787 | |
4788 | AliveSuccessors.clear(); |
4789 | |
4790 | bool UsedAssumedInformation = false; |
4791 | switch (I->getOpcode()) { |
4792 | // TODO: look for (assumed) UB to backwards propagate "deadness". |
4793 | default: |
4794 | assert(I->isTerminator() && |
4795 | "Expected non-terminators to be handled already!" ); |
4796 | for (const BasicBlock *SuccBB : successors(BB: I->getParent())) |
4797 | AliveSuccessors.push_back(Elt: &SuccBB->front()); |
4798 | break; |
4799 | case Instruction::Call: |
4800 | UsedAssumedInformation = identifyAliveSuccessors(A, CB: cast<CallInst>(Val: *I), |
4801 | AA&: *this, AliveSuccessors); |
4802 | break; |
4803 | case Instruction::Invoke: |
4804 | UsedAssumedInformation = identifyAliveSuccessors(A, II: cast<InvokeInst>(Val: *I), |
4805 | AA&: *this, AliveSuccessors); |
4806 | break; |
4807 | case Instruction::Br: |
4808 | UsedAssumedInformation = identifyAliveSuccessors(A, BI: cast<BranchInst>(Val: *I), |
4809 | AA&: *this, AliveSuccessors); |
4810 | break; |
4811 | case Instruction::Switch: |
4812 | UsedAssumedInformation = identifyAliveSuccessors(A, SI: cast<SwitchInst>(Val: *I), |
4813 | AA&: *this, AliveSuccessors); |
4814 | break; |
4815 | } |
4816 | |
4817 | if (UsedAssumedInformation) { |
4818 | NewToBeExploredFrom.insert(X: I); |
4819 | } else if (AliveSuccessors.empty() || |
4820 | (I->isTerminator() && |
4821 | AliveSuccessors.size() < I->getNumSuccessors())) { |
4822 | if (KnownDeadEnds.insert(X: I)) |
4823 | Change = ChangeStatus::CHANGED; |
4824 | } |
4825 | |
4826 | LLVM_DEBUG(dbgs() << "[AAIsDead] #AliveSuccessors: " |
4827 | << AliveSuccessors.size() << " UsedAssumedInformation: " |
4828 | << UsedAssumedInformation << "\n" ); |
4829 | |
4830 | for (const Instruction *AliveSuccessor : AliveSuccessors) { |
4831 | if (!I->isTerminator()) { |
4832 | assert(AliveSuccessors.size() == 1 && |
4833 | "Non-terminator expected to have a single successor!" ); |
4834 | Worklist.push_back(Elt: AliveSuccessor); |
4835 | } else { |
4836 | // record the assumed live edge |
4837 | auto Edge = std::make_pair(x: I->getParent(), y: AliveSuccessor->getParent()); |
4838 | if (AssumedLiveEdges.insert(V: Edge).second) |
4839 | Change = ChangeStatus::CHANGED; |
4840 | if (assumeLive(A, BB: *AliveSuccessor->getParent())) |
4841 | Worklist.push_back(Elt: AliveSuccessor); |
4842 | } |
4843 | } |
4844 | } |
4845 | |
4846 | // Check if the content of ToBeExploredFrom changed, ignore the order. |
4847 | if (NewToBeExploredFrom.size() != ToBeExploredFrom.size() || |
4848 | llvm::any_of(Range&: NewToBeExploredFrom, P: [&](const Instruction *I) { |
4849 | return !ToBeExploredFrom.count(key: I); |
4850 | })) { |
4851 | Change = ChangeStatus::CHANGED; |
4852 | ToBeExploredFrom = std::move(NewToBeExploredFrom); |
4853 | } |
4854 | |
4855 | // If we know everything is live there is no need to query for liveness. |
4856 | // Instead, indicating a pessimistic fixpoint will cause the state to be |
4857 | // "invalid" and all queries to be answered conservatively without lookups. |
4858 | // To be in this state we have to (1) finished the exploration and (3) not |
4859 | // discovered any non-trivial dead end and (2) not ruled unreachable code |
4860 | // dead. |
4861 | if (ToBeExploredFrom.empty() && |
4862 | getAnchorScope()->size() == AssumedLiveBlocks.size() && |
4863 | llvm::all_of(Range&: KnownDeadEnds, P: [](const Instruction *DeadEndI) { |
4864 | return DeadEndI->isTerminator() && DeadEndI->getNumSuccessors() == 0; |
4865 | })) |
4866 | return indicatePessimisticFixpoint(); |
4867 | return Change; |
4868 | } |
4869 | |
4870 | /// Liveness information for a call sites. |
4871 | struct AAIsDeadCallSite final : AAIsDeadFunction { |
4872 | AAIsDeadCallSite(const IRPosition &IRP, Attributor &A) |
4873 | : AAIsDeadFunction(IRP, A) {} |
4874 | |
4875 | /// See AbstractAttribute::initialize(...). |
4876 | void initialize(Attributor &A) override { |
4877 | // TODO: Once we have call site specific value information we can provide |
4878 | // call site specific liveness information and then it makes |
4879 | // sense to specialize attributes for call sites instead of |
4880 | // redirecting requests to the callee. |
4881 | llvm_unreachable("Abstract attributes for liveness are not " |
4882 | "supported for call sites yet!" ); |
4883 | } |
4884 | |
4885 | /// See AbstractAttribute::updateImpl(...). |
4886 | ChangeStatus updateImpl(Attributor &A) override { |
4887 | return indicatePessimisticFixpoint(); |
4888 | } |
4889 | |
4890 | /// See AbstractAttribute::trackStatistics() |
4891 | void trackStatistics() const override {} |
4892 | }; |
4893 | } // namespace |
4894 | |
4895 | /// -------------------- Dereferenceable Argument Attribute -------------------- |
4896 | |
4897 | namespace { |
4898 | struct AADereferenceableImpl : AADereferenceable { |
4899 | AADereferenceableImpl(const IRPosition &IRP, Attributor &A) |
4900 | : AADereferenceable(IRP, A) {} |
4901 | using StateType = DerefState; |
4902 | |
4903 | /// See AbstractAttribute::initialize(...). |
4904 | void initialize(Attributor &A) override { |
4905 | Value &V = *getAssociatedValue().stripPointerCasts(); |
4906 | SmallVector<Attribute, 4> Attrs; |
4907 | A.getAttrs(getIRPosition(), |
4908 | {Attribute::Dereferenceable, Attribute::DereferenceableOrNull}, |
4909 | Attrs, /* IgnoreSubsumingPositions */ false); |
4910 | for (const Attribute &Attr : Attrs) |
4911 | takeKnownDerefBytesMaximum(Attr.getValueAsInt()); |
4912 | |
4913 | // Ensure we initialize the non-null AA (if necessary). |
4914 | bool IsKnownNonNull; |
4915 | AA::hasAssumedIRAttr<Attribute::NonNull>( |
4916 | A, this, getIRPosition(), DepClassTy::OPTIONAL, IsKnownNonNull); |
4917 | |
4918 | bool CanBeNull, CanBeFreed; |
4919 | takeKnownDerefBytesMaximum(V.getPointerDereferenceableBytes( |
4920 | DL: A.getDataLayout(), CanBeNull, CanBeFreed)); |
4921 | |
4922 | if (Instruction *CtxI = getCtxI()) |
4923 | followUsesInMBEC(AA&: *this, A, S&: getState(), CtxI&: *CtxI); |
4924 | } |
4925 | |
4926 | /// See AbstractAttribute::getState() |
4927 | /// { |
4928 | StateType &getState() override { return *this; } |
4929 | const StateType &getState() const override { return *this; } |
4930 | /// } |
4931 | |
4932 | /// Helper function for collecting accessed bytes in must-be-executed-context |
4933 | void addAccessedBytesForUse(Attributor &A, const Use *U, const Instruction *I, |
4934 | DerefState &State) { |
4935 | const Value *UseV = U->get(); |
4936 | if (!UseV->getType()->isPointerTy()) |
4937 | return; |
4938 | |
4939 | std::optional<MemoryLocation> Loc = MemoryLocation::getOrNone(Inst: I); |
4940 | if (!Loc || Loc->Ptr != UseV || !Loc->Size.isPrecise() || I->isVolatile()) |
4941 | return; |
4942 | |
4943 | int64_t Offset; |
4944 | const Value *Base = GetPointerBaseWithConstantOffset( |
4945 | Ptr: Loc->Ptr, Offset, DL: A.getDataLayout(), /*AllowNonInbounds*/ true); |
4946 | if (Base && Base == &getAssociatedValue()) |
4947 | State.addAccessedBytes(Offset, Size: Loc->Size.getValue()); |
4948 | } |
4949 | |
4950 | /// See followUsesInMBEC |
4951 | bool followUseInMBEC(Attributor &A, const Use *U, const Instruction *I, |
4952 | AADereferenceable::StateType &State) { |
4953 | bool IsNonNull = false; |
4954 | bool TrackUse = false; |
4955 | int64_t DerefBytes = getKnownNonNullAndDerefBytesForUse( |
4956 | A, *this, getAssociatedValue(), U, I, IsNonNull, TrackUse); |
4957 | LLVM_DEBUG(dbgs() << "[AADereferenceable] Deref bytes: " << DerefBytes |
4958 | << " for instruction " << *I << "\n" ); |
4959 | |
4960 | addAccessedBytesForUse(A, U, I, State&: State); |
4961 | State.takeKnownDerefBytesMaximum(DerefBytes); |
4962 | return TrackUse; |
4963 | } |
4964 | |
4965 | /// See AbstractAttribute::manifest(...). |
4966 | ChangeStatus manifest(Attributor &A) override { |
4967 | ChangeStatus Change = AADereferenceable::manifest(A); |
4968 | bool IsKnownNonNull; |
4969 | bool IsAssumedNonNull = AA::hasAssumedIRAttr<Attribute::NonNull>( |
4970 | A, this, getIRPosition(), DepClassTy::NONE, IsKnownNonNull); |
4971 | if (IsAssumedNonNull && |
4972 | A.hasAttr(getIRPosition(), Attribute::DereferenceableOrNull)) { |
4973 | A.removeAttrs(getIRPosition(), {Attribute::DereferenceableOrNull}); |
4974 | return ChangeStatus::CHANGED; |
4975 | } |
4976 | return Change; |
4977 | } |
4978 | |
4979 | void getDeducedAttributes(Attributor &A, LLVMContext &Ctx, |
4980 | SmallVectorImpl<Attribute> &Attrs) const override { |
4981 | // TODO: Add *_globally support |
4982 | bool IsKnownNonNull; |
4983 | bool IsAssumedNonNull = AA::hasAssumedIRAttr<Attribute::NonNull>( |
4984 | A, this, getIRPosition(), DepClassTy::NONE, IsKnownNonNull); |
4985 | if (IsAssumedNonNull) |
4986 | Attrs.emplace_back(Attribute::getWithDereferenceableBytes( |
4987 | Context&: Ctx, Bytes: getAssumedDereferenceableBytes())); |
4988 | else |
4989 | Attrs.emplace_back(Attribute::getWithDereferenceableOrNullBytes( |
4990 | Context&: Ctx, Bytes: getAssumedDereferenceableBytes())); |
4991 | } |
4992 | |
4993 | /// See AbstractAttribute::getAsStr(). |
4994 | const std::string getAsStr(Attributor *A) const override { |
4995 | if (!getAssumedDereferenceableBytes()) |
4996 | return "unknown-dereferenceable" ; |
4997 | bool IsKnownNonNull; |
4998 | bool IsAssumedNonNull = false; |
4999 | if (A) |
5000 | IsAssumedNonNull = AA::hasAssumedIRAttr<Attribute::NonNull>( |
5001 | *A, this, getIRPosition(), DepClassTy::NONE, IsKnownNonNull); |
5002 | return std::string("dereferenceable" ) + |
5003 | (IsAssumedNonNull ? "" : "_or_null" ) + |
5004 | (isAssumedGlobal() ? "_globally" : "" ) + "<" + |
5005 | std::to_string(getKnownDereferenceableBytes()) + "-" + |
5006 | std::to_string(getAssumedDereferenceableBytes()) + ">" + |
5007 | (!A ? " [non-null is unknown]" : "" ); |
5008 | } |
5009 | }; |
5010 | |
5011 | /// Dereferenceable attribute for a floating value. |
5012 | struct AADereferenceableFloating : AADereferenceableImpl { |
5013 | AADereferenceableFloating(const IRPosition &IRP, Attributor &A) |
5014 | : AADereferenceableImpl(IRP, A) {} |
5015 | |
5016 | /// See AbstractAttribute::updateImpl(...). |
5017 | ChangeStatus updateImpl(Attributor &A) override { |
5018 | bool Stripped; |
5019 | bool UsedAssumedInformation = false; |
5020 | SmallVector<AA::ValueAndContext> Values; |
5021 | if (!A.getAssumedSimplifiedValues(IRP: getIRPosition(), AA: *this, Values, |
5022 | S: AA::AnyScope, UsedAssumedInformation)) { |
5023 | Values.push_back(Elt: {getAssociatedValue(), getCtxI()}); |
5024 | Stripped = false; |
5025 | } else { |
5026 | Stripped = Values.size() != 1 || |
5027 | Values.front().getValue() != &getAssociatedValue(); |
5028 | } |
5029 | |
5030 | const DataLayout &DL = A.getDataLayout(); |
5031 | DerefState T; |
5032 | |
5033 | auto VisitValueCB = [&](const Value &V) -> bool { |
5034 | unsigned IdxWidth = |
5035 | DL.getIndexSizeInBits(AS: V.getType()->getPointerAddressSpace()); |
5036 | APInt Offset(IdxWidth, 0); |
5037 | const Value *Base = stripAndAccumulateOffsets( |
5038 | A, *this, &V, DL, Offset, /* GetMinOffset */ false, |
5039 | /* AllowNonInbounds */ true); |
5040 | |
5041 | const auto *AA = A.getAAFor<AADereferenceable>( |
5042 | *this, IRPosition::value(V: *Base), DepClassTy::REQUIRED); |
5043 | int64_t DerefBytes = 0; |
5044 | if (!AA || (!Stripped && this == AA)) { |
5045 | // Use IR information if we did not strip anything. |
5046 | // TODO: track globally. |
5047 | bool CanBeNull, CanBeFreed; |
5048 | DerefBytes = |
5049 | Base->getPointerDereferenceableBytes(DL, CanBeNull, CanBeFreed); |
5050 | T.GlobalState.indicatePessimisticFixpoint(); |
5051 | } else { |
5052 | const DerefState &DS = AA->getState(); |
5053 | DerefBytes = DS.DerefBytesState.getAssumed(); |
5054 | T.GlobalState &= DS.GlobalState; |
5055 | } |
5056 | |
5057 | // For now we do not try to "increase" dereferenceability due to negative |
5058 | // indices as we first have to come up with code to deal with loops and |
5059 | // for overflows of the dereferenceable bytes. |
5060 | int64_t OffsetSExt = Offset.getSExtValue(); |
5061 | if (OffsetSExt < 0) |
5062 | OffsetSExt = 0; |
5063 | |
5064 | T.takeAssumedDerefBytesMinimum( |
5065 | Bytes: std::max(a: int64_t(0), b: DerefBytes - OffsetSExt)); |
5066 | |
5067 | if (this == AA) { |
5068 | if (!Stripped) { |
5069 | // If nothing was stripped IR information is all we got. |
5070 | T.takeKnownDerefBytesMaximum( |
5071 | Bytes: std::max(a: int64_t(0), b: DerefBytes - OffsetSExt)); |
5072 | T.indicatePessimisticFixpoint(); |
5073 | } else if (OffsetSExt > 0) { |
5074 | // If something was stripped but there is circular reasoning we look |
5075 | // for the offset. If it is positive we basically decrease the |
5076 | // dereferenceable bytes in a circular loop now, which will simply |
5077 | // drive them down to the known value in a very slow way which we |
5078 | // can accelerate. |
5079 | T.indicatePessimisticFixpoint(); |
5080 | } |
5081 | } |
5082 | |
5083 | return T.isValidState(); |
5084 | }; |
5085 | |
5086 | for (const auto &VAC : Values) |
5087 | if (!VisitValueCB(*VAC.getValue())) |
5088 | return indicatePessimisticFixpoint(); |
5089 | |
5090 | return clampStateAndIndicateChange(getState(), T); |
5091 | } |
5092 | |
5093 | /// See AbstractAttribute::trackStatistics() |
5094 | void trackStatistics() const override { |
5095 | STATS_DECLTRACK_FLOATING_ATTR(dereferenceable) |
5096 | } |
5097 | }; |
5098 | |
5099 | /// Dereferenceable attribute for a return value. |
5100 | struct AADereferenceableReturned final |
5101 | : AAReturnedFromReturnedValues<AADereferenceable, AADereferenceableImpl> { |
5102 | using Base = |
5103 | AAReturnedFromReturnedValues<AADereferenceable, AADereferenceableImpl>; |
5104 | AADereferenceableReturned(const IRPosition &IRP, Attributor &A) |
5105 | : Base(IRP, A) {} |
5106 | |
5107 | /// See AbstractAttribute::trackStatistics() |
5108 | void trackStatistics() const override { |
5109 | STATS_DECLTRACK_FNRET_ATTR(dereferenceable) |
5110 | } |
5111 | }; |
5112 | |
5113 | /// Dereferenceable attribute for an argument |
5114 | struct AADereferenceableArgument final |
5115 | : AAArgumentFromCallSiteArguments<AADereferenceable, |
5116 | AADereferenceableImpl> { |
5117 | using Base = |
5118 | AAArgumentFromCallSiteArguments<AADereferenceable, AADereferenceableImpl>; |
5119 | AADereferenceableArgument(const IRPosition &IRP, Attributor &A) |
5120 | : Base(IRP, A) {} |
5121 | |
5122 | /// See AbstractAttribute::trackStatistics() |
5123 | void trackStatistics() const override { |
5124 | STATS_DECLTRACK_ARG_ATTR(dereferenceable) |
5125 | } |
5126 | }; |
5127 | |
5128 | /// Dereferenceable attribute for a call site argument. |
5129 | struct AADereferenceableCallSiteArgument final : AADereferenceableFloating { |
5130 | AADereferenceableCallSiteArgument(const IRPosition &IRP, Attributor &A) |
5131 | : AADereferenceableFloating(IRP, A) {} |
5132 | |
5133 | /// See AbstractAttribute::trackStatistics() |
5134 | void trackStatistics() const override { |
5135 | STATS_DECLTRACK_CSARG_ATTR(dereferenceable) |
5136 | } |
5137 | }; |
5138 | |
5139 | /// Dereferenceable attribute deduction for a call site return value. |
5140 | struct AADereferenceableCallSiteReturned final |
5141 | : AACalleeToCallSite<AADereferenceable, AADereferenceableImpl> { |
5142 | using Base = AACalleeToCallSite<AADereferenceable, AADereferenceableImpl>; |
5143 | AADereferenceableCallSiteReturned(const IRPosition &IRP, Attributor &A) |
5144 | : Base(IRP, A) {} |
5145 | |
5146 | /// See AbstractAttribute::trackStatistics() |
5147 | void trackStatistics() const override { |
5148 | STATS_DECLTRACK_CS_ATTR(dereferenceable); |
5149 | } |
5150 | }; |
5151 | } // namespace |
5152 | |
5153 | // ------------------------ Align Argument Attribute ------------------------ |
5154 | |
5155 | namespace { |
5156 | static unsigned getKnownAlignForUse(Attributor &A, AAAlign &QueryingAA, |
5157 | Value &AssociatedValue, const Use *U, |
5158 | const Instruction *I, bool &TrackUse) { |
5159 | // We need to follow common pointer manipulation uses to the accesses they |
5160 | // feed into. |
5161 | if (isa<CastInst>(Val: I)) { |
5162 | // Follow all but ptr2int casts. |
5163 | TrackUse = !isa<PtrToIntInst>(Val: I); |
5164 | return 0; |
5165 | } |
5166 | if (auto *GEP = dyn_cast<GetElementPtrInst>(Val: I)) { |
5167 | if (GEP->hasAllConstantIndices()) |
5168 | TrackUse = true; |
5169 | return 0; |
5170 | } |
5171 | |
5172 | MaybeAlign MA; |
5173 | if (const auto *CB = dyn_cast<CallBase>(Val: I)) { |
5174 | if (CB->isBundleOperand(U) || CB->isCallee(U)) |
5175 | return 0; |
5176 | |
5177 | unsigned ArgNo = CB->getArgOperandNo(U); |
5178 | IRPosition IRP = IRPosition::callsite_argument(CB: *CB, ArgNo); |
5179 | // As long as we only use known information there is no need to track |
5180 | // dependences here. |
5181 | auto *AlignAA = A.getAAFor<AAAlign>(QueryingAA, IRP, DepClassTy::NONE); |
5182 | if (AlignAA) |
5183 | MA = MaybeAlign(AlignAA->getKnownAlign()); |
5184 | } |
5185 | |
5186 | const DataLayout &DL = A.getDataLayout(); |
5187 | const Value *UseV = U->get(); |
5188 | if (auto *SI = dyn_cast<StoreInst>(Val: I)) { |
5189 | if (SI->getPointerOperand() == UseV) |
5190 | MA = SI->getAlign(); |
5191 | } else if (auto *LI = dyn_cast<LoadInst>(Val: I)) { |
5192 | if (LI->getPointerOperand() == UseV) |
5193 | MA = LI->getAlign(); |
5194 | } else if (auto *AI = dyn_cast<AtomicRMWInst>(Val: I)) { |
5195 | if (AI->getPointerOperand() == UseV) |
5196 | MA = AI->getAlign(); |
5197 | } else if (auto *AI = dyn_cast<AtomicCmpXchgInst>(Val: I)) { |
5198 | if (AI->getPointerOperand() == UseV) |
5199 | MA = AI->getAlign(); |
5200 | } |
5201 | |
5202 | if (!MA || *MA <= QueryingAA.getKnownAlign()) |
5203 | return 0; |
5204 | |
5205 | unsigned Alignment = MA->value(); |
5206 | int64_t Offset; |
5207 | |
5208 | if (const Value *Base = GetPointerBaseWithConstantOffset(Ptr: UseV, Offset, DL)) { |
5209 | if (Base == &AssociatedValue) { |
5210 | // BasePointerAddr + Offset = Alignment * Q for some integer Q. |
5211 | // So we can say that the maximum power of two which is a divisor of |
5212 | // gcd(Offset, Alignment) is an alignment. |
5213 | |
5214 | uint32_t gcd = std::gcd(m: uint32_t(abs(x: (int32_t)Offset)), n: Alignment); |
5215 | Alignment = llvm::bit_floor(Value: gcd); |
5216 | } |
5217 | } |
5218 | |
5219 | return Alignment; |
5220 | } |
5221 | |
5222 | struct AAAlignImpl : AAAlign { |
5223 | AAAlignImpl(const IRPosition &IRP, Attributor &A) : AAAlign(IRP, A) {} |
5224 | |
5225 | /// See AbstractAttribute::initialize(...). |
5226 | void initialize(Attributor &A) override { |
5227 | SmallVector<Attribute, 4> Attrs; |
5228 | A.getAttrs(getIRPosition(), {Attribute::Alignment}, Attrs); |
5229 | for (const Attribute &Attr : Attrs) |
5230 | takeKnownMaximum(Attr.getValueAsInt()); |
5231 | |
5232 | Value &V = *getAssociatedValue().stripPointerCasts(); |
5233 | takeKnownMaximum(V.getPointerAlignment(DL: A.getDataLayout()).value()); |
5234 | |
5235 | if (Instruction *CtxI = getCtxI()) |
5236 | followUsesInMBEC(*this, A, getState(), *CtxI); |
5237 | } |
5238 | |
5239 | /// See AbstractAttribute::manifest(...). |
5240 | ChangeStatus manifest(Attributor &A) override { |
5241 | ChangeStatus LoadStoreChanged = ChangeStatus::UNCHANGED; |
5242 | |
5243 | // Check for users that allow alignment annotations. |
5244 | Value &AssociatedValue = getAssociatedValue(); |
5245 | for (const Use &U : AssociatedValue.uses()) { |
5246 | if (auto *SI = dyn_cast<StoreInst>(U.getUser())) { |
5247 | if (SI->getPointerOperand() == &AssociatedValue) |
5248 | if (SI->getAlign() < getAssumedAlign()) { |
5249 | STATS_DECLTRACK(AAAlign, Store, |
5250 | "Number of times alignment added to a store" ); |
5251 | SI->setAlignment(getAssumedAlign()); |
5252 | LoadStoreChanged = ChangeStatus::CHANGED; |
5253 | } |
5254 | } else if (auto *LI = dyn_cast<LoadInst>(U.getUser())) { |
5255 | if (LI->getPointerOperand() == &AssociatedValue) |
5256 | if (LI->getAlign() < getAssumedAlign()) { |
5257 | LI->setAlignment(getAssumedAlign()); |
5258 | STATS_DECLTRACK(AAAlign, Load, |
5259 | "Number of times alignment added to a load" ); |
5260 | LoadStoreChanged = ChangeStatus::CHANGED; |
5261 | } |
5262 | } |
5263 | } |
5264 | |
5265 | ChangeStatus Changed = AAAlign::manifest(A); |
5266 | |
5267 | Align InheritAlign = |
5268 | getAssociatedValue().getPointerAlignment(A.getDataLayout()); |
5269 | if (InheritAlign >= getAssumedAlign()) |
5270 | return LoadStoreChanged; |
5271 | return Changed | LoadStoreChanged; |
5272 | } |
5273 | |
5274 | // TODO: Provide a helper to determine the implied ABI alignment and check in |
5275 | // the existing manifest method and a new one for AAAlignImpl that value |
5276 | // to avoid making the alignment explicit if it did not improve. |
5277 | |
5278 | /// See AbstractAttribute::getDeducedAttributes |
5279 | void getDeducedAttributes(Attributor &A, LLVMContext &Ctx, |
5280 | SmallVectorImpl<Attribute> &Attrs) const override { |
5281 | if (getAssumedAlign() > 1) |
5282 | Attrs.emplace_back( |
5283 | Args: Attribute::getWithAlignment(Context&: Ctx, Alignment: Align(getAssumedAlign()))); |
5284 | } |
5285 | |
5286 | /// See followUsesInMBEC |
5287 | bool followUseInMBEC(Attributor &A, const Use *U, const Instruction *I, |
5288 | AAAlign::StateType &State) { |
5289 | bool TrackUse = false; |
5290 | |
5291 | unsigned int KnownAlign = |
5292 | getKnownAlignForUse(A, *this, getAssociatedValue(), U, I, TrackUse); |
5293 | State.takeKnownMaximum(KnownAlign); |
5294 | |
5295 | return TrackUse; |
5296 | } |
5297 | |
5298 | /// See AbstractAttribute::getAsStr(). |
5299 | const std::string getAsStr(Attributor *A) const override { |
5300 | return "align<" + std::to_string(getKnownAlign().value()) + "-" + |
5301 | std::to_string(getAssumedAlign().value()) + ">" ; |
5302 | } |
5303 | }; |
5304 | |
5305 | /// Align attribute for a floating value. |
5306 | struct AAAlignFloating : AAAlignImpl { |
5307 | AAAlignFloating(const IRPosition &IRP, Attributor &A) : AAAlignImpl(IRP, A) {} |
5308 | |
5309 | /// See AbstractAttribute::updateImpl(...). |
5310 | ChangeStatus updateImpl(Attributor &A) override { |
5311 | const DataLayout &DL = A.getDataLayout(); |
5312 | |
5313 | bool Stripped; |
5314 | bool UsedAssumedInformation = false; |
5315 | SmallVector<AA::ValueAndContext> Values; |
5316 | if (!A.getAssumedSimplifiedValues(IRP: getIRPosition(), AA: *this, Values, |
5317 | S: AA::AnyScope, UsedAssumedInformation)) { |
5318 | Values.push_back(Elt: {getAssociatedValue(), getCtxI()}); |
5319 | Stripped = false; |
5320 | } else { |
5321 | Stripped = Values.size() != 1 || |
5322 | Values.front().getValue() != &getAssociatedValue(); |
5323 | } |
5324 | |
5325 | StateType T; |
5326 | auto VisitValueCB = [&](Value &V) -> bool { |
5327 | if (isa<UndefValue>(Val: V) || isa<ConstantPointerNull>(Val: V)) |
5328 | return true; |
5329 | const auto *AA = A.getAAFor<AAAlign>(*this, IRPosition::value(V), |
5330 | DepClassTy::REQUIRED); |
5331 | if (!AA || (!Stripped && this == AA)) { |
5332 | int64_t Offset; |
5333 | unsigned Alignment = 1; |
5334 | if (const Value *Base = |
5335 | GetPointerBaseWithConstantOffset(Ptr: &V, Offset, DL)) { |
5336 | // TODO: Use AAAlign for the base too. |
5337 | Align PA = Base->getPointerAlignment(DL); |
5338 | // BasePointerAddr + Offset = Alignment * Q for some integer Q. |
5339 | // So we can say that the maximum power of two which is a divisor of |
5340 | // gcd(Offset, Alignment) is an alignment. |
5341 | |
5342 | uint32_t gcd = |
5343 | std::gcd(m: uint32_t(abs(x: (int32_t)Offset)), n: uint32_t(PA.value())); |
5344 | Alignment = llvm::bit_floor(Value: gcd); |
5345 | } else { |
5346 | Alignment = V.getPointerAlignment(DL).value(); |
5347 | } |
5348 | // Use only IR information if we did not strip anything. |
5349 | T.takeKnownMaximum(Alignment); |
5350 | T.indicatePessimisticFixpoint(); |
5351 | } else { |
5352 | // Use abstract attribute information. |
5353 | const AAAlign::StateType &DS = AA->getState(); |
5354 | T ^= DS; |
5355 | } |
5356 | return T.isValidState(); |
5357 | }; |
5358 | |
5359 | for (const auto &VAC : Values) { |
5360 | if (!VisitValueCB(*VAC.getValue())) |
5361 | return indicatePessimisticFixpoint(); |
5362 | } |
5363 | |
5364 | // TODO: If we know we visited all incoming values, thus no are assumed |
5365 | // dead, we can take the known information from the state T. |
5366 | return clampStateAndIndicateChange(getState(), T); |
5367 | } |
5368 | |
5369 | /// See AbstractAttribute::trackStatistics() |
5370 | void trackStatistics() const override { STATS_DECLTRACK_FLOATING_ATTR(align) } |
5371 | }; |
5372 | |
5373 | /// Align attribute for function return value. |
5374 | struct AAAlignReturned final |
5375 | : AAReturnedFromReturnedValues<AAAlign, AAAlignImpl> { |
5376 | using Base = AAReturnedFromReturnedValues<AAAlign, AAAlignImpl>; |
5377 | AAAlignReturned(const IRPosition &IRP, Attributor &A) : Base(IRP, A) {} |
5378 | |
5379 | /// See AbstractAttribute::trackStatistics() |
5380 | void trackStatistics() const override { STATS_DECLTRACK_FNRET_ATTR(aligned) } |
5381 | }; |
5382 | |
5383 | /// Align attribute for function argument. |
5384 | struct AAAlignArgument final |
5385 | : AAArgumentFromCallSiteArguments<AAAlign, AAAlignImpl> { |
5386 | using Base = AAArgumentFromCallSiteArguments<AAAlign, AAAlignImpl>; |
5387 | AAAlignArgument(const IRPosition &IRP, Attributor &A) : Base(IRP, A) {} |
5388 | |
5389 | /// See AbstractAttribute::manifest(...). |
5390 | ChangeStatus manifest(Attributor &A) override { |
5391 | // If the associated argument is involved in a must-tail call we give up |
5392 | // because we would need to keep the argument alignments of caller and |
5393 | // callee in-sync. Just does not seem worth the trouble right now. |
5394 | if (A.getInfoCache().isInvolvedInMustTailCall(Arg: *getAssociatedArgument())) |
5395 | return ChangeStatus::UNCHANGED; |
5396 | return Base::manifest(A); |
5397 | } |
5398 | |
5399 | /// See AbstractAttribute::trackStatistics() |
5400 | void trackStatistics() const override { STATS_DECLTRACK_ARG_ATTR(aligned) } |
5401 | }; |
5402 | |
5403 | struct AAAlignCallSiteArgument final : AAAlignFloating { |
5404 | AAAlignCallSiteArgument(const IRPosition &IRP, Attributor &A) |
5405 | : AAAlignFloating(IRP, A) {} |
5406 | |
5407 | /// See AbstractAttribute::manifest(...). |
5408 | ChangeStatus manifest(Attributor &A) override { |
5409 | // If the associated argument is involved in a must-tail call we give up |
5410 | // because we would need to keep the argument alignments of caller and |
5411 | // callee in-sync. Just does not seem worth the trouble right now. |
5412 | if (Argument *Arg = getAssociatedArgument()) |
5413 | if (A.getInfoCache().isInvolvedInMustTailCall(Arg: *Arg)) |
5414 | return ChangeStatus::UNCHANGED; |
5415 | ChangeStatus Changed = AAAlignImpl::manifest(A); |
5416 | Align InheritAlign = |
5417 | getAssociatedValue().getPointerAlignment(A.getDataLayout()); |
5418 | if (InheritAlign >= getAssumedAlign()) |
5419 | Changed = ChangeStatus::UNCHANGED; |
5420 | return Changed; |
5421 | } |
5422 | |
5423 | /// See AbstractAttribute::updateImpl(Attributor &A). |
5424 | ChangeStatus updateImpl(Attributor &A) override { |
5425 | ChangeStatus Changed = AAAlignFloating::updateImpl(A); |
5426 | if (Argument *Arg = getAssociatedArgument()) { |
5427 | // We only take known information from the argument |
5428 | // so we do not need to track a dependence. |
5429 | const auto *ArgAlignAA = A.getAAFor<AAAlign>( |
5430 | *this, IRPosition::argument(Arg: *Arg), DepClassTy::NONE); |
5431 | if (ArgAlignAA) |
5432 | takeKnownMaximum(ArgAlignAA->getKnownAlign().value()); |
5433 | } |
5434 | return Changed; |
5435 | } |
5436 | |
5437 | /// See AbstractAttribute::trackStatistics() |
5438 | void trackStatistics() const override { STATS_DECLTRACK_CSARG_ATTR(aligned) } |
5439 | }; |
5440 | |
5441 | /// Align attribute deduction for a call site return value. |
5442 | struct AAAlignCallSiteReturned final |
5443 | : AACalleeToCallSite<AAAlign, AAAlignImpl> { |
5444 | using Base = AACalleeToCallSite<AAAlign, AAAlignImpl>; |
5445 | AAAlignCallSiteReturned(const IRPosition &IRP, Attributor &A) |
5446 | : Base(IRP, A) {} |
5447 | |
5448 | /// See AbstractAttribute::trackStatistics() |
5449 | void trackStatistics() const override { STATS_DECLTRACK_CS_ATTR(align); } |
5450 | }; |
5451 | } // namespace |
5452 | |
5453 | /// ------------------ Function No-Return Attribute ---------------------------- |
5454 | namespace { |
5455 | struct AANoReturnImpl : public AANoReturn { |
5456 | AANoReturnImpl(const IRPosition &IRP, Attributor &A) : AANoReturn(IRP, A) {} |
5457 | |
5458 | /// See AbstractAttribute::initialize(...). |
5459 | void initialize(Attributor &A) override { |
5460 | bool IsKnown; |
5461 | assert(!AA::hasAssumedIRAttr<Attribute::NoReturn>( |
5462 | A, nullptr, getIRPosition(), DepClassTy::NONE, IsKnown)); |
5463 | (void)IsKnown; |
5464 | } |
5465 | |
5466 | /// See AbstractAttribute::getAsStr(). |
5467 | const std::string getAsStr(Attributor *A) const override { |
5468 | return getAssumed() ? "noreturn" : "may-return" ; |
5469 | } |
5470 | |
5471 | /// See AbstractAttribute::updateImpl(Attributor &A). |
5472 | ChangeStatus updateImpl(Attributor &A) override { |
5473 | auto CheckForNoReturn = [](Instruction &) { return false; }; |
5474 | bool UsedAssumedInformation = false; |
5475 | if (!A.checkForAllInstructions(CheckForNoReturn, *this, |
5476 | {(unsigned)Instruction::Ret}, |
5477 | UsedAssumedInformation)) |
5478 | return indicatePessimisticFixpoint(); |
5479 | return ChangeStatus::UNCHANGED; |
5480 | } |
5481 | }; |
5482 | |
5483 | struct AANoReturnFunction final : AANoReturnImpl { |
5484 | AANoReturnFunction(const IRPosition &IRP, Attributor &A) |
5485 | : AANoReturnImpl(IRP, A) {} |
5486 | |
5487 | /// See AbstractAttribute::trackStatistics() |
5488 | void trackStatistics() const override { STATS_DECLTRACK_FN_ATTR(noreturn) } |
5489 | }; |
5490 | |
5491 | /// NoReturn attribute deduction for a call sites. |
5492 | struct AANoReturnCallSite final |
5493 | : AACalleeToCallSite<AANoReturn, AANoReturnImpl> { |
5494 | AANoReturnCallSite(const IRPosition &IRP, Attributor &A) |
5495 | : AACalleeToCallSite<AANoReturn, AANoReturnImpl>(IRP, A) {} |
5496 | |
5497 | /// See AbstractAttribute::trackStatistics() |
5498 | void trackStatistics() const override { STATS_DECLTRACK_CS_ATTR(noreturn); } |
5499 | }; |
5500 | } // namespace |
5501 | |
5502 | /// ----------------------- Instance Info --------------------------------- |
5503 | |
5504 | namespace { |
5505 | /// A class to hold the state of for no-capture attributes. |
5506 | struct AAInstanceInfoImpl : public AAInstanceInfo { |
5507 | AAInstanceInfoImpl(const IRPosition &IRP, Attributor &A) |
5508 | : AAInstanceInfo(IRP, A) {} |
5509 | |
5510 | /// See AbstractAttribute::initialize(...). |
5511 | void initialize(Attributor &A) override { |
5512 | Value &V = getAssociatedValue(); |
5513 | if (auto *C = dyn_cast<Constant>(Val: &V)) { |
5514 | if (C->isThreadDependent()) |
5515 | indicatePessimisticFixpoint(); |
5516 | else |
5517 | indicateOptimisticFixpoint(); |
5518 | return; |
5519 | } |
5520 | if (auto *CB = dyn_cast<CallBase>(Val: &V)) |
5521 | if (CB->arg_size() == 0 && !CB->mayHaveSideEffects() && |
5522 | !CB->mayReadFromMemory()) { |
5523 | indicateOptimisticFixpoint(); |
5524 | return; |
5525 | } |
5526 | if (auto *I = dyn_cast<Instruction>(Val: &V)) { |
5527 | const auto *CI = |
5528 | A.getInfoCache().getAnalysisResultForFunction<CycleAnalysis>( |
5529 | F: *I->getFunction()); |
5530 | if (mayBeInCycle(CI, I, /* HeaderOnly */ false)) { |
5531 | indicatePessimisticFixpoint(); |
5532 | return; |
5533 | } |
5534 | } |
5535 | } |
5536 | |
5537 | /// See AbstractAttribute::updateImpl(...). |
5538 | ChangeStatus updateImpl(Attributor &A) override { |
5539 | ChangeStatus Changed = ChangeStatus::UNCHANGED; |
5540 | |
5541 | Value &V = getAssociatedValue(); |
5542 | const Function *Scope = nullptr; |
5543 | if (auto *I = dyn_cast<Instruction>(Val: &V)) |
5544 | Scope = I->getFunction(); |
5545 | if (auto *A = dyn_cast<Argument>(Val: &V)) { |
5546 | Scope = A->getParent(); |
5547 | if (!Scope->hasLocalLinkage()) |
5548 | return Changed; |
5549 | } |
5550 | if (!Scope) |
5551 | return indicateOptimisticFixpoint(); |
5552 | |
5553 | bool IsKnownNoRecurse; |
5554 | if (AA::hasAssumedIRAttr<Attribute::NoRecurse>( |
5555 | A, this, IRPosition::function(*Scope), DepClassTy::OPTIONAL, |
5556 | IsKnownNoRecurse)) |
5557 | return Changed; |
5558 | |
5559 | auto UsePred = [&](const Use &U, bool &Follow) { |
5560 | const Instruction *UserI = dyn_cast<Instruction>(Val: U.getUser()); |
5561 | if (!UserI || isa<GetElementPtrInst>(Val: UserI) || isa<CastInst>(Val: UserI) || |
5562 | isa<PHINode>(Val: UserI) || isa<SelectInst>(Val: UserI)) { |
5563 | Follow = true; |
5564 | return true; |
5565 | } |
5566 | if (isa<LoadInst>(Val: UserI) || isa<CmpInst>(Val: UserI) || |
5567 | (isa<StoreInst>(Val: UserI) && |
5568 | cast<StoreInst>(Val: UserI)->getValueOperand() != U.get())) |
5569 | return true; |
5570 | if (auto *CB = dyn_cast<CallBase>(Val: UserI)) { |
5571 | // This check is not guaranteeing uniqueness but for now that we cannot |
5572 | // end up with two versions of \p U thinking it was one. |
5573 | auto *Callee = dyn_cast_if_present<Function>(Val: CB->getCalledOperand()); |
5574 | if (!Callee || !Callee->hasLocalLinkage()) |
5575 | return true; |
5576 | if (!CB->isArgOperand(U: &U)) |
5577 | return false; |
5578 | const auto *ArgInstanceInfoAA = A.getAAFor<AAInstanceInfo>( |
5579 | QueryingAA: *this, IRP: IRPosition::callsite_argument(CB: *CB, ArgNo: CB->getArgOperandNo(U: &U)), |
5580 | DepClass: DepClassTy::OPTIONAL); |
5581 | if (!ArgInstanceInfoAA || |
5582 | !ArgInstanceInfoAA->isAssumedUniqueForAnalysis()) |
5583 | return false; |
5584 | // If this call base might reach the scope again we might forward the |
5585 | // argument back here. This is very conservative. |
5586 | if (AA::isPotentiallyReachable( |
5587 | A, FromI: *CB, ToFn: *Scope, QueryingAA: *this, /* ExclusionSet */ nullptr, |
5588 | GoBackwardsCB: [Scope](const Function &Fn) { return &Fn != Scope; })) |
5589 | return false; |
5590 | return true; |
5591 | } |
5592 | return false; |
5593 | }; |
5594 | |
5595 | auto EquivalentUseCB = [&](const Use &OldU, const Use &NewU) { |
5596 | if (auto *SI = dyn_cast<StoreInst>(Val: OldU.getUser())) { |
5597 | auto *Ptr = SI->getPointerOperand()->stripPointerCasts(); |
5598 | if ((isa<AllocaInst>(Val: Ptr) || isNoAliasCall(V: Ptr)) && |
5599 | AA::isDynamicallyUnique(A, QueryingAA: *this, V: *Ptr)) |
5600 | return true; |
5601 | } |
5602 | return false; |
5603 | }; |
5604 | |
5605 | if (!A.checkForAllUses(Pred: UsePred, QueryingAA: *this, V, /* CheckBBLivenessOnly */ true, |
5606 | LivenessDepClass: DepClassTy::OPTIONAL, |
5607 | /* IgnoreDroppableUses */ true, EquivalentUseCB)) |
5608 | return indicatePessimisticFixpoint(); |
5609 | |
5610 | return Changed; |
5611 | } |
5612 | |
5613 | /// See AbstractState::getAsStr(). |
5614 | const std::string getAsStr(Attributor *A) const override { |
5615 | return isAssumedUniqueForAnalysis() ? "<unique [fAa]>" : "<unknown>" ; |
5616 | } |
5617 | |
5618 | /// See AbstractAttribute::trackStatistics() |
5619 | void trackStatistics() const override {} |
5620 | }; |
5621 | |
5622 | /// InstanceInfo attribute for floating values. |
5623 | struct AAInstanceInfoFloating : AAInstanceInfoImpl { |
5624 | AAInstanceInfoFloating(const IRPosition &IRP, Attributor &A) |
5625 | : AAInstanceInfoImpl(IRP, A) {} |
5626 | }; |
5627 | |
5628 | /// NoCapture attribute for function arguments. |
5629 | struct AAInstanceInfoArgument final : AAInstanceInfoFloating { |
5630 | AAInstanceInfoArgument(const IRPosition &IRP, Attributor &A) |
5631 | : AAInstanceInfoFloating(IRP, A) {} |
5632 | }; |
5633 | |
5634 | /// InstanceInfo attribute for call site arguments. |
5635 | struct AAInstanceInfoCallSiteArgument final : AAInstanceInfoImpl { |
5636 | AAInstanceInfoCallSiteArgument(const IRPosition &IRP, Attributor &A) |
5637 | : AAInstanceInfoImpl(IRP, A) {} |
5638 | |
5639 | /// See AbstractAttribute::updateImpl(...). |
5640 | ChangeStatus updateImpl(Attributor &A) override { |
5641 | // TODO: Once we have call site specific value information we can provide |
5642 | // call site specific liveness information and then it makes |
5643 | // sense to specialize attributes for call sites arguments instead of |
5644 | // redirecting requests to the callee argument. |
5645 | Argument *Arg = getAssociatedArgument(); |
5646 | if (!Arg) |
5647 | return indicatePessimisticFixpoint(); |
5648 | const IRPosition &ArgPos = IRPosition::argument(Arg: *Arg); |
5649 | auto *ArgAA = |
5650 | A.getAAFor<AAInstanceInfo>(QueryingAA: *this, IRP: ArgPos, DepClass: DepClassTy::REQUIRED); |
5651 | if (!ArgAA) |
5652 | return indicatePessimisticFixpoint(); |
5653 | return clampStateAndIndicateChange(S&: getState(), R: ArgAA->getState()); |
5654 | } |
5655 | }; |
5656 | |
5657 | /// InstanceInfo attribute for function return value. |
5658 | struct AAInstanceInfoReturned final : AAInstanceInfoImpl { |
5659 | AAInstanceInfoReturned(const IRPosition &IRP, Attributor &A) |
5660 | : AAInstanceInfoImpl(IRP, A) { |
5661 | llvm_unreachable("InstanceInfo is not applicable to function returns!" ); |
5662 | } |
5663 | |
5664 | /// See AbstractAttribute::initialize(...). |
5665 | void initialize(Attributor &A) override { |
5666 | llvm_unreachable("InstanceInfo is not applicable to function returns!" ); |
5667 | } |
5668 | |
5669 | /// See AbstractAttribute::updateImpl(...). |
5670 | ChangeStatus updateImpl(Attributor &A) override { |
5671 | llvm_unreachable("InstanceInfo is not applicable to function returns!" ); |
5672 | } |
5673 | }; |
5674 | |
5675 | /// InstanceInfo attribute deduction for a call site return value. |
5676 | struct AAInstanceInfoCallSiteReturned final : AAInstanceInfoFloating { |
5677 | AAInstanceInfoCallSiteReturned(const IRPosition &IRP, Attributor &A) |
5678 | : AAInstanceInfoFloating(IRP, A) {} |
5679 | }; |
5680 | } // namespace |
5681 | |
5682 | /// ----------------------- Variable Capturing --------------------------------- |
5683 | bool AANoCapture::isImpliedByIR(Attributor &A, const IRPosition &IRP, |
5684 | Attribute::AttrKind ImpliedAttributeKind, |
5685 | bool IgnoreSubsumingPositions) { |
5686 | assert(ImpliedAttributeKind == Attribute::NoCapture && |
5687 | "Unexpected attribute kind" ); |
5688 | Value &V = IRP.getAssociatedValue(); |
5689 | if (!IRP.isArgumentPosition()) |
5690 | return V.use_empty(); |
5691 | |
5692 | // You cannot "capture" null in the default address space. |
5693 | if (isa<UndefValue>(Val: V) || (isa<ConstantPointerNull>(Val: V) && |
5694 | V.getType()->getPointerAddressSpace() == 0)) { |
5695 | return true; |
5696 | } |
5697 | |
5698 | if (A.hasAttr(IRP, {Attribute::NoCapture}, |
5699 | /* IgnoreSubsumingPositions */ true, Attribute::NoCapture)) |
5700 | return true; |
5701 | |
5702 | if (IRP.getPositionKind() == IRP_CALL_SITE_ARGUMENT) |
5703 | if (Argument *Arg = IRP.getAssociatedArgument()) |
5704 | if (A.hasAttr(IRPosition::argument(*Arg), |
5705 | {Attribute::NoCapture, Attribute::ByVal}, |
5706 | /* IgnoreSubsumingPositions */ true)) { |
5707 | A.manifestAttrs(IRP, |
5708 | Attribute::get(V.getContext(), Attribute::NoCapture)); |
5709 | return true; |
5710 | } |
5711 | |
5712 | if (const Function *F = IRP.getAssociatedFunction()) { |
5713 | // Check what state the associated function can actually capture. |
5714 | AANoCapture::StateType State; |
5715 | determineFunctionCaptureCapabilities(IRP, *F, State); |
5716 | if (State.isKnown(NO_CAPTURE)) { |
5717 | A.manifestAttrs(IRP, |
5718 | Attribute::get(V.getContext(), Attribute::NoCapture)); |
5719 | return true; |
5720 | } |
5721 | } |
5722 | |
5723 | return false; |
5724 | } |
5725 | |
5726 | /// Set the NOT_CAPTURED_IN_MEM and NOT_CAPTURED_IN_RET bits in \p Known |
5727 | /// depending on the ability of the function associated with \p IRP to capture |
5728 | /// state in memory and through "returning/throwing", respectively. |
5729 | void AANoCapture::determineFunctionCaptureCapabilities(const IRPosition &IRP, |
5730 | const Function &F, |
5731 | BitIntegerState &State) { |
5732 | // TODO: Once we have memory behavior attributes we should use them here. |
5733 | |
5734 | // If we know we cannot communicate or write to memory, we do not care about |
5735 | // ptr2int anymore. |
5736 | bool ReadOnly = F.onlyReadsMemory(); |
5737 | bool NoThrow = F.doesNotThrow(); |
5738 | bool IsVoidReturn = F.getReturnType()->isVoidTy(); |
5739 | if (ReadOnly && NoThrow && IsVoidReturn) { |
5740 | State.addKnownBits(NO_CAPTURE); |
5741 | return; |
5742 | } |
5743 | |
5744 | // A function cannot capture state in memory if it only reads memory, it can |
5745 | // however return/throw state and the state might be influenced by the |
5746 | // pointer value, e.g., loading from a returned pointer might reveal a bit. |
5747 | if (ReadOnly) |
5748 | State.addKnownBits(NOT_CAPTURED_IN_MEM); |
5749 | |
5750 | // A function cannot communicate state back if it does not through |
5751 | // exceptions and doesn not return values. |
5752 | if (NoThrow && IsVoidReturn) |
5753 | State.addKnownBits(NOT_CAPTURED_IN_RET); |
5754 | |
5755 | // Check existing "returned" attributes. |
5756 | int ArgNo = IRP.getCalleeArgNo(); |
5757 | if (!NoThrow || ArgNo < 0 || |
5758 | !F.getAttributes().hasAttrSomewhere(Attribute::Returned)) |
5759 | return; |
5760 | |
5761 | for (unsigned U = 0, E = F.arg_size(); U < E; ++U) |
5762 | if (F.hasParamAttribute(U, Attribute::Returned)) { |
5763 | if (U == unsigned(ArgNo)) |
5764 | State.removeAssumedBits(NOT_CAPTURED_IN_RET); |
5765 | else if (ReadOnly) |
5766 | State.addKnownBits(NO_CAPTURE); |
5767 | else |
5768 | State.addKnownBits(NOT_CAPTURED_IN_RET); |
5769 | break; |
5770 | } |
5771 | } |
5772 | |
5773 | namespace { |
5774 | /// A class to hold the state of for no-capture attributes. |
5775 | struct AANoCaptureImpl : public AANoCapture { |
5776 | AANoCaptureImpl(const IRPosition &IRP, Attributor &A) : AANoCapture(IRP, A) {} |
5777 | |
5778 | /// See AbstractAttribute::initialize(...). |
5779 | void initialize(Attributor &A) override { |
5780 | bool IsKnown; |
5781 | assert(!AA::hasAssumedIRAttr<Attribute::NoCapture>( |
5782 | A, nullptr, getIRPosition(), DepClassTy::NONE, IsKnown)); |
5783 | (void)IsKnown; |
5784 | } |
5785 | |
5786 | /// See AbstractAttribute::updateImpl(...). |
5787 | ChangeStatus updateImpl(Attributor &A) override; |
5788 | |
5789 | /// see AbstractAttribute::isAssumedNoCaptureMaybeReturned(...). |
5790 | void getDeducedAttributes(Attributor &A, LLVMContext &Ctx, |
5791 | SmallVectorImpl<Attribute> &Attrs) const override { |
5792 | if (!isAssumedNoCaptureMaybeReturned()) |
5793 | return; |
5794 | |
5795 | if (isArgumentPosition()) { |
5796 | if (isAssumedNoCapture()) |
5797 | Attrs.emplace_back(Attribute::get(Ctx, Attribute::NoCapture)); |
5798 | else if (ManifestInternal) |
5799 | Attrs.emplace_back(Args: Attribute::get(Context&: Ctx, Kind: "no-capture-maybe-returned" )); |
5800 | } |
5801 | } |
5802 | |
5803 | /// See AbstractState::getAsStr(). |
5804 | const std::string getAsStr(Attributor *A) const override { |
5805 | if (isKnownNoCapture()) |
5806 | return "known not-captured" ; |
5807 | if (isAssumedNoCapture()) |
5808 | return "assumed not-captured" ; |
5809 | if (isKnownNoCaptureMaybeReturned()) |
5810 | return "known not-captured-maybe-returned" ; |
5811 | if (isAssumedNoCaptureMaybeReturned()) |
5812 | return "assumed not-captured-maybe-returned" ; |
5813 | return "assumed-captured" ; |
5814 | } |
5815 | |
5816 | /// Check the use \p U and update \p State accordingly. Return true if we |
5817 | /// should continue to update the state. |
5818 | bool checkUse(Attributor &A, AANoCapture::StateType &State, const Use &U, |
5819 | bool &Follow) { |
5820 | Instruction *UInst = cast<Instruction>(Val: U.getUser()); |
5821 | LLVM_DEBUG(dbgs() << "[AANoCapture] Check use: " << *U.get() << " in " |
5822 | << *UInst << "\n" ); |
5823 | |
5824 | // Deal with ptr2int by following uses. |
5825 | if (isa<PtrToIntInst>(Val: UInst)) { |
5826 | LLVM_DEBUG(dbgs() << " - ptr2int assume the worst!\n" ); |
5827 | return isCapturedIn(State, /* Memory */ CapturedInMem: true, /* Integer */ CapturedInInt: true, |
5828 | /* Return */ CapturedInRet: true); |
5829 | } |
5830 | |
5831 | // For stores we already checked if we can follow them, if they make it |
5832 | // here we give up. |
5833 | if (isa<StoreInst>(Val: UInst)) |
5834 | return isCapturedIn(State, /* Memory */ CapturedInMem: true, /* Integer */ CapturedInInt: true, |
5835 | /* Return */ CapturedInRet: true); |
5836 | |
5837 | // Explicitly catch return instructions. |
5838 | if (isa<ReturnInst>(Val: UInst)) { |
5839 | if (UInst->getFunction() == getAnchorScope()) |
5840 | return isCapturedIn(State, /* Memory */ CapturedInMem: false, /* Integer */ CapturedInInt: false, |
5841 | /* Return */ CapturedInRet: true); |
5842 | return isCapturedIn(State, /* Memory */ CapturedInMem: true, /* Integer */ CapturedInInt: true, |
5843 | /* Return */ CapturedInRet: true); |
5844 | } |
5845 | |
5846 | // For now we only use special logic for call sites. However, the tracker |
5847 | // itself knows about a lot of other non-capturing cases already. |
5848 | auto *CB = dyn_cast<CallBase>(Val: UInst); |
5849 | if (!CB || !CB->isArgOperand(U: &U)) |
5850 | return isCapturedIn(State, /* Memory */ CapturedInMem: true, /* Integer */ CapturedInInt: true, |
5851 | /* Return */ CapturedInRet: true); |
5852 | |
5853 | unsigned ArgNo = CB->getArgOperandNo(U: &U); |
5854 | const IRPosition &CSArgPos = IRPosition::callsite_argument(CB: *CB, ArgNo); |
5855 | // If we have a abstract no-capture attribute for the argument we can use |
5856 | // it to justify a non-capture attribute here. This allows recursion! |
5857 | bool IsKnownNoCapture; |
5858 | const AANoCapture *ArgNoCaptureAA = nullptr; |
5859 | bool IsAssumedNoCapture = AA::hasAssumedIRAttr<Attribute::NoCapture>( |
5860 | A, this, CSArgPos, DepClassTy::REQUIRED, IsKnownNoCapture, false, |
5861 | &ArgNoCaptureAA); |
5862 | if (IsAssumedNoCapture) |
5863 | return isCapturedIn(State, /* Memory */ CapturedInMem: false, /* Integer */ CapturedInInt: false, |
5864 | /* Return */ CapturedInRet: false); |
5865 | if (ArgNoCaptureAA && ArgNoCaptureAA->isAssumedNoCaptureMaybeReturned()) { |
5866 | Follow = true; |
5867 | return isCapturedIn(State, /* Memory */ CapturedInMem: false, /* Integer */ CapturedInInt: false, |
5868 | /* Return */ CapturedInRet: false); |
5869 | } |
5870 | |
5871 | // Lastly, we could not find a reason no-capture can be assumed so we don't. |
5872 | return isCapturedIn(State, /* Memory */ CapturedInMem: true, /* Integer */ CapturedInInt: true, |
5873 | /* Return */ CapturedInRet: true); |
5874 | } |
5875 | |
5876 | /// Update \p State according to \p CapturedInMem, \p CapturedInInt, and |
5877 | /// \p CapturedInRet, then return true if we should continue updating the |
5878 | /// state. |
5879 | static bool isCapturedIn(AANoCapture::StateType &State, bool CapturedInMem, |
5880 | bool CapturedInInt, bool CapturedInRet) { |
5881 | LLVM_DEBUG(dbgs() << " - captures [Mem " << CapturedInMem << "|Int " |
5882 | << CapturedInInt << "|Ret " << CapturedInRet << "]\n" ); |
5883 | if (CapturedInMem) |
5884 | State.removeAssumedBits(AANoCapture::NOT_CAPTURED_IN_MEM); |
5885 | if (CapturedInInt) |
5886 | State.removeAssumedBits(AANoCapture::NOT_CAPTURED_IN_INT); |
5887 | if (CapturedInRet) |
5888 | State.removeAssumedBits(AANoCapture::NOT_CAPTURED_IN_RET); |
5889 | return State.isAssumed(AANoCapture::NO_CAPTURE_MAYBE_RETURNED); |
5890 | } |
5891 | }; |
5892 | |
5893 | ChangeStatus AANoCaptureImpl::updateImpl(Attributor &A) { |
5894 | const IRPosition &IRP = getIRPosition(); |
5895 | Value *V = isArgumentPosition() ? IRP.getAssociatedArgument() |
5896 | : &IRP.getAssociatedValue(); |
5897 | if (!V) |
5898 | return indicatePessimisticFixpoint(); |
5899 | |
5900 | const Function *F = |
5901 | isArgumentPosition() ? IRP.getAssociatedFunction() : IRP.getAnchorScope(); |
5902 | assert(F && "Expected a function!" ); |
5903 | const IRPosition &FnPos = IRPosition::function(F: *F); |
5904 | |
5905 | AANoCapture::StateType T; |
5906 | |
5907 | // Readonly means we cannot capture through memory. |
5908 | bool IsKnown; |
5909 | if (AA::isAssumedReadOnly(A, FnPos, *this, IsKnown)) { |
5910 | T.addKnownBits(NOT_CAPTURED_IN_MEM); |
5911 | if (IsKnown) |
5912 | addKnownBits(NOT_CAPTURED_IN_MEM); |
5913 | } |
5914 | |
5915 | // Make sure all returned values are different than the underlying value. |
5916 | // TODO: we could do this in a more sophisticated way inside |
5917 | // AAReturnedValues, e.g., track all values that escape through returns |
5918 | // directly somehow. |
5919 | auto CheckReturnedArgs = [&](bool &UsedAssumedInformation) { |
5920 | SmallVector<AA::ValueAndContext> Values; |
5921 | if (!A.getAssumedSimplifiedValues(IRPosition::returned(F: *F), this, Values, |
5922 | AA::ValueScope::Intraprocedural, |
5923 | UsedAssumedInformation)) |
5924 | return false; |
5925 | bool SeenConstant = false; |
5926 | for (const AA::ValueAndContext &VAC : Values) { |
5927 | if (isa<Constant>(Val: VAC.getValue())) { |
5928 | if (SeenConstant) |
5929 | return false; |
5930 | SeenConstant = true; |
5931 | } else if (!isa<Argument>(Val: VAC.getValue()) || |
5932 | VAC.getValue() == getAssociatedArgument()) |
5933 | return false; |
5934 | } |
5935 | return true; |
5936 | }; |
5937 | |
5938 | bool IsKnownNoUnwind; |
5939 | if (AA::hasAssumedIRAttr<Attribute::NoUnwind>( |
5940 | A, this, FnPos, DepClassTy::OPTIONAL, IsKnownNoUnwind)) { |
5941 | bool IsVoidTy = F->getReturnType()->isVoidTy(); |
5942 | bool UsedAssumedInformation = false; |
5943 | if (IsVoidTy || CheckReturnedArgs(UsedAssumedInformation)) { |
5944 | T.addKnownBits(NOT_CAPTURED_IN_RET); |
5945 | if (T.isKnown(NOT_CAPTURED_IN_MEM)) |
5946 | return ChangeStatus::UNCHANGED; |
5947 | if (IsKnownNoUnwind && (IsVoidTy || !UsedAssumedInformation)) { |
5948 | addKnownBits(NOT_CAPTURED_IN_RET); |
5949 | if (isKnown(NOT_CAPTURED_IN_MEM)) |
5950 | return indicateOptimisticFixpoint(); |
5951 | } |
5952 | } |
5953 | } |
5954 | |
5955 | auto IsDereferenceableOrNull = [&](Value *O, const DataLayout &DL) { |
5956 | const auto *DerefAA = A.getAAFor<AADereferenceable>( |
5957 | *this, IRPosition::value(V: *O), DepClassTy::OPTIONAL); |
5958 | return DerefAA && DerefAA->getAssumedDereferenceableBytes(); |
5959 | }; |
5960 | |
5961 | auto UseCheck = [&](const Use &U, bool &Follow) -> bool { |
5962 | switch (DetermineUseCaptureKind(U, IsDereferenceableOrNull)) { |
5963 | case UseCaptureKind::NO_CAPTURE: |
5964 | return true; |
5965 | case UseCaptureKind::MAY_CAPTURE: |
5966 | return checkUse(A, State&: T, U, Follow); |
5967 | case UseCaptureKind::PASSTHROUGH: |
5968 | Follow = true; |
5969 | return true; |
5970 | } |
5971 | llvm_unreachable("Unexpected use capture kind!" ); |
5972 | }; |
5973 | |
5974 | if (!A.checkForAllUses(UseCheck, *this, *V)) |
5975 | return indicatePessimisticFixpoint(); |
5976 | |
5977 | AANoCapture::StateType &S = getState(); |
5978 | auto Assumed = S.getAssumed(); |
5979 | S.intersectAssumedBits(T.getAssumed()); |
5980 | if (!isAssumedNoCaptureMaybeReturned()) |
5981 | return indicatePessimisticFixpoint(); |
5982 | return Assumed == S.getAssumed() ? ChangeStatus::UNCHANGED |
5983 | : ChangeStatus::CHANGED; |
5984 | } |
5985 | |
5986 | /// NoCapture attribute for function arguments. |
5987 | struct AANoCaptureArgument final : AANoCaptureImpl { |
5988 | AANoCaptureArgument(const IRPosition &IRP, Attributor &A) |
5989 | : AANoCaptureImpl(IRP, A) {} |
5990 | |
5991 | /// See AbstractAttribute::trackStatistics() |
5992 | void trackStatistics() const override { STATS_DECLTRACK_ARG_ATTR(nocapture) } |
5993 | }; |
5994 | |
5995 | /// NoCapture attribute for call site arguments. |
5996 | struct AANoCaptureCallSiteArgument final : AANoCaptureImpl { |
5997 | AANoCaptureCallSiteArgument(const IRPosition &IRP, Attributor &A) |
5998 | : AANoCaptureImpl(IRP, A) {} |
5999 | |
6000 | /// See AbstractAttribute::updateImpl(...). |
6001 | ChangeStatus updateImpl(Attributor &A) override { |
6002 | // TODO: Once we have call site specific value information we can provide |
6003 | // call site specific liveness information and then it makes |
6004 | // sense to specialize attributes for call sites arguments instead of |
6005 | // redirecting requests to the callee argument. |
6006 | Argument *Arg = getAssociatedArgument(); |
6007 | if (!Arg) |
6008 | return indicatePessimisticFixpoint(); |
6009 | const IRPosition &ArgPos = IRPosition::argument(Arg: *Arg); |
6010 | bool IsKnownNoCapture; |
6011 | const AANoCapture *ArgAA = nullptr; |
6012 | if (AA::hasAssumedIRAttr<Attribute::NoCapture>( |
6013 | A, this, ArgPos, DepClassTy::REQUIRED, IsKnownNoCapture, false, |
6014 | &ArgAA)) |
6015 | return ChangeStatus::UNCHANGED; |
6016 | if (!ArgAA || !ArgAA->isAssumedNoCaptureMaybeReturned()) |
6017 | return indicatePessimisticFixpoint(); |
6018 | return clampStateAndIndicateChange(getState(), ArgAA->getState()); |
6019 | } |
6020 | |
6021 | /// See AbstractAttribute::trackStatistics() |
6022 | void trackStatistics() const override{STATS_DECLTRACK_CSARG_ATTR(nocapture)}; |
6023 | }; |
6024 | |
6025 | /// NoCapture attribute for floating values. |
6026 | struct AANoCaptureFloating final : AANoCaptureImpl { |
6027 | AANoCaptureFloating(const IRPosition &IRP, Attributor &A) |
6028 | : AANoCaptureImpl(IRP, A) {} |
6029 | |
6030 | /// See AbstractAttribute::trackStatistics() |
6031 | void trackStatistics() const override { |
6032 | STATS_DECLTRACK_FLOATING_ATTR(nocapture) |
6033 | } |
6034 | }; |
6035 | |
6036 | /// NoCapture attribute for function return value. |
6037 | struct AANoCaptureReturned final : AANoCaptureImpl { |
6038 | AANoCaptureReturned(const IRPosition &IRP, Attributor &A) |
6039 | : AANoCaptureImpl(IRP, A) { |
6040 | llvm_unreachable("NoCapture is not applicable to function returns!" ); |
6041 | } |
6042 | |
6043 | /// See AbstractAttribute::initialize(...). |
6044 | void initialize(Attributor &A) override { |
6045 | llvm_unreachable("NoCapture is not applicable to function returns!" ); |
6046 | } |
6047 | |
6048 | /// See AbstractAttribute::updateImpl(...). |
6049 | ChangeStatus updateImpl(Attributor &A) override { |
6050 | llvm_unreachable("NoCapture is not applicable to function returns!" ); |
6051 | } |
6052 | |
6053 | /// See AbstractAttribute::trackStatistics() |
6054 | void trackStatistics() const override {} |
6055 | }; |
6056 | |
6057 | /// NoCapture attribute deduction for a call site return value. |
6058 | struct AANoCaptureCallSiteReturned final : AANoCaptureImpl { |
6059 | AANoCaptureCallSiteReturned(const IRPosition &IRP, Attributor &A) |
6060 | : AANoCaptureImpl(IRP, A) {} |
6061 | |
6062 | /// See AbstractAttribute::initialize(...). |
6063 | void initialize(Attributor &A) override { |
6064 | const Function *F = getAnchorScope(); |
6065 | // Check what state the associated function can actually capture. |
6066 | determineFunctionCaptureCapabilities(getIRPosition(), *F, *this); |
6067 | } |
6068 | |
6069 | /// See AbstractAttribute::trackStatistics() |
6070 | void trackStatistics() const override { |
6071 | STATS_DECLTRACK_CSRET_ATTR(nocapture) |
6072 | } |
6073 | }; |
6074 | } // namespace |
6075 | |
6076 | /// ------------------ Value Simplify Attribute ---------------------------- |
6077 | |
6078 | bool ValueSimplifyStateType::unionAssumed(std::optional<Value *> Other) { |
6079 | // FIXME: Add a typecast support. |
6080 | SimplifiedAssociatedValue = AA::combineOptionalValuesInAAValueLatice( |
6081 | A: SimplifiedAssociatedValue, B: Other, Ty); |
6082 | if (SimplifiedAssociatedValue == std::optional<Value *>(nullptr)) |
6083 | return false; |
6084 | |
6085 | LLVM_DEBUG({ |
6086 | if (SimplifiedAssociatedValue) |
6087 | dbgs() << "[ValueSimplify] is assumed to be " |
6088 | << **SimplifiedAssociatedValue << "\n" ; |
6089 | else |
6090 | dbgs() << "[ValueSimplify] is assumed to be <none>\n" ; |
6091 | }); |
6092 | return true; |
6093 | } |
6094 | |
6095 | namespace { |
6096 | struct AAValueSimplifyImpl : AAValueSimplify { |
6097 | AAValueSimplifyImpl(const IRPosition &IRP, Attributor &A) |
6098 | : AAValueSimplify(IRP, A) {} |
6099 | |
6100 | /// See AbstractAttribute::initialize(...). |
6101 | void initialize(Attributor &A) override { |
6102 | if (getAssociatedValue().getType()->isVoidTy()) |
6103 | indicatePessimisticFixpoint(); |
6104 | if (A.hasSimplificationCallback(IRP: getIRPosition())) |
6105 | indicatePessimisticFixpoint(); |
6106 | } |
6107 | |
6108 | /// See AbstractAttribute::getAsStr(). |
6109 | const std::string getAsStr(Attributor *A) const override { |
6110 | LLVM_DEBUG({ |
6111 | dbgs() << "SAV: " << (bool)SimplifiedAssociatedValue << " " ; |
6112 | if (SimplifiedAssociatedValue && *SimplifiedAssociatedValue) |
6113 | dbgs() << "SAV: " << **SimplifiedAssociatedValue << " " ; |
6114 | }); |
6115 | return isValidState() ? (isAtFixpoint() ? "simplified" : "maybe-simple" ) |
6116 | : "not-simple" ; |
6117 | } |
6118 | |
6119 | /// See AbstractAttribute::trackStatistics() |
6120 | void trackStatistics() const override {} |
6121 | |
6122 | /// See AAValueSimplify::getAssumedSimplifiedValue() |
6123 | std::optional<Value *> |
6124 | getAssumedSimplifiedValue(Attributor &A) const override { |
6125 | return SimplifiedAssociatedValue; |
6126 | } |
6127 | |
6128 | /// Ensure the return value is \p V with type \p Ty, if not possible return |
6129 | /// nullptr. If \p Check is true we will only verify such an operation would |
6130 | /// suceed and return a non-nullptr value if that is the case. No IR is |
6131 | /// generated or modified. |
6132 | static Value *ensureType(Attributor &A, Value &V, Type &Ty, Instruction *CtxI, |
6133 | bool Check) { |
6134 | if (auto *TypedV = AA::getWithType(V, Ty)) |
6135 | return TypedV; |
6136 | if (CtxI && V.getType()->canLosslesslyBitCastTo(Ty: &Ty)) |
6137 | return Check ? &V |
6138 | : BitCastInst::CreatePointerBitCastOrAddrSpaceCast( |
6139 | S: &V, Ty: &Ty, Name: "" , InsertBefore: CtxI->getIterator()); |
6140 | return nullptr; |
6141 | } |
6142 | |
6143 | /// Reproduce \p I with type \p Ty or return nullptr if that is not posisble. |
6144 | /// If \p Check is true we will only verify such an operation would suceed and |
6145 | /// return a non-nullptr value if that is the case. No IR is generated or |
6146 | /// modified. |
6147 | static Value *reproduceInst(Attributor &A, |
6148 | const AbstractAttribute &QueryingAA, |
6149 | Instruction &I, Type &Ty, Instruction *CtxI, |
6150 | bool Check, ValueToValueMapTy &VMap) { |
6151 | assert(CtxI && "Cannot reproduce an instruction without context!" ); |
6152 | if (Check && (I.mayReadFromMemory() || |
6153 | !isSafeToSpeculativelyExecute(I: &I, CtxI, /* DT */ AC: nullptr, |
6154 | /* TLI */ DT: nullptr))) |
6155 | return nullptr; |
6156 | for (Value *Op : I.operands()) { |
6157 | Value *NewOp = reproduceValue(A, QueryingAA, V&: *Op, Ty, CtxI, Check, VMap); |
6158 | if (!NewOp) { |
6159 | assert(Check && "Manifest of new value unexpectedly failed!" ); |
6160 | return nullptr; |
6161 | } |
6162 | if (!Check) |
6163 | VMap[Op] = NewOp; |
6164 | } |
6165 | if (Check) |
6166 | return &I; |
6167 | |
6168 | Instruction *CloneI = I.clone(); |
6169 | // TODO: Try to salvage debug information here. |
6170 | CloneI->setDebugLoc(DebugLoc()); |
6171 | VMap[&I] = CloneI; |
6172 | CloneI->insertBefore(InsertPos: CtxI); |
6173 | RemapInstruction(I: CloneI, VM&: VMap); |
6174 | return CloneI; |
6175 | } |
6176 | |
6177 | /// Reproduce \p V with type \p Ty or return nullptr if that is not posisble. |
6178 | /// If \p Check is true we will only verify such an operation would suceed and |
6179 | /// return a non-nullptr value if that is the case. No IR is generated or |
6180 | /// modified. |
6181 | static Value *reproduceValue(Attributor &A, |
6182 | const AbstractAttribute &QueryingAA, Value &V, |
6183 | Type &Ty, Instruction *CtxI, bool Check, |
6184 | ValueToValueMapTy &VMap) { |
6185 | if (const auto &NewV = VMap.lookup(Val: &V)) |
6186 | return NewV; |
6187 | bool UsedAssumedInformation = false; |
6188 | std::optional<Value *> SimpleV = A.getAssumedSimplified( |
6189 | V, AA: QueryingAA, UsedAssumedInformation, S: AA::Interprocedural); |
6190 | if (!SimpleV.has_value()) |
6191 | return PoisonValue::get(T: &Ty); |
6192 | Value *EffectiveV = &V; |
6193 | if (*SimpleV) |
6194 | EffectiveV = *SimpleV; |
6195 | if (auto *C = dyn_cast<Constant>(Val: EffectiveV)) |
6196 | return C; |
6197 | if (CtxI && AA::isValidAtPosition(VAC: AA::ValueAndContext(*EffectiveV, *CtxI), |
6198 | InfoCache&: A.getInfoCache())) |
6199 | return ensureType(A, V&: *EffectiveV, Ty, CtxI, Check); |
6200 | if (auto *I = dyn_cast<Instruction>(Val: EffectiveV)) |
6201 | if (Value *NewV = reproduceInst(A, QueryingAA, I&: *I, Ty, CtxI, Check, VMap)) |
6202 | return ensureType(A, V&: *NewV, Ty, CtxI, Check); |
6203 | return nullptr; |
6204 | } |
6205 | |
6206 | /// Return a value we can use as replacement for the associated one, or |
6207 | /// nullptr if we don't have one that makes sense. |
6208 | Value *manifestReplacementValue(Attributor &A, Instruction *CtxI) const { |
6209 | Value *NewV = SimplifiedAssociatedValue |
6210 | ? *SimplifiedAssociatedValue |
6211 | : UndefValue::get(T: getAssociatedType()); |
6212 | if (NewV && NewV != &getAssociatedValue()) { |
6213 | ValueToValueMapTy VMap; |
6214 | // First verify we can reprduce the value with the required type at the |
6215 | // context location before we actually start modifying the IR. |
6216 | if (reproduceValue(A, QueryingAA: *this, V&: *NewV, Ty&: *getAssociatedType(), CtxI, |
6217 | /* CheckOnly */ Check: true, VMap)) |
6218 | return reproduceValue(A, QueryingAA: *this, V&: *NewV, Ty&: *getAssociatedType(), CtxI, |
6219 | /* CheckOnly */ Check: false, VMap); |
6220 | } |
6221 | return nullptr; |
6222 | } |
6223 | |
6224 | /// Helper function for querying AAValueSimplify and updating candidate. |
6225 | /// \param IRP The value position we are trying to unify with SimplifiedValue |
6226 | bool checkAndUpdate(Attributor &A, const AbstractAttribute &QueryingAA, |
6227 | const IRPosition &IRP, bool Simplify = true) { |
6228 | bool UsedAssumedInformation = false; |
6229 | std::optional<Value *> QueryingValueSimplified = &IRP.getAssociatedValue(); |
6230 | if (Simplify) |
6231 | QueryingValueSimplified = A.getAssumedSimplified( |
6232 | IRP, AA: QueryingAA, UsedAssumedInformation, S: AA::Interprocedural); |
6233 | return unionAssumed(Other: QueryingValueSimplified); |
6234 | } |
6235 | |
6236 | /// Returns a candidate is found or not |
6237 | template <typename AAType> bool askSimplifiedValueFor(Attributor &A) { |
6238 | if (!getAssociatedValue().getType()->isIntegerTy()) |
6239 | return false; |
6240 | |
6241 | // This will also pass the call base context. |
6242 | const auto *AA = |
6243 | A.getAAFor<AAType>(*this, getIRPosition(), DepClassTy::NONE); |
6244 | if (!AA) |
6245 | return false; |
6246 | |
6247 | std::optional<Constant *> COpt = AA->getAssumedConstant(A); |
6248 | |
6249 | if (!COpt) { |
6250 | SimplifiedAssociatedValue = std::nullopt; |
6251 | A.recordDependence(FromAA: *AA, ToAA: *this, DepClass: DepClassTy::OPTIONAL); |
6252 | return true; |
6253 | } |
6254 | if (auto *C = *COpt) { |
6255 | SimplifiedAssociatedValue = C; |
6256 | A.recordDependence(FromAA: *AA, ToAA: *this, DepClass: DepClassTy::OPTIONAL); |
6257 | return true; |
6258 | } |
6259 | return false; |
6260 | } |
6261 | |
6262 | bool askSimplifiedValueForOtherAAs(Attributor &A) { |
6263 | if (askSimplifiedValueFor<AAValueConstantRange>(A)) |
6264 | return true; |
6265 | if (askSimplifiedValueFor<AAPotentialConstantValues>(A)) |
6266 | return true; |
6267 | return false; |
6268 | } |
6269 | |
6270 | /// See AbstractAttribute::manifest(...). |
6271 | ChangeStatus manifest(Attributor &A) override { |
6272 | ChangeStatus Changed = ChangeStatus::UNCHANGED; |
6273 | for (auto &U : getAssociatedValue().uses()) { |
6274 | // Check if we need to adjust the insertion point to make sure the IR is |
6275 | // valid. |
6276 | Instruction *IP = dyn_cast<Instruction>(Val: U.getUser()); |
6277 | if (auto *PHI = dyn_cast_or_null<PHINode>(Val: IP)) |
6278 | IP = PHI->getIncomingBlock(U)->getTerminator(); |
6279 | if (auto *NewV = manifestReplacementValue(A, CtxI: IP)) { |
6280 | LLVM_DEBUG(dbgs() << "[ValueSimplify] " << getAssociatedValue() |
6281 | << " -> " << *NewV << " :: " << *this << "\n" ); |
6282 | if (A.changeUseAfterManifest(U, NV&: *NewV)) |
6283 | Changed = ChangeStatus::CHANGED; |
6284 | } |
6285 | } |
6286 | |
6287 | return Changed | AAValueSimplify::manifest(A); |
6288 | } |
6289 | |
6290 | /// See AbstractState::indicatePessimisticFixpoint(...). |
6291 | ChangeStatus indicatePessimisticFixpoint() override { |
6292 | SimplifiedAssociatedValue = &getAssociatedValue(); |
6293 | return AAValueSimplify::indicatePessimisticFixpoint(); |
6294 | } |
6295 | }; |
6296 | |
6297 | struct AAValueSimplifyArgument final : AAValueSimplifyImpl { |
6298 | AAValueSimplifyArgument(const IRPosition &IRP, Attributor &A) |
6299 | : AAValueSimplifyImpl(IRP, A) {} |
6300 | |
6301 | void initialize(Attributor &A) override { |
6302 | AAValueSimplifyImpl::initialize(A); |
6303 | if (A.hasAttr(getIRPosition(), |
6304 | {Attribute::InAlloca, Attribute::Preallocated, |
6305 | Attribute::StructRet, Attribute::Nest, Attribute::ByVal}, |
6306 | /* IgnoreSubsumingPositions */ true)) |
6307 | indicatePessimisticFixpoint(); |
6308 | } |
6309 | |
6310 | /// See AbstractAttribute::updateImpl(...). |
6311 | ChangeStatus updateImpl(Attributor &A) override { |
6312 | // Byval is only replacable if it is readonly otherwise we would write into |
6313 | // the replaced value and not the copy that byval creates implicitly. |
6314 | Argument *Arg = getAssociatedArgument(); |
6315 | if (Arg->hasByValAttr()) { |
6316 | // TODO: We probably need to verify synchronization is not an issue, e.g., |
6317 | // there is no race by not copying a constant byval. |
6318 | bool IsKnown; |
6319 | if (!AA::isAssumedReadOnly(A, IRP: getIRPosition(), QueryingAA: *this, IsKnown)) |
6320 | return indicatePessimisticFixpoint(); |
6321 | } |
6322 | |
6323 | auto Before = SimplifiedAssociatedValue; |
6324 | |
6325 | auto PredForCallSite = [&](AbstractCallSite ACS) { |
6326 | const IRPosition &ACSArgPos = |
6327 | IRPosition::callsite_argument(ACS, ArgNo: getCallSiteArgNo()); |
6328 | // Check if a coresponding argument was found or if it is on not |
6329 | // associated (which can happen for callback calls). |
6330 | if (ACSArgPos.getPositionKind() == IRPosition::IRP_INVALID) |
6331 | return false; |
6332 | |
6333 | // Simplify the argument operand explicitly and check if the result is |
6334 | // valid in the current scope. This avoids refering to simplified values |
6335 | // in other functions, e.g., we don't want to say a an argument in a |
6336 | // static function is actually an argument in a different function. |
6337 | bool UsedAssumedInformation = false; |
6338 | std::optional<Constant *> SimpleArgOp = |
6339 | A.getAssumedConstant(IRP: ACSArgPos, AA: *this, UsedAssumedInformation); |
6340 | if (!SimpleArgOp) |
6341 | return true; |
6342 | if (!*SimpleArgOp) |
6343 | return false; |
6344 | if (!AA::isDynamicallyUnique(A, QueryingAA: *this, V: **SimpleArgOp)) |
6345 | return false; |
6346 | return unionAssumed(Other: *SimpleArgOp); |
6347 | }; |
6348 | |
6349 | // Generate a answer specific to a call site context. |
6350 | bool Success; |
6351 | bool UsedAssumedInformation = false; |
6352 | if (hasCallBaseContext() && |
6353 | getCallBaseContext()->getCalledOperand() == Arg->getParent()) |
6354 | Success = PredForCallSite( |
6355 | AbstractCallSite(&getCallBaseContext()->getCalledOperandUse())); |
6356 | else |
6357 | Success = A.checkForAllCallSites(Pred: PredForCallSite, QueryingAA: *this, RequireAllCallSites: true, |
6358 | UsedAssumedInformation); |
6359 | |
6360 | if (!Success) |
6361 | if (!askSimplifiedValueForOtherAAs(A)) |
6362 | return indicatePessimisticFixpoint(); |
6363 | |
6364 | // If a candidate was found in this update, return CHANGED. |
6365 | return Before == SimplifiedAssociatedValue ? ChangeStatus::UNCHANGED |
6366 | : ChangeStatus ::CHANGED; |
6367 | } |
6368 | |
6369 | /// See AbstractAttribute::trackStatistics() |
6370 | void trackStatistics() const override { |
6371 | STATS_DECLTRACK_ARG_ATTR(value_simplify) |
6372 | } |
6373 | }; |
6374 | |
6375 | struct AAValueSimplifyReturned : AAValueSimplifyImpl { |
6376 | AAValueSimplifyReturned(const IRPosition &IRP, Attributor &A) |
6377 | : AAValueSimplifyImpl(IRP, A) {} |
6378 | |
6379 | /// See AAValueSimplify::getAssumedSimplifiedValue() |
6380 | std::optional<Value *> |
6381 | getAssumedSimplifiedValue(Attributor &A) const override { |
6382 | if (!isValidState()) |
6383 | return nullptr; |
6384 | return SimplifiedAssociatedValue; |
6385 | } |
6386 | |
6387 | /// See AbstractAttribute::updateImpl(...). |
6388 | ChangeStatus updateImpl(Attributor &A) override { |
6389 | auto Before = SimplifiedAssociatedValue; |
6390 | |
6391 | auto ReturnInstCB = [&](Instruction &I) { |
6392 | auto &RI = cast<ReturnInst>(Val&: I); |
6393 | return checkAndUpdate( |
6394 | A, QueryingAA: *this, |
6395 | IRP: IRPosition::value(V: *RI.getReturnValue(), CBContext: getCallBaseContext())); |
6396 | }; |
6397 | |
6398 | bool UsedAssumedInformation = false; |
6399 | if (!A.checkForAllInstructions(Pred: ReturnInstCB, QueryingAA: *this, Opcodes: {Instruction::Ret}, |
6400 | UsedAssumedInformation)) |
6401 | if (!askSimplifiedValueForOtherAAs(A)) |
6402 | return indicatePessimisticFixpoint(); |
6403 | |
6404 | // If a candidate was found in this update, return CHANGED. |
6405 | return Before == SimplifiedAssociatedValue ? ChangeStatus::UNCHANGED |
6406 | : ChangeStatus ::CHANGED; |
6407 | } |
6408 | |
6409 | ChangeStatus manifest(Attributor &A) override { |
6410 | // We queried AAValueSimplify for the returned values so they will be |
6411 | // replaced if a simplified form was found. Nothing to do here. |
6412 | return ChangeStatus::UNCHANGED; |
6413 | } |
6414 | |
6415 | /// See AbstractAttribute::trackStatistics() |
6416 | void trackStatistics() const override { |
6417 | STATS_DECLTRACK_FNRET_ATTR(value_simplify) |
6418 | } |
6419 | }; |
6420 | |
6421 | struct AAValueSimplifyFloating : AAValueSimplifyImpl { |
6422 | AAValueSimplifyFloating(const IRPosition &IRP, Attributor &A) |
6423 | : AAValueSimplifyImpl(IRP, A) {} |
6424 | |
6425 | /// See AbstractAttribute::initialize(...). |
6426 | void initialize(Attributor &A) override { |
6427 | AAValueSimplifyImpl::initialize(A); |
6428 | Value &V = getAnchorValue(); |
6429 | |
6430 | // TODO: add other stuffs |
6431 | if (isa<Constant>(Val: V)) |
6432 | indicatePessimisticFixpoint(); |
6433 | } |
6434 | |
6435 | /// See AbstractAttribute::updateImpl(...). |
6436 | ChangeStatus updateImpl(Attributor &A) override { |
6437 | auto Before = SimplifiedAssociatedValue; |
6438 | if (!askSimplifiedValueForOtherAAs(A)) |
6439 | return indicatePessimisticFixpoint(); |
6440 | |
6441 | // If a candidate was found in this update, return CHANGED. |
6442 | return Before == SimplifiedAssociatedValue ? ChangeStatus::UNCHANGED |
6443 | : ChangeStatus ::CHANGED; |
6444 | } |
6445 | |
6446 | /// See AbstractAttribute::trackStatistics() |
6447 | void trackStatistics() const override { |
6448 | STATS_DECLTRACK_FLOATING_ATTR(value_simplify) |
6449 | } |
6450 | }; |
6451 | |
6452 | struct AAValueSimplifyFunction : AAValueSimplifyImpl { |
6453 | AAValueSimplifyFunction(const IRPosition &IRP, Attributor &A) |
6454 | : AAValueSimplifyImpl(IRP, A) {} |
6455 | |
6456 | /// See AbstractAttribute::initialize(...). |
6457 | void initialize(Attributor &A) override { |
6458 | SimplifiedAssociatedValue = nullptr; |
6459 | indicateOptimisticFixpoint(); |
6460 | } |
6461 | /// See AbstractAttribute::initialize(...). |
6462 | ChangeStatus updateImpl(Attributor &A) override { |
6463 | llvm_unreachable( |
6464 | "AAValueSimplify(Function|CallSite)::updateImpl will not be called" ); |
6465 | } |
6466 | /// See AbstractAttribute::trackStatistics() |
6467 | void trackStatistics() const override { |
6468 | STATS_DECLTRACK_FN_ATTR(value_simplify) |
6469 | } |
6470 | }; |
6471 | |
6472 | struct AAValueSimplifyCallSite : AAValueSimplifyFunction { |
6473 | AAValueSimplifyCallSite(const IRPosition &IRP, Attributor &A) |
6474 | : AAValueSimplifyFunction(IRP, A) {} |
6475 | /// See AbstractAttribute::trackStatistics() |
6476 | void trackStatistics() const override { |
6477 | STATS_DECLTRACK_CS_ATTR(value_simplify) |
6478 | } |
6479 | }; |
6480 | |
6481 | struct AAValueSimplifyCallSiteReturned : AAValueSimplifyImpl { |
6482 | AAValueSimplifyCallSiteReturned(const IRPosition &IRP, Attributor &A) |
6483 | : AAValueSimplifyImpl(IRP, A) {} |
6484 | |
6485 | void initialize(Attributor &A) override { |
6486 | AAValueSimplifyImpl::initialize(A); |
6487 | Function *Fn = getAssociatedFunction(); |
6488 | assert(Fn && "Did expect an associted function" ); |
6489 | for (Argument &Arg : Fn->args()) { |
6490 | if (Arg.hasReturnedAttr()) { |
6491 | auto IRP = IRPosition::callsite_argument(CB: *cast<CallBase>(Val: getCtxI()), |
6492 | ArgNo: Arg.getArgNo()); |
6493 | if (IRP.getPositionKind() == IRPosition::IRP_CALL_SITE_ARGUMENT && |
6494 | checkAndUpdate(A, QueryingAA: *this, IRP)) |
6495 | indicateOptimisticFixpoint(); |
6496 | else |
6497 | indicatePessimisticFixpoint(); |
6498 | return; |
6499 | } |
6500 | } |
6501 | } |
6502 | |
6503 | /// See AbstractAttribute::updateImpl(...). |
6504 | ChangeStatus updateImpl(Attributor &A) override { |
6505 | return indicatePessimisticFixpoint(); |
6506 | } |
6507 | |
6508 | void trackStatistics() const override { |
6509 | STATS_DECLTRACK_CSRET_ATTR(value_simplify) |
6510 | } |
6511 | }; |
6512 | |
6513 | struct AAValueSimplifyCallSiteArgument : AAValueSimplifyFloating { |
6514 | AAValueSimplifyCallSiteArgument(const IRPosition &IRP, Attributor &A) |
6515 | : AAValueSimplifyFloating(IRP, A) {} |
6516 | |
6517 | /// See AbstractAttribute::manifest(...). |
6518 | ChangeStatus manifest(Attributor &A) override { |
6519 | ChangeStatus Changed = ChangeStatus::UNCHANGED; |
6520 | // TODO: We should avoid simplification duplication to begin with. |
6521 | auto *FloatAA = A.lookupAAFor<AAValueSimplify>( |
6522 | IRP: IRPosition::value(V: getAssociatedValue()), QueryingAA: this, DepClass: DepClassTy::NONE); |
6523 | if (FloatAA && FloatAA->getState().isValidState()) |
6524 | return Changed; |
6525 | |
6526 | if (auto *NewV = manifestReplacementValue(A, CtxI: getCtxI())) { |
6527 | Use &U = cast<CallBase>(Val: &getAnchorValue()) |
6528 | ->getArgOperandUse(i: getCallSiteArgNo()); |
6529 | if (A.changeUseAfterManifest(U, NV&: *NewV)) |
6530 | Changed = ChangeStatus::CHANGED; |
6531 | } |
6532 | |
6533 | return Changed | AAValueSimplify::manifest(A); |
6534 | } |
6535 | |
6536 | void trackStatistics() const override { |
6537 | STATS_DECLTRACK_CSARG_ATTR(value_simplify) |
6538 | } |
6539 | }; |
6540 | } // namespace |
6541 | |
6542 | /// ----------------------- Heap-To-Stack Conversion --------------------------- |
6543 | namespace { |
6544 | struct AAHeapToStackFunction final : public AAHeapToStack { |
6545 | |
6546 | struct AllocationInfo { |
6547 | /// The call that allocates the memory. |
6548 | CallBase *const CB; |
6549 | |
6550 | /// The library function id for the allocation. |
6551 | LibFunc LibraryFunctionId = NotLibFunc; |
6552 | |
6553 | /// The status wrt. a rewrite. |
6554 | enum { |
6555 | STACK_DUE_TO_USE, |
6556 | STACK_DUE_TO_FREE, |
6557 | INVALID, |
6558 | } Status = STACK_DUE_TO_USE; |
6559 | |
6560 | /// Flag to indicate if we encountered a use that might free this allocation |
6561 | /// but which is not in the deallocation infos. |
6562 | bool HasPotentiallyFreeingUnknownUses = false; |
6563 | |
6564 | /// Flag to indicate that we should place the new alloca in the function |
6565 | /// entry block rather than where the call site (CB) is. |
6566 | bool MoveAllocaIntoEntry = true; |
6567 | |
6568 | /// The set of free calls that use this allocation. |
6569 | SmallSetVector<CallBase *, 1> PotentialFreeCalls{}; |
6570 | }; |
6571 | |
6572 | struct DeallocationInfo { |
6573 | /// The call that deallocates the memory. |
6574 | CallBase *const CB; |
6575 | /// The value freed by the call. |
6576 | Value *FreedOp; |
6577 | |
6578 | /// Flag to indicate if we don't know all objects this deallocation might |
6579 | /// free. |
6580 | bool MightFreeUnknownObjects = false; |
6581 | |
6582 | /// The set of allocation calls that are potentially freed. |
6583 | SmallSetVector<CallBase *, 1> PotentialAllocationCalls{}; |
6584 | }; |
6585 | |
6586 | AAHeapToStackFunction(const IRPosition &IRP, Attributor &A) |
6587 | : AAHeapToStack(IRP, A) {} |
6588 | |
6589 | ~AAHeapToStackFunction() { |
6590 | // Ensure we call the destructor so we release any memory allocated in the |
6591 | // sets. |
6592 | for (auto &It : AllocationInfos) |
6593 | It.second->~AllocationInfo(); |
6594 | for (auto &It : DeallocationInfos) |
6595 | It.second->~DeallocationInfo(); |
6596 | } |
6597 | |
6598 | void initialize(Attributor &A) override { |
6599 | AAHeapToStack::initialize(A); |
6600 | |
6601 | const Function *F = getAnchorScope(); |
6602 | const auto *TLI = A.getInfoCache().getTargetLibraryInfoForFunction(F: *F); |
6603 | |
6604 | auto AllocationIdentifierCB = [&](Instruction &I) { |
6605 | CallBase *CB = dyn_cast<CallBase>(Val: &I); |
6606 | if (!CB) |
6607 | return true; |
6608 | if (Value *FreedOp = getFreedOperand(CB, TLI)) { |
6609 | DeallocationInfos[CB] = new (A.Allocator) DeallocationInfo{.CB: CB, .FreedOp: FreedOp}; |
6610 | return true; |
6611 | } |
6612 | // To do heap to stack, we need to know that the allocation itself is |
6613 | // removable once uses are rewritten, and that we can initialize the |
6614 | // alloca to the same pattern as the original allocation result. |
6615 | if (isRemovableAlloc(V: CB, TLI)) { |
6616 | auto *I8Ty = Type::getInt8Ty(C&: CB->getParent()->getContext()); |
6617 | if (nullptr != getInitialValueOfAllocation(V: CB, TLI, Ty: I8Ty)) { |
6618 | AllocationInfo *AI = new (A.Allocator) AllocationInfo{.CB: CB}; |
6619 | AllocationInfos[CB] = AI; |
6620 | if (TLI) |
6621 | TLI->getLibFunc(CB: *CB, F&: AI->LibraryFunctionId); |
6622 | } |
6623 | } |
6624 | return true; |
6625 | }; |
6626 | |
6627 | bool UsedAssumedInformation = false; |
6628 | bool Success = A.checkForAllCallLikeInstructions( |
6629 | Pred: AllocationIdentifierCB, QueryingAA: *this, UsedAssumedInformation, |
6630 | /* CheckBBLivenessOnly */ false, |
6631 | /* CheckPotentiallyDead */ true); |
6632 | (void)Success; |
6633 | assert(Success && "Did not expect the call base visit callback to fail!" ); |
6634 | |
6635 | Attributor::SimplifictionCallbackTy SCB = |
6636 | [](const IRPosition &, const AbstractAttribute *, |
6637 | bool &) -> std::optional<Value *> { return nullptr; }; |
6638 | for (const auto &It : AllocationInfos) |
6639 | A.registerSimplificationCallback(IRP: IRPosition::callsite_returned(CB: *It.first), |
6640 | CB: SCB); |
6641 | for (const auto &It : DeallocationInfos) |
6642 | A.registerSimplificationCallback(IRP: IRPosition::callsite_returned(CB: *It.first), |
6643 | CB: SCB); |
6644 | } |
6645 | |
6646 | const std::string getAsStr(Attributor *A) const override { |
6647 | unsigned NumH2SMallocs = 0, NumInvalidMallocs = 0; |
6648 | for (const auto &It : AllocationInfos) { |
6649 | if (It.second->Status == AllocationInfo::INVALID) |
6650 | ++NumInvalidMallocs; |
6651 | else |
6652 | ++NumH2SMallocs; |
6653 | } |
6654 | return "[H2S] Mallocs Good/Bad: " + std::to_string(val: NumH2SMallocs) + "/" + |
6655 | std::to_string(val: NumInvalidMallocs); |
6656 | } |
6657 | |
6658 | /// See AbstractAttribute::trackStatistics(). |
6659 | void trackStatistics() const override { |
6660 | STATS_DECL( |
6661 | MallocCalls, Function, |
6662 | "Number of malloc/calloc/aligned_alloc calls converted to allocas" ); |
6663 | for (const auto &It : AllocationInfos) |
6664 | if (It.second->Status != AllocationInfo::INVALID) |
6665 | ++BUILD_STAT_NAME(MallocCalls, Function); |
6666 | } |
6667 | |
6668 | bool isAssumedHeapToStack(const CallBase &CB) const override { |
6669 | if (isValidState()) |
6670 | if (AllocationInfo *AI = |
6671 | AllocationInfos.lookup(Key: const_cast<CallBase *>(&CB))) |
6672 | return AI->Status != AllocationInfo::INVALID; |
6673 | return false; |
6674 | } |
6675 | |
6676 | bool isAssumedHeapToStackRemovedFree(CallBase &CB) const override { |
6677 | if (!isValidState()) |
6678 | return false; |
6679 | |
6680 | for (const auto &It : AllocationInfos) { |
6681 | AllocationInfo &AI = *It.second; |
6682 | if (AI.Status == AllocationInfo::INVALID) |
6683 | continue; |
6684 | |
6685 | if (AI.PotentialFreeCalls.count(key: &CB)) |
6686 | return true; |
6687 | } |
6688 | |
6689 | return false; |
6690 | } |
6691 | |
6692 | ChangeStatus manifest(Attributor &A) override { |
6693 | assert(getState().isValidState() && |
6694 | "Attempted to manifest an invalid state!" ); |
6695 | |
6696 | ChangeStatus HasChanged = ChangeStatus::UNCHANGED; |
6697 | Function *F = getAnchorScope(); |
6698 | const auto *TLI = A.getInfoCache().getTargetLibraryInfoForFunction(F: *F); |
6699 | |
6700 | for (auto &It : AllocationInfos) { |
6701 | AllocationInfo &AI = *It.second; |
6702 | if (AI.Status == AllocationInfo::INVALID) |
6703 | continue; |
6704 | |
6705 | for (CallBase *FreeCall : AI.PotentialFreeCalls) { |
6706 | LLVM_DEBUG(dbgs() << "H2S: Removing free call: " << *FreeCall << "\n" ); |
6707 | A.deleteAfterManifest(I&: *FreeCall); |
6708 | HasChanged = ChangeStatus::CHANGED; |
6709 | } |
6710 | |
6711 | LLVM_DEBUG(dbgs() << "H2S: Removing malloc-like call: " << *AI.CB |
6712 | << "\n" ); |
6713 | |
6714 | auto = [&](OptimizationRemark OR) { |
6715 | LibFunc IsAllocShared; |
6716 | if (TLI->getLibFunc(CB: *AI.CB, F&: IsAllocShared)) |
6717 | if (IsAllocShared == LibFunc___kmpc_alloc_shared) |
6718 | return OR << "Moving globalized variable to the stack." ; |
6719 | return OR << "Moving memory allocation from the heap to the stack." ; |
6720 | }; |
6721 | if (AI.LibraryFunctionId == LibFunc___kmpc_alloc_shared) |
6722 | A.emitRemark<OptimizationRemark>(I: AI.CB, RemarkName: "OMP110" , RemarkCB&: Remark); |
6723 | else |
6724 | A.emitRemark<OptimizationRemark>(I: AI.CB, RemarkName: "HeapToStack" , RemarkCB&: Remark); |
6725 | |
6726 | const DataLayout &DL = A.getInfoCache().getDL(); |
6727 | Value *Size; |
6728 | std::optional<APInt> SizeAPI = getSize(A, AA: *this, AI); |
6729 | if (SizeAPI) { |
6730 | Size = ConstantInt::get(Context&: AI.CB->getContext(), V: *SizeAPI); |
6731 | } else { |
6732 | LLVMContext &Ctx = AI.CB->getContext(); |
6733 | ObjectSizeOpts Opts; |
6734 | ObjectSizeOffsetEvaluator Eval(DL, TLI, Ctx, Opts); |
6735 | SizeOffsetValue SizeOffsetPair = Eval.compute(V: AI.CB); |
6736 | assert(SizeOffsetPair != ObjectSizeOffsetEvaluator::unknown() && |
6737 | cast<ConstantInt>(SizeOffsetPair.Offset)->isZero()); |
6738 | Size = SizeOffsetPair.Size; |
6739 | } |
6740 | |
6741 | BasicBlock::iterator IP = AI.MoveAllocaIntoEntry |
6742 | ? F->getEntryBlock().begin() |
6743 | : AI.CB->getIterator(); |
6744 | |
6745 | Align Alignment(1); |
6746 | if (MaybeAlign RetAlign = AI.CB->getRetAlign()) |
6747 | Alignment = std::max(a: Alignment, b: *RetAlign); |
6748 | if (Value *Align = getAllocAlignment(V: AI.CB, TLI)) { |
6749 | std::optional<APInt> AlignmentAPI = getAPInt(A, AA: *this, V&: *Align); |
6750 | assert(AlignmentAPI && AlignmentAPI->getZExtValue() > 0 && |
6751 | "Expected an alignment during manifest!" ); |
6752 | Alignment = |
6753 | std::max(a: Alignment, b: assumeAligned(Value: AlignmentAPI->getZExtValue())); |
6754 | } |
6755 | |
6756 | // TODO: Hoist the alloca towards the function entry. |
6757 | unsigned AS = DL.getAllocaAddrSpace(); |
6758 | Instruction *Alloca = |
6759 | new AllocaInst(Type::getInt8Ty(C&: F->getContext()), AS, Size, Alignment, |
6760 | AI.CB->getName() + ".h2s" , IP); |
6761 | |
6762 | if (Alloca->getType() != AI.CB->getType()) |
6763 | Alloca = BitCastInst::CreatePointerBitCastOrAddrSpaceCast( |
6764 | S: Alloca, Ty: AI.CB->getType(), Name: "malloc_cast" , InsertBefore: AI.CB->getIterator()); |
6765 | |
6766 | auto *I8Ty = Type::getInt8Ty(C&: F->getContext()); |
6767 | auto *InitVal = getInitialValueOfAllocation(V: AI.CB, TLI, Ty: I8Ty); |
6768 | assert(InitVal && |
6769 | "Must be able to materialize initial memory state of allocation" ); |
6770 | |
6771 | A.changeAfterManifest(IRP: IRPosition::inst(I: *AI.CB), NV&: *Alloca); |
6772 | |
6773 | if (auto *II = dyn_cast<InvokeInst>(Val: AI.CB)) { |
6774 | auto *NBB = II->getNormalDest(); |
6775 | BranchInst::Create(IfTrue: NBB, InsertAtEnd: AI.CB->getParent()); |
6776 | A.deleteAfterManifest(I&: *AI.CB); |
6777 | } else { |
6778 | A.deleteAfterManifest(I&: *AI.CB); |
6779 | } |
6780 | |
6781 | // Initialize the alloca with the same value as used by the allocation |
6782 | // function. We can skip undef as the initial value of an alloc is |
6783 | // undef, and the memset would simply end up being DSEd. |
6784 | if (!isa<UndefValue>(Val: InitVal)) { |
6785 | IRBuilder<> Builder(Alloca->getNextNode()); |
6786 | // TODO: Use alignment above if align!=1 |
6787 | Builder.CreateMemSet(Ptr: Alloca, Val: InitVal, Size, Align: std::nullopt); |
6788 | } |
6789 | HasChanged = ChangeStatus::CHANGED; |
6790 | } |
6791 | |
6792 | return HasChanged; |
6793 | } |
6794 | |
6795 | std::optional<APInt> getAPInt(Attributor &A, const AbstractAttribute &AA, |
6796 | Value &V) { |
6797 | bool UsedAssumedInformation = false; |
6798 | std::optional<Constant *> SimpleV = |
6799 | A.getAssumedConstant(V, AA, UsedAssumedInformation); |
6800 | if (!SimpleV) |
6801 | return APInt(64, 0); |
6802 | if (auto *CI = dyn_cast_or_null<ConstantInt>(Val: *SimpleV)) |
6803 | return CI->getValue(); |
6804 | return std::nullopt; |
6805 | } |
6806 | |
6807 | std::optional<APInt> getSize(Attributor &A, const AbstractAttribute &AA, |
6808 | AllocationInfo &AI) { |
6809 | auto Mapper = [&](const Value *V) -> const Value * { |
6810 | bool UsedAssumedInformation = false; |
6811 | if (std::optional<Constant *> SimpleV = |
6812 | A.getAssumedConstant(V: *V, AA, UsedAssumedInformation)) |
6813 | if (*SimpleV) |
6814 | return *SimpleV; |
6815 | return V; |
6816 | }; |
6817 | |
6818 | const Function *F = getAnchorScope(); |
6819 | const auto *TLI = A.getInfoCache().getTargetLibraryInfoForFunction(F: *F); |
6820 | return getAllocSize(CB: AI.CB, TLI, Mapper); |
6821 | } |
6822 | |
6823 | /// Collection of all malloc-like calls in a function with associated |
6824 | /// information. |
6825 | MapVector<CallBase *, AllocationInfo *> AllocationInfos; |
6826 | |
6827 | /// Collection of all free-like calls in a function with associated |
6828 | /// information. |
6829 | MapVector<CallBase *, DeallocationInfo *> DeallocationInfos; |
6830 | |
6831 | ChangeStatus updateImpl(Attributor &A) override; |
6832 | }; |
6833 | |
6834 | ChangeStatus AAHeapToStackFunction::updateImpl(Attributor &A) { |
6835 | ChangeStatus Changed = ChangeStatus::UNCHANGED; |
6836 | const Function *F = getAnchorScope(); |
6837 | const auto *TLI = A.getInfoCache().getTargetLibraryInfoForFunction(F: *F); |
6838 | |
6839 | const auto *LivenessAA = |
6840 | A.getAAFor<AAIsDead>(QueryingAA: *this, IRP: IRPosition::function(F: *F), DepClass: DepClassTy::NONE); |
6841 | |
6842 | MustBeExecutedContextExplorer *Explorer = |
6843 | A.getInfoCache().getMustBeExecutedContextExplorer(); |
6844 | |
6845 | bool StackIsAccessibleByOtherThreads = |
6846 | A.getInfoCache().stackIsAccessibleByOtherThreads(); |
6847 | |
6848 | LoopInfo *LI = |
6849 | A.getInfoCache().getAnalysisResultForFunction<LoopAnalysis>(F: *F); |
6850 | std::optional<bool> MayContainIrreducibleControl; |
6851 | auto IsInLoop = [&](BasicBlock &BB) { |
6852 | if (&F->getEntryBlock() == &BB) |
6853 | return false; |
6854 | if (!MayContainIrreducibleControl.has_value()) |
6855 | MayContainIrreducibleControl = mayContainIrreducibleControl(F: *F, LI); |
6856 | if (*MayContainIrreducibleControl) |
6857 | return true; |
6858 | if (!LI) |
6859 | return true; |
6860 | return LI->getLoopFor(BB: &BB) != nullptr; |
6861 | }; |
6862 | |
6863 | // Flag to ensure we update our deallocation information at most once per |
6864 | // updateImpl call and only if we use the free check reasoning. |
6865 | bool HasUpdatedFrees = false; |
6866 | |
6867 | auto UpdateFrees = [&]() { |
6868 | HasUpdatedFrees = true; |
6869 | |
6870 | for (auto &It : DeallocationInfos) { |
6871 | DeallocationInfo &DI = *It.second; |
6872 | // For now we cannot use deallocations that have unknown inputs, skip |
6873 | // them. |
6874 | if (DI.MightFreeUnknownObjects) |
6875 | continue; |
6876 | |
6877 | // No need to analyze dead calls, ignore them instead. |
6878 | bool UsedAssumedInformation = false; |
6879 | if (A.isAssumedDead(I: *DI.CB, QueryingAA: this, LivenessAA, UsedAssumedInformation, |
6880 | /* CheckBBLivenessOnly */ true)) |
6881 | continue; |
6882 | |
6883 | // Use the non-optimistic version to get the freed object. |
6884 | Value *Obj = getUnderlyingObject(V: DI.FreedOp); |
6885 | if (!Obj) { |
6886 | LLVM_DEBUG(dbgs() << "[H2S] Unknown underlying object for free!\n" ); |
6887 | DI.MightFreeUnknownObjects = true; |
6888 | continue; |
6889 | } |
6890 | |
6891 | // Free of null and undef can be ignored as no-ops (or UB in the latter |
6892 | // case). |
6893 | if (isa<ConstantPointerNull>(Val: Obj) || isa<UndefValue>(Val: Obj)) |
6894 | continue; |
6895 | |
6896 | CallBase *ObjCB = dyn_cast<CallBase>(Val: Obj); |
6897 | if (!ObjCB) { |
6898 | LLVM_DEBUG(dbgs() << "[H2S] Free of a non-call object: " << *Obj |
6899 | << "\n" ); |
6900 | DI.MightFreeUnknownObjects = true; |
6901 | continue; |
6902 | } |
6903 | |
6904 | AllocationInfo *AI = AllocationInfos.lookup(Key: ObjCB); |
6905 | if (!AI) { |
6906 | LLVM_DEBUG(dbgs() << "[H2S] Free of a non-allocation object: " << *Obj |
6907 | << "\n" ); |
6908 | DI.MightFreeUnknownObjects = true; |
6909 | continue; |
6910 | } |
6911 | |
6912 | DI.PotentialAllocationCalls.insert(X: ObjCB); |
6913 | } |
6914 | }; |
6915 | |
6916 | auto FreeCheck = [&](AllocationInfo &AI) { |
6917 | // If the stack is not accessible by other threads, the "must-free" logic |
6918 | // doesn't apply as the pointer could be shared and needs to be places in |
6919 | // "shareable" memory. |
6920 | if (!StackIsAccessibleByOtherThreads) { |
6921 | bool IsKnownNoSycn; |
6922 | if (!AA::hasAssumedIRAttr<Attribute::NoSync>( |
6923 | A, this, getIRPosition(), DepClassTy::OPTIONAL, IsKnownNoSycn)) { |
6924 | LLVM_DEBUG( |
6925 | dbgs() << "[H2S] found an escaping use, stack is not accessible by " |
6926 | "other threads and function is not nosync:\n" ); |
6927 | return false; |
6928 | } |
6929 | } |
6930 | if (!HasUpdatedFrees) |
6931 | UpdateFrees(); |
6932 | |
6933 | // TODO: Allow multi exit functions that have different free calls. |
6934 | if (AI.PotentialFreeCalls.size() != 1) { |
6935 | LLVM_DEBUG(dbgs() << "[H2S] did not find one free call but " |
6936 | << AI.PotentialFreeCalls.size() << "\n" ); |
6937 | return false; |
6938 | } |
6939 | CallBase *UniqueFree = *AI.PotentialFreeCalls.begin(); |
6940 | DeallocationInfo *DI = DeallocationInfos.lookup(Key: UniqueFree); |
6941 | if (!DI) { |
6942 | LLVM_DEBUG( |
6943 | dbgs() << "[H2S] unique free call was not known as deallocation call " |
6944 | << *UniqueFree << "\n" ); |
6945 | return false; |
6946 | } |
6947 | if (DI->MightFreeUnknownObjects) { |
6948 | LLVM_DEBUG( |
6949 | dbgs() << "[H2S] unique free call might free unknown allocations\n" ); |
6950 | return false; |
6951 | } |
6952 | if (DI->PotentialAllocationCalls.empty()) |
6953 | return true; |
6954 | if (DI->PotentialAllocationCalls.size() > 1) { |
6955 | LLVM_DEBUG(dbgs() << "[H2S] unique free call might free " |
6956 | << DI->PotentialAllocationCalls.size() |
6957 | << " different allocations\n" ); |
6958 | return false; |
6959 | } |
6960 | if (*DI->PotentialAllocationCalls.begin() != AI.CB) { |
6961 | LLVM_DEBUG( |
6962 | dbgs() |
6963 | << "[H2S] unique free call not known to free this allocation but " |
6964 | << **DI->PotentialAllocationCalls.begin() << "\n" ); |
6965 | return false; |
6966 | } |
6967 | |
6968 | // __kmpc_alloc_shared and __kmpc_alloc_free are by construction matched. |
6969 | if (AI.LibraryFunctionId != LibFunc___kmpc_alloc_shared) { |
6970 | Instruction *CtxI = isa<InvokeInst>(Val: AI.CB) ? AI.CB : AI.CB->getNextNode(); |
6971 | if (!Explorer || !Explorer->findInContextOf(I: UniqueFree, PP: CtxI)) { |
6972 | LLVM_DEBUG( |
6973 | dbgs() |
6974 | << "[H2S] unique free call might not be executed with the allocation " |
6975 | << *UniqueFree << "\n" ); |
6976 | return false; |
6977 | } |
6978 | } |
6979 | return true; |
6980 | }; |
6981 | |
6982 | auto UsesCheck = [&](AllocationInfo &AI) { |
6983 | bool ValidUsesOnly = true; |
6984 | |
6985 | auto Pred = [&](const Use &U, bool &Follow) -> bool { |
6986 | Instruction *UserI = cast<Instruction>(Val: U.getUser()); |
6987 | if (isa<LoadInst>(Val: UserI)) |
6988 | return true; |
6989 | if (auto *SI = dyn_cast<StoreInst>(Val: UserI)) { |
6990 | if (SI->getValueOperand() == U.get()) { |
6991 | LLVM_DEBUG(dbgs() |
6992 | << "[H2S] escaping store to memory: " << *UserI << "\n" ); |
6993 | ValidUsesOnly = false; |
6994 | } else { |
6995 | // A store into the malloc'ed memory is fine. |
6996 | } |
6997 | return true; |
6998 | } |
6999 | if (auto *CB = dyn_cast<CallBase>(Val: UserI)) { |
7000 | if (!CB->isArgOperand(U: &U) || CB->isLifetimeStartOrEnd()) |
7001 | return true; |
7002 | if (DeallocationInfos.count(Key: CB)) { |
7003 | AI.PotentialFreeCalls.insert(X: CB); |
7004 | return true; |
7005 | } |
7006 | |
7007 | unsigned ArgNo = CB->getArgOperandNo(U: &U); |
7008 | auto CBIRP = IRPosition::callsite_argument(CB: *CB, ArgNo); |
7009 | |
7010 | bool IsKnownNoCapture; |
7011 | bool IsAssumedNoCapture = AA::hasAssumedIRAttr<Attribute::NoCapture>( |
7012 | A, this, CBIRP, DepClassTy::OPTIONAL, IsKnownNoCapture); |
7013 | |
7014 | // If a call site argument use is nofree, we are fine. |
7015 | bool IsKnownNoFree; |
7016 | bool IsAssumedNoFree = AA::hasAssumedIRAttr<Attribute::NoFree>( |
7017 | A, this, CBIRP, DepClassTy::OPTIONAL, IsKnownNoFree); |
7018 | |
7019 | if (!IsAssumedNoCapture || |
7020 | (AI.LibraryFunctionId != LibFunc___kmpc_alloc_shared && |
7021 | !IsAssumedNoFree)) { |
7022 | AI.HasPotentiallyFreeingUnknownUses |= !IsAssumedNoFree; |
7023 | |
7024 | // Emit a missed remark if this is missed OpenMP globalization. |
7025 | auto = [&](OptimizationRemarkMissed ORM) { |
7026 | return ORM |
7027 | << "Could not move globalized variable to the stack. " |
7028 | "Variable is potentially captured in call. Mark " |
7029 | "parameter as `__attribute__((noescape))` to override." ; |
7030 | }; |
7031 | |
7032 | if (ValidUsesOnly && |
7033 | AI.LibraryFunctionId == LibFunc___kmpc_alloc_shared) |
7034 | A.emitRemark<OptimizationRemarkMissed>(I: CB, RemarkName: "OMP113" , RemarkCB&: Remark); |
7035 | |
7036 | LLVM_DEBUG(dbgs() << "[H2S] Bad user: " << *UserI << "\n" ); |
7037 | ValidUsesOnly = false; |
7038 | } |
7039 | return true; |
7040 | } |
7041 | |
7042 | if (isa<GetElementPtrInst>(Val: UserI) || isa<BitCastInst>(Val: UserI) || |
7043 | isa<PHINode>(Val: UserI) || isa<SelectInst>(Val: UserI)) { |
7044 | Follow = true; |
7045 | return true; |
7046 | } |
7047 | // Unknown user for which we can not track uses further (in a way that |
7048 | // makes sense). |
7049 | LLVM_DEBUG(dbgs() << "[H2S] Unknown user: " << *UserI << "\n" ); |
7050 | ValidUsesOnly = false; |
7051 | return true; |
7052 | }; |
7053 | if (!A.checkForAllUses(Pred, QueryingAA: *this, V: *AI.CB, /* CheckBBLivenessOnly */ false, |
7054 | LivenessDepClass: DepClassTy::OPTIONAL, /* IgnoreDroppableUses */ true, |
7055 | EquivalentUseCB: [&](const Use &OldU, const Use &NewU) { |
7056 | auto *SI = dyn_cast<StoreInst>(Val: OldU.getUser()); |
7057 | return !SI || StackIsAccessibleByOtherThreads || |
7058 | AA::isAssumedThreadLocalObject( |
7059 | A, Obj&: *SI->getPointerOperand(), QueryingAA: *this); |
7060 | })) |
7061 | return false; |
7062 | return ValidUsesOnly; |
7063 | }; |
7064 | |
7065 | // The actual update starts here. We look at all allocations and depending on |
7066 | // their status perform the appropriate check(s). |
7067 | for (auto &It : AllocationInfos) { |
7068 | AllocationInfo &AI = *It.second; |
7069 | if (AI.Status == AllocationInfo::INVALID) |
7070 | continue; |
7071 | |
7072 | if (Value *Align = getAllocAlignment(V: AI.CB, TLI)) { |
7073 | std::optional<APInt> APAlign = getAPInt(A, AA: *this, V&: *Align); |
7074 | if (!APAlign) { |
7075 | // Can't generate an alloca which respects the required alignment |
7076 | // on the allocation. |
7077 | LLVM_DEBUG(dbgs() << "[H2S] Unknown allocation alignment: " << *AI.CB |
7078 | << "\n" ); |
7079 | AI.Status = AllocationInfo::INVALID; |
7080 | Changed = ChangeStatus::CHANGED; |
7081 | continue; |
7082 | } |
7083 | if (APAlign->ugt(RHS: llvm::Value::MaximumAlignment) || |
7084 | !APAlign->isPowerOf2()) { |
7085 | LLVM_DEBUG(dbgs() << "[H2S] Invalid allocation alignment: " << APAlign |
7086 | << "\n" ); |
7087 | AI.Status = AllocationInfo::INVALID; |
7088 | Changed = ChangeStatus::CHANGED; |
7089 | continue; |
7090 | } |
7091 | } |
7092 | |
7093 | std::optional<APInt> Size = getSize(A, AA: *this, AI); |
7094 | if (AI.LibraryFunctionId != LibFunc___kmpc_alloc_shared && |
7095 | MaxHeapToStackSize != -1) { |
7096 | if (!Size || Size->ugt(RHS: MaxHeapToStackSize)) { |
7097 | LLVM_DEBUG({ |
7098 | if (!Size) |
7099 | dbgs() << "[H2S] Unknown allocation size: " << *AI.CB << "\n" ; |
7100 | else |
7101 | dbgs() << "[H2S] Allocation size too large: " << *AI.CB << " vs. " |
7102 | << MaxHeapToStackSize << "\n" ; |
7103 | }); |
7104 | |
7105 | AI.Status = AllocationInfo::INVALID; |
7106 | Changed = ChangeStatus::CHANGED; |
7107 | continue; |
7108 | } |
7109 | } |
7110 | |
7111 | switch (AI.Status) { |
7112 | case AllocationInfo::STACK_DUE_TO_USE: |
7113 | if (UsesCheck(AI)) |
7114 | break; |
7115 | AI.Status = AllocationInfo::STACK_DUE_TO_FREE; |
7116 | [[fallthrough]]; |
7117 | case AllocationInfo::STACK_DUE_TO_FREE: |
7118 | if (FreeCheck(AI)) |
7119 | break; |
7120 | AI.Status = AllocationInfo::INVALID; |
7121 | Changed = ChangeStatus::CHANGED; |
7122 | break; |
7123 | case AllocationInfo::INVALID: |
7124 | llvm_unreachable("Invalid allocations should never reach this point!" ); |
7125 | }; |
7126 | |
7127 | // Check if we still think we can move it into the entry block. If the |
7128 | // alloca comes from a converted __kmpc_alloc_shared then we can usually |
7129 | // ignore the potential compilations associated with loops. |
7130 | bool IsGlobalizedLocal = |
7131 | AI.LibraryFunctionId == LibFunc___kmpc_alloc_shared; |
7132 | if (AI.MoveAllocaIntoEntry && |
7133 | (!Size.has_value() || |
7134 | (!IsGlobalizedLocal && IsInLoop(*AI.CB->getParent())))) |
7135 | AI.MoveAllocaIntoEntry = false; |
7136 | } |
7137 | |
7138 | return Changed; |
7139 | } |
7140 | } // namespace |
7141 | |
7142 | /// ----------------------- Privatizable Pointers ------------------------------ |
7143 | namespace { |
7144 | struct AAPrivatizablePtrImpl : public AAPrivatizablePtr { |
7145 | AAPrivatizablePtrImpl(const IRPosition &IRP, Attributor &A) |
7146 | : AAPrivatizablePtr(IRP, A), PrivatizableType(std::nullopt) {} |
7147 | |
7148 | ChangeStatus indicatePessimisticFixpoint() override { |
7149 | AAPrivatizablePtr::indicatePessimisticFixpoint(); |
7150 | PrivatizableType = nullptr; |
7151 | return ChangeStatus::CHANGED; |
7152 | } |
7153 | |
7154 | /// Identify the type we can chose for a private copy of the underlying |
7155 | /// argument. std::nullopt means it is not clear yet, nullptr means there is |
7156 | /// none. |
7157 | virtual std::optional<Type *> identifyPrivatizableType(Attributor &A) = 0; |
7158 | |
7159 | /// Return a privatizable type that encloses both T0 and T1. |
7160 | /// TODO: This is merely a stub for now as we should manage a mapping as well. |
7161 | std::optional<Type *> combineTypes(std::optional<Type *> T0, |
7162 | std::optional<Type *> T1) { |
7163 | if (!T0) |
7164 | return T1; |
7165 | if (!T1) |
7166 | return T0; |
7167 | if (T0 == T1) |
7168 | return T0; |
7169 | return nullptr; |
7170 | } |
7171 | |
7172 | std::optional<Type *> getPrivatizableType() const override { |
7173 | return PrivatizableType; |
7174 | } |
7175 | |
7176 | const std::string getAsStr(Attributor *A) const override { |
7177 | return isAssumedPrivatizablePtr() ? "[priv]" : "[no-priv]" ; |
7178 | } |
7179 | |
7180 | protected: |
7181 | std::optional<Type *> PrivatizableType; |
7182 | }; |
7183 | |
7184 | // TODO: Do this for call site arguments (probably also other values) as well. |
7185 | |
7186 | struct AAPrivatizablePtrArgument final : public AAPrivatizablePtrImpl { |
7187 | AAPrivatizablePtrArgument(const IRPosition &IRP, Attributor &A) |
7188 | : AAPrivatizablePtrImpl(IRP, A) {} |
7189 | |
7190 | /// See AAPrivatizablePtrImpl::identifyPrivatizableType(...) |
7191 | std::optional<Type *> identifyPrivatizableType(Attributor &A) override { |
7192 | // If this is a byval argument and we know all the call sites (so we can |
7193 | // rewrite them), there is no need to check them explicitly. |
7194 | bool UsedAssumedInformation = false; |
7195 | SmallVector<Attribute, 1> Attrs; |
7196 | A.getAttrs(getIRPosition(), {Attribute::ByVal}, Attrs, |
7197 | /* IgnoreSubsumingPositions */ true); |
7198 | if (!Attrs.empty() && |
7199 | A.checkForAllCallSites(Pred: [](AbstractCallSite ACS) { return true; }, QueryingAA: *this, |
7200 | RequireAllCallSites: true, UsedAssumedInformation)) |
7201 | return Attrs[0].getValueAsType(); |
7202 | |
7203 | std::optional<Type *> Ty; |
7204 | unsigned ArgNo = getIRPosition().getCallSiteArgNo(); |
7205 | |
7206 | // Make sure the associated call site argument has the same type at all call |
7207 | // sites and it is an allocation we know is safe to privatize, for now that |
7208 | // means we only allow alloca instructions. |
7209 | // TODO: We can additionally analyze the accesses in the callee to create |
7210 | // the type from that information instead. That is a little more |
7211 | // involved and will be done in a follow up patch. |
7212 | auto CallSiteCheck = [&](AbstractCallSite ACS) { |
7213 | IRPosition ACSArgPos = IRPosition::callsite_argument(ACS, ArgNo); |
7214 | // Check if a coresponding argument was found or if it is one not |
7215 | // associated (which can happen for callback calls). |
7216 | if (ACSArgPos.getPositionKind() == IRPosition::IRP_INVALID) |
7217 | return false; |
7218 | |
7219 | // Check that all call sites agree on a type. |
7220 | auto *PrivCSArgAA = |
7221 | A.getAAFor<AAPrivatizablePtr>(QueryingAA: *this, IRP: ACSArgPos, DepClass: DepClassTy::REQUIRED); |
7222 | if (!PrivCSArgAA) |
7223 | return false; |
7224 | std::optional<Type *> CSTy = PrivCSArgAA->getPrivatizableType(); |
7225 | |
7226 | LLVM_DEBUG({ |
7227 | dbgs() << "[AAPrivatizablePtr] ACSPos: " << ACSArgPos << ", CSTy: " ; |
7228 | if (CSTy && *CSTy) |
7229 | (*CSTy)->print(dbgs()); |
7230 | else if (CSTy) |
7231 | dbgs() << "<nullptr>" ; |
7232 | else |
7233 | dbgs() << "<none>" ; |
7234 | }); |
7235 | |
7236 | Ty = combineTypes(T0: Ty, T1: CSTy); |
7237 | |
7238 | LLVM_DEBUG({ |
7239 | dbgs() << " : New Type: " ; |
7240 | if (Ty && *Ty) |
7241 | (*Ty)->print(dbgs()); |
7242 | else if (Ty) |
7243 | dbgs() << "<nullptr>" ; |
7244 | else |
7245 | dbgs() << "<none>" ; |
7246 | dbgs() << "\n" ; |
7247 | }); |
7248 | |
7249 | return !Ty || *Ty; |
7250 | }; |
7251 | |
7252 | if (!A.checkForAllCallSites(Pred: CallSiteCheck, QueryingAA: *this, RequireAllCallSites: true, |
7253 | UsedAssumedInformation)) |
7254 | return nullptr; |
7255 | return Ty; |
7256 | } |
7257 | |
7258 | /// See AbstractAttribute::updateImpl(...). |
7259 | ChangeStatus updateImpl(Attributor &A) override { |
7260 | PrivatizableType = identifyPrivatizableType(A); |
7261 | if (!PrivatizableType) |
7262 | return ChangeStatus::UNCHANGED; |
7263 | if (!*PrivatizableType) |
7264 | return indicatePessimisticFixpoint(); |
7265 | |
7266 | // The dependence is optional so we don't give up once we give up on the |
7267 | // alignment. |
7268 | A.getAAFor<AAAlign>(QueryingAA: *this, IRP: IRPosition::value(V: getAssociatedValue()), |
7269 | DepClass: DepClassTy::OPTIONAL); |
7270 | |
7271 | // Avoid arguments with padding for now. |
7272 | if (!A.hasAttr(getIRPosition(), Attribute::ByVal) && |
7273 | !isDenselyPacked(*PrivatizableType, A.getInfoCache().getDL())) { |
7274 | LLVM_DEBUG(dbgs() << "[AAPrivatizablePtr] Padding detected\n" ); |
7275 | return indicatePessimisticFixpoint(); |
7276 | } |
7277 | |
7278 | // Collect the types that will replace the privatizable type in the function |
7279 | // signature. |
7280 | SmallVector<Type *, 16> ReplacementTypes; |
7281 | identifyReplacementTypes(PrivType: *PrivatizableType, ReplacementTypes); |
7282 | |
7283 | // Verify callee and caller agree on how the promoted argument would be |
7284 | // passed. |
7285 | Function &Fn = *getIRPosition().getAnchorScope(); |
7286 | const auto *TTI = |
7287 | A.getInfoCache().getAnalysisResultForFunction<TargetIRAnalysis>(F: Fn); |
7288 | if (!TTI) { |
7289 | LLVM_DEBUG(dbgs() << "[AAPrivatizablePtr] Missing TTI for function " |
7290 | << Fn.getName() << "\n" ); |
7291 | return indicatePessimisticFixpoint(); |
7292 | } |
7293 | |
7294 | auto CallSiteCheck = [&](AbstractCallSite ACS) { |
7295 | CallBase *CB = ACS.getInstruction(); |
7296 | return TTI->areTypesABICompatible( |
7297 | Caller: CB->getCaller(), |
7298 | Callee: dyn_cast_if_present<Function>(Val: CB->getCalledOperand()), |
7299 | Types: ReplacementTypes); |
7300 | }; |
7301 | bool UsedAssumedInformation = false; |
7302 | if (!A.checkForAllCallSites(Pred: CallSiteCheck, QueryingAA: *this, RequireAllCallSites: true, |
7303 | UsedAssumedInformation)) { |
7304 | LLVM_DEBUG( |
7305 | dbgs() << "[AAPrivatizablePtr] ABI incompatibility detected for " |
7306 | << Fn.getName() << "\n" ); |
7307 | return indicatePessimisticFixpoint(); |
7308 | } |
7309 | |
7310 | // Register a rewrite of the argument. |
7311 | Argument *Arg = getAssociatedArgument(); |
7312 | if (!A.isValidFunctionSignatureRewrite(Arg&: *Arg, ReplacementTypes)) { |
7313 | LLVM_DEBUG(dbgs() << "[AAPrivatizablePtr] Rewrite not valid\n" ); |
7314 | return indicatePessimisticFixpoint(); |
7315 | } |
7316 | |
7317 | unsigned ArgNo = Arg->getArgNo(); |
7318 | |
7319 | // Helper to check if for the given call site the associated argument is |
7320 | // passed to a callback where the privatization would be different. |
7321 | auto IsCompatiblePrivArgOfCallback = [&](CallBase &CB) { |
7322 | SmallVector<const Use *, 4> CallbackUses; |
7323 | AbstractCallSite::getCallbackUses(CB, CallbackUses); |
7324 | for (const Use *U : CallbackUses) { |
7325 | AbstractCallSite CBACS(U); |
7326 | assert(CBACS && CBACS.isCallbackCall()); |
7327 | for (Argument &CBArg : CBACS.getCalledFunction()->args()) { |
7328 | int CBArgNo = CBACS.getCallArgOperandNo(Arg&: CBArg); |
7329 | |
7330 | LLVM_DEBUG({ |
7331 | dbgs() |
7332 | << "[AAPrivatizablePtr] Argument " << *Arg |
7333 | << "check if can be privatized in the context of its parent (" |
7334 | << Arg->getParent()->getName() |
7335 | << ")\n[AAPrivatizablePtr] because it is an argument in a " |
7336 | "callback (" |
7337 | << CBArgNo << "@" << CBACS.getCalledFunction()->getName() |
7338 | << ")\n[AAPrivatizablePtr] " << CBArg << " : " |
7339 | << CBACS.getCallArgOperand(CBArg) << " vs " |
7340 | << CB.getArgOperand(ArgNo) << "\n" |
7341 | << "[AAPrivatizablePtr] " << CBArg << " : " |
7342 | << CBACS.getCallArgOperandNo(CBArg) << " vs " << ArgNo << "\n" ; |
7343 | }); |
7344 | |
7345 | if (CBArgNo != int(ArgNo)) |
7346 | continue; |
7347 | const auto *CBArgPrivAA = A.getAAFor<AAPrivatizablePtr>( |
7348 | QueryingAA: *this, IRP: IRPosition::argument(Arg: CBArg), DepClass: DepClassTy::REQUIRED); |
7349 | if (CBArgPrivAA && CBArgPrivAA->isValidState()) { |
7350 | auto CBArgPrivTy = CBArgPrivAA->getPrivatizableType(); |
7351 | if (!CBArgPrivTy) |
7352 | continue; |
7353 | if (*CBArgPrivTy == PrivatizableType) |
7354 | continue; |
7355 | } |
7356 | |
7357 | LLVM_DEBUG({ |
7358 | dbgs() << "[AAPrivatizablePtr] Argument " << *Arg |
7359 | << " cannot be privatized in the context of its parent (" |
7360 | << Arg->getParent()->getName() |
7361 | << ")\n[AAPrivatizablePtr] because it is an argument in a " |
7362 | "callback (" |
7363 | << CBArgNo << "@" << CBACS.getCalledFunction()->getName() |
7364 | << ").\n[AAPrivatizablePtr] for which the argument " |
7365 | "privatization is not compatible.\n" ; |
7366 | }); |
7367 | return false; |
7368 | } |
7369 | } |
7370 | return true; |
7371 | }; |
7372 | |
7373 | // Helper to check if for the given call site the associated argument is |
7374 | // passed to a direct call where the privatization would be different. |
7375 | auto IsCompatiblePrivArgOfDirectCS = [&](AbstractCallSite ACS) { |
7376 | CallBase *DC = cast<CallBase>(Val: ACS.getInstruction()); |
7377 | int DCArgNo = ACS.getCallArgOperandNo(ArgNo); |
7378 | assert(DCArgNo >= 0 && unsigned(DCArgNo) < DC->arg_size() && |
7379 | "Expected a direct call operand for callback call operand" ); |
7380 | |
7381 | Function *DCCallee = |
7382 | dyn_cast_if_present<Function>(Val: DC->getCalledOperand()); |
7383 | LLVM_DEBUG({ |
7384 | dbgs() << "[AAPrivatizablePtr] Argument " << *Arg |
7385 | << " check if be privatized in the context of its parent (" |
7386 | << Arg->getParent()->getName() |
7387 | << ")\n[AAPrivatizablePtr] because it is an argument in a " |
7388 | "direct call of (" |
7389 | << DCArgNo << "@" << DCCallee->getName() << ").\n" ; |
7390 | }); |
7391 | |
7392 | if (unsigned(DCArgNo) < DCCallee->arg_size()) { |
7393 | const auto *DCArgPrivAA = A.getAAFor<AAPrivatizablePtr>( |
7394 | QueryingAA: *this, IRP: IRPosition::argument(Arg: *DCCallee->getArg(i: DCArgNo)), |
7395 | DepClass: DepClassTy::REQUIRED); |
7396 | if (DCArgPrivAA && DCArgPrivAA->isValidState()) { |
7397 | auto DCArgPrivTy = DCArgPrivAA->getPrivatizableType(); |
7398 | if (!DCArgPrivTy) |
7399 | return true; |
7400 | if (*DCArgPrivTy == PrivatizableType) |
7401 | return true; |
7402 | } |
7403 | } |
7404 | |
7405 | LLVM_DEBUG({ |
7406 | dbgs() << "[AAPrivatizablePtr] Argument " << *Arg |
7407 | << " cannot be privatized in the context of its parent (" |
7408 | << Arg->getParent()->getName() |
7409 | << ")\n[AAPrivatizablePtr] because it is an argument in a " |
7410 | "direct call of (" |
7411 | << ACS.getInstruction()->getCalledOperand()->getName() |
7412 | << ").\n[AAPrivatizablePtr] for which the argument " |
7413 | "privatization is not compatible.\n" ; |
7414 | }); |
7415 | return false; |
7416 | }; |
7417 | |
7418 | // Helper to check if the associated argument is used at the given abstract |
7419 | // call site in a way that is incompatible with the privatization assumed |
7420 | // here. |
7421 | auto IsCompatiblePrivArgOfOtherCallSite = [&](AbstractCallSite ACS) { |
7422 | if (ACS.isDirectCall()) |
7423 | return IsCompatiblePrivArgOfCallback(*ACS.getInstruction()); |
7424 | if (ACS.isCallbackCall()) |
7425 | return IsCompatiblePrivArgOfDirectCS(ACS); |
7426 | return false; |
7427 | }; |
7428 | |
7429 | if (!A.checkForAllCallSites(Pred: IsCompatiblePrivArgOfOtherCallSite, QueryingAA: *this, RequireAllCallSites: true, |
7430 | UsedAssumedInformation)) |
7431 | return indicatePessimisticFixpoint(); |
7432 | |
7433 | return ChangeStatus::UNCHANGED; |
7434 | } |
7435 | |
7436 | /// Given a type to private \p PrivType, collect the constituates (which are |
7437 | /// used) in \p ReplacementTypes. |
7438 | static void |
7439 | identifyReplacementTypes(Type *PrivType, |
7440 | SmallVectorImpl<Type *> &ReplacementTypes) { |
7441 | // TODO: For now we expand the privatization type to the fullest which can |
7442 | // lead to dead arguments that need to be removed later. |
7443 | assert(PrivType && "Expected privatizable type!" ); |
7444 | |
7445 | // Traverse the type, extract constituate types on the outermost level. |
7446 | if (auto *PrivStructType = dyn_cast<StructType>(Val: PrivType)) { |
7447 | for (unsigned u = 0, e = PrivStructType->getNumElements(); u < e; u++) |
7448 | ReplacementTypes.push_back(Elt: PrivStructType->getElementType(N: u)); |
7449 | } else if (auto *PrivArrayType = dyn_cast<ArrayType>(Val: PrivType)) { |
7450 | ReplacementTypes.append(NumInputs: PrivArrayType->getNumElements(), |
7451 | Elt: PrivArrayType->getElementType()); |
7452 | } else { |
7453 | ReplacementTypes.push_back(Elt: PrivType); |
7454 | } |
7455 | } |
7456 | |
7457 | /// Initialize \p Base according to the type \p PrivType at position \p IP. |
7458 | /// The values needed are taken from the arguments of \p F starting at |
7459 | /// position \p ArgNo. |
7460 | static void createInitialization(Type *PrivType, Value &Base, Function &F, |
7461 | unsigned ArgNo, BasicBlock::iterator IP) { |
7462 | assert(PrivType && "Expected privatizable type!" ); |
7463 | |
7464 | IRBuilder<NoFolder> IRB(IP->getParent(), IP); |
7465 | const DataLayout &DL = F.getParent()->getDataLayout(); |
7466 | |
7467 | // Traverse the type, build GEPs and stores. |
7468 | if (auto *PrivStructType = dyn_cast<StructType>(Val: PrivType)) { |
7469 | const StructLayout *PrivStructLayout = DL.getStructLayout(Ty: PrivStructType); |
7470 | for (unsigned u = 0, e = PrivStructType->getNumElements(); u < e; u++) { |
7471 | Value *Ptr = |
7472 | constructPointer(Ptr: &Base, Offset: PrivStructLayout->getElementOffset(Idx: u), IRB); |
7473 | new StoreInst(F.getArg(i: ArgNo + u), Ptr, IP); |
7474 | } |
7475 | } else if (auto *PrivArrayType = dyn_cast<ArrayType>(Val: PrivType)) { |
7476 | Type *PointeeTy = PrivArrayType->getElementType(); |
7477 | uint64_t PointeeTySize = DL.getTypeStoreSize(Ty: PointeeTy); |
7478 | for (unsigned u = 0, e = PrivArrayType->getNumElements(); u < e; u++) { |
7479 | Value *Ptr = constructPointer(Ptr: &Base, Offset: u * PointeeTySize, IRB); |
7480 | new StoreInst(F.getArg(i: ArgNo + u), Ptr, IP); |
7481 | } |
7482 | } else { |
7483 | new StoreInst(F.getArg(i: ArgNo), &Base, IP); |
7484 | } |
7485 | } |
7486 | |
7487 | /// Extract values from \p Base according to the type \p PrivType at the |
7488 | /// call position \p ACS. The values are appended to \p ReplacementValues. |
7489 | void createReplacementValues(Align Alignment, Type *PrivType, |
7490 | AbstractCallSite ACS, Value *Base, |
7491 | SmallVectorImpl<Value *> &ReplacementValues) { |
7492 | assert(Base && "Expected base value!" ); |
7493 | assert(PrivType && "Expected privatizable type!" ); |
7494 | Instruction *IP = ACS.getInstruction(); |
7495 | |
7496 | IRBuilder<NoFolder> IRB(IP); |
7497 | const DataLayout &DL = IP->getModule()->getDataLayout(); |
7498 | |
7499 | // Traverse the type, build GEPs and loads. |
7500 | if (auto *PrivStructType = dyn_cast<StructType>(Val: PrivType)) { |
7501 | const StructLayout *PrivStructLayout = DL.getStructLayout(Ty: PrivStructType); |
7502 | for (unsigned u = 0, e = PrivStructType->getNumElements(); u < e; u++) { |
7503 | Type *PointeeTy = PrivStructType->getElementType(N: u); |
7504 | Value *Ptr = |
7505 | constructPointer(Ptr: Base, Offset: PrivStructLayout->getElementOffset(Idx: u), IRB); |
7506 | LoadInst *L = new LoadInst(PointeeTy, Ptr, "" , IP->getIterator()); |
7507 | L->setAlignment(Alignment); |
7508 | ReplacementValues.push_back(Elt: L); |
7509 | } |
7510 | } else if (auto *PrivArrayType = dyn_cast<ArrayType>(Val: PrivType)) { |
7511 | Type *PointeeTy = PrivArrayType->getElementType(); |
7512 | uint64_t PointeeTySize = DL.getTypeStoreSize(Ty: PointeeTy); |
7513 | for (unsigned u = 0, e = PrivArrayType->getNumElements(); u < e; u++) { |
7514 | Value *Ptr = constructPointer(Ptr: Base, Offset: u * PointeeTySize, IRB); |
7515 | LoadInst *L = new LoadInst(PointeeTy, Ptr, "" , IP->getIterator()); |
7516 | L->setAlignment(Alignment); |
7517 | ReplacementValues.push_back(Elt: L); |
7518 | } |
7519 | } else { |
7520 | LoadInst *L = new LoadInst(PrivType, Base, "" , IP->getIterator()); |
7521 | L->setAlignment(Alignment); |
7522 | ReplacementValues.push_back(Elt: L); |
7523 | } |
7524 | } |
7525 | |
7526 | /// See AbstractAttribute::manifest(...) |
7527 | ChangeStatus manifest(Attributor &A) override { |
7528 | if (!PrivatizableType) |
7529 | return ChangeStatus::UNCHANGED; |
7530 | assert(*PrivatizableType && "Expected privatizable type!" ); |
7531 | |
7532 | // Collect all tail calls in the function as we cannot allow new allocas to |
7533 | // escape into tail recursion. |
7534 | // TODO: Be smarter about new allocas escaping into tail calls. |
7535 | SmallVector<CallInst *, 16> TailCalls; |
7536 | bool UsedAssumedInformation = false; |
7537 | if (!A.checkForAllInstructions( |
7538 | Pred: [&](Instruction &I) { |
7539 | CallInst &CI = cast<CallInst>(Val&: I); |
7540 | if (CI.isTailCall()) |
7541 | TailCalls.push_back(Elt: &CI); |
7542 | return true; |
7543 | }, |
7544 | QueryingAA: *this, Opcodes: {Instruction::Call}, UsedAssumedInformation)) |
7545 | return ChangeStatus::UNCHANGED; |
7546 | |
7547 | Argument *Arg = getAssociatedArgument(); |
7548 | // Query AAAlign attribute for alignment of associated argument to |
7549 | // determine the best alignment of loads. |
7550 | const auto *AlignAA = |
7551 | A.getAAFor<AAAlign>(QueryingAA: *this, IRP: IRPosition::value(V: *Arg), DepClass: DepClassTy::NONE); |
7552 | |
7553 | // Callback to repair the associated function. A new alloca is placed at the |
7554 | // beginning and initialized with the values passed through arguments. The |
7555 | // new alloca replaces the use of the old pointer argument. |
7556 | Attributor::ArgumentReplacementInfo::CalleeRepairCBTy FnRepairCB = |
7557 | [=](const Attributor::ArgumentReplacementInfo &ARI, |
7558 | Function &ReplacementFn, Function::arg_iterator ArgIt) { |
7559 | BasicBlock &EntryBB = ReplacementFn.getEntryBlock(); |
7560 | BasicBlock::iterator IP = EntryBB.getFirstInsertionPt(); |
7561 | const DataLayout &DL = IP->getModule()->getDataLayout(); |
7562 | unsigned AS = DL.getAllocaAddrSpace(); |
7563 | Instruction *AI = new AllocaInst(*PrivatizableType, AS, |
7564 | Arg->getName() + ".priv" , IP); |
7565 | createInitialization(PrivType: *PrivatizableType, Base&: *AI, F&: ReplacementFn, |
7566 | ArgNo: ArgIt->getArgNo(), IP); |
7567 | |
7568 | if (AI->getType() != Arg->getType()) |
7569 | AI = BitCastInst::CreatePointerBitCastOrAddrSpaceCast( |
7570 | S: AI, Ty: Arg->getType(), Name: "" , InsertBefore: IP); |
7571 | Arg->replaceAllUsesWith(V: AI); |
7572 | |
7573 | for (CallInst *CI : TailCalls) |
7574 | CI->setTailCall(false); |
7575 | }; |
7576 | |
7577 | // Callback to repair a call site of the associated function. The elements |
7578 | // of the privatizable type are loaded prior to the call and passed to the |
7579 | // new function version. |
7580 | Attributor::ArgumentReplacementInfo::ACSRepairCBTy ACSRepairCB = |
7581 | [=](const Attributor::ArgumentReplacementInfo &ARI, |
7582 | AbstractCallSite ACS, SmallVectorImpl<Value *> &NewArgOperands) { |
7583 | // When no alignment is specified for the load instruction, |
7584 | // natural alignment is assumed. |
7585 | createReplacementValues( |
7586 | Alignment: AlignAA ? AlignAA->getAssumedAlign() : Align(0), |
7587 | PrivType: *PrivatizableType, ACS, |
7588 | Base: ACS.getCallArgOperand(ArgNo: ARI.getReplacedArg().getArgNo()), |
7589 | ReplacementValues&: NewArgOperands); |
7590 | }; |
7591 | |
7592 | // Collect the types that will replace the privatizable type in the function |
7593 | // signature. |
7594 | SmallVector<Type *, 16> ReplacementTypes; |
7595 | identifyReplacementTypes(PrivType: *PrivatizableType, ReplacementTypes); |
7596 | |
7597 | // Register a rewrite of the argument. |
7598 | if (A.registerFunctionSignatureRewrite(Arg&: *Arg, ReplacementTypes, |
7599 | CalleeRepairCB: std::move(FnRepairCB), |
7600 | ACSRepairCB: std::move(ACSRepairCB))) |
7601 | return ChangeStatus::CHANGED; |
7602 | return ChangeStatus::UNCHANGED; |
7603 | } |
7604 | |
7605 | /// See AbstractAttribute::trackStatistics() |
7606 | void trackStatistics() const override { |
7607 | STATS_DECLTRACK_ARG_ATTR(privatizable_ptr); |
7608 | } |
7609 | }; |
7610 | |
7611 | struct AAPrivatizablePtrFloating : public AAPrivatizablePtrImpl { |
7612 | AAPrivatizablePtrFloating(const IRPosition &IRP, Attributor &A) |
7613 | : AAPrivatizablePtrImpl(IRP, A) {} |
7614 | |
7615 | /// See AbstractAttribute::initialize(...). |
7616 | void initialize(Attributor &A) override { |
7617 | // TODO: We can privatize more than arguments. |
7618 | indicatePessimisticFixpoint(); |
7619 | } |
7620 | |
7621 | ChangeStatus updateImpl(Attributor &A) override { |
7622 | llvm_unreachable("AAPrivatizablePtr(Floating|Returned|CallSiteReturned)::" |
7623 | "updateImpl will not be called" ); |
7624 | } |
7625 | |
7626 | /// See AAPrivatizablePtrImpl::identifyPrivatizableType(...) |
7627 | std::optional<Type *> identifyPrivatizableType(Attributor &A) override { |
7628 | Value *Obj = getUnderlyingObject(V: &getAssociatedValue()); |
7629 | if (!Obj) { |
7630 | LLVM_DEBUG(dbgs() << "[AAPrivatizablePtr] No underlying object found!\n" ); |
7631 | return nullptr; |
7632 | } |
7633 | |
7634 | if (auto *AI = dyn_cast<AllocaInst>(Val: Obj)) |
7635 | if (auto *CI = dyn_cast<ConstantInt>(Val: AI->getArraySize())) |
7636 | if (CI->isOne()) |
7637 | return AI->getAllocatedType(); |
7638 | if (auto *Arg = dyn_cast<Argument>(Val: Obj)) { |
7639 | auto *PrivArgAA = A.getAAFor<AAPrivatizablePtr>( |
7640 | QueryingAA: *this, IRP: IRPosition::argument(Arg: *Arg), DepClass: DepClassTy::REQUIRED); |
7641 | if (PrivArgAA && PrivArgAA->isAssumedPrivatizablePtr()) |
7642 | return PrivArgAA->getPrivatizableType(); |
7643 | } |
7644 | |
7645 | LLVM_DEBUG(dbgs() << "[AAPrivatizablePtr] Underlying object neither valid " |
7646 | "alloca nor privatizable argument: " |
7647 | << *Obj << "!\n" ); |
7648 | return nullptr; |
7649 | } |
7650 | |
7651 | /// See AbstractAttribute::trackStatistics() |
7652 | void trackStatistics() const override { |
7653 | STATS_DECLTRACK_FLOATING_ATTR(privatizable_ptr); |
7654 | } |
7655 | }; |
7656 | |
7657 | struct AAPrivatizablePtrCallSiteArgument final |
7658 | : public AAPrivatizablePtrFloating { |
7659 | AAPrivatizablePtrCallSiteArgument(const IRPosition &IRP, Attributor &A) |
7660 | : AAPrivatizablePtrFloating(IRP, A) {} |
7661 | |
7662 | /// See AbstractAttribute::initialize(...). |
7663 | void initialize(Attributor &A) override { |
7664 | if (A.hasAttr(getIRPosition(), Attribute::ByVal)) |
7665 | indicateOptimisticFixpoint(); |
7666 | } |
7667 | |
7668 | /// See AbstractAttribute::updateImpl(...). |
7669 | ChangeStatus updateImpl(Attributor &A) override { |
7670 | PrivatizableType = identifyPrivatizableType(A); |
7671 | if (!PrivatizableType) |
7672 | return ChangeStatus::UNCHANGED; |
7673 | if (!*PrivatizableType) |
7674 | return indicatePessimisticFixpoint(); |
7675 | |
7676 | const IRPosition &IRP = getIRPosition(); |
7677 | bool IsKnownNoCapture; |
7678 | bool IsAssumedNoCapture = AA::hasAssumedIRAttr<Attribute::NoCapture>( |
7679 | A, this, IRP, DepClassTy::REQUIRED, IsKnownNoCapture); |
7680 | if (!IsAssumedNoCapture) { |
7681 | LLVM_DEBUG(dbgs() << "[AAPrivatizablePtr] pointer might be captured!\n" ); |
7682 | return indicatePessimisticFixpoint(); |
7683 | } |
7684 | |
7685 | bool IsKnownNoAlias; |
7686 | if (!AA::hasAssumedIRAttr<Attribute::NoAlias>( |
7687 | A, this, IRP, DepClassTy::REQUIRED, IsKnownNoAlias)) { |
7688 | LLVM_DEBUG(dbgs() << "[AAPrivatizablePtr] pointer might alias!\n" ); |
7689 | return indicatePessimisticFixpoint(); |
7690 | } |
7691 | |
7692 | bool IsKnown; |
7693 | if (!AA::isAssumedReadOnly(A, IRP, QueryingAA: *this, IsKnown)) { |
7694 | LLVM_DEBUG(dbgs() << "[AAPrivatizablePtr] pointer is written!\n" ); |
7695 | return indicatePessimisticFixpoint(); |
7696 | } |
7697 | |
7698 | return ChangeStatus::UNCHANGED; |
7699 | } |
7700 | |
7701 | /// See AbstractAttribute::trackStatistics() |
7702 | void trackStatistics() const override { |
7703 | STATS_DECLTRACK_CSARG_ATTR(privatizable_ptr); |
7704 | } |
7705 | }; |
7706 | |
7707 | struct AAPrivatizablePtrCallSiteReturned final |
7708 | : public AAPrivatizablePtrFloating { |
7709 | AAPrivatizablePtrCallSiteReturned(const IRPosition &IRP, Attributor &A) |
7710 | : AAPrivatizablePtrFloating(IRP, A) {} |
7711 | |
7712 | /// See AbstractAttribute::initialize(...). |
7713 | void initialize(Attributor &A) override { |
7714 | // TODO: We can privatize more than arguments. |
7715 | indicatePessimisticFixpoint(); |
7716 | } |
7717 | |
7718 | /// See AbstractAttribute::trackStatistics() |
7719 | void trackStatistics() const override { |
7720 | STATS_DECLTRACK_CSRET_ATTR(privatizable_ptr); |
7721 | } |
7722 | }; |
7723 | |
7724 | struct AAPrivatizablePtrReturned final : public AAPrivatizablePtrFloating { |
7725 | AAPrivatizablePtrReturned(const IRPosition &IRP, Attributor &A) |
7726 | : AAPrivatizablePtrFloating(IRP, A) {} |
7727 | |
7728 | /// See AbstractAttribute::initialize(...). |
7729 | void initialize(Attributor &A) override { |
7730 | // TODO: We can privatize more than arguments. |
7731 | indicatePessimisticFixpoint(); |
7732 | } |
7733 | |
7734 | /// See AbstractAttribute::trackStatistics() |
7735 | void trackStatistics() const override { |
7736 | STATS_DECLTRACK_FNRET_ATTR(privatizable_ptr); |
7737 | } |
7738 | }; |
7739 | } // namespace |
7740 | |
7741 | /// -------------------- Memory Behavior Attributes ---------------------------- |
7742 | /// Includes read-none, read-only, and write-only. |
7743 | /// ---------------------------------------------------------------------------- |
7744 | namespace { |
7745 | struct AAMemoryBehaviorImpl : public AAMemoryBehavior { |
7746 | AAMemoryBehaviorImpl(const IRPosition &IRP, Attributor &A) |
7747 | : AAMemoryBehavior(IRP, A) {} |
7748 | |
7749 | /// See AbstractAttribute::initialize(...). |
7750 | void initialize(Attributor &A) override { |
7751 | intersectAssumedBits(BitsEncoding: BEST_STATE); |
7752 | getKnownStateFromValue(A, IRP: getIRPosition(), State&: getState()); |
7753 | AAMemoryBehavior::initialize(A); |
7754 | } |
7755 | |
7756 | /// Return the memory behavior information encoded in the IR for \p IRP. |
7757 | static void getKnownStateFromValue(Attributor &A, const IRPosition &IRP, |
7758 | BitIntegerState &State, |
7759 | bool IgnoreSubsumingPositions = false) { |
7760 | SmallVector<Attribute, 2> Attrs; |
7761 | A.getAttrs(IRP, AKs: AttrKinds, Attrs, IgnoreSubsumingPositions); |
7762 | for (const Attribute &Attr : Attrs) { |
7763 | switch (Attr.getKindAsEnum()) { |
7764 | case Attribute::ReadNone: |
7765 | State.addKnownBits(Bits: NO_ACCESSES); |
7766 | break; |
7767 | case Attribute::ReadOnly: |
7768 | State.addKnownBits(Bits: NO_WRITES); |
7769 | break; |
7770 | case Attribute::WriteOnly: |
7771 | State.addKnownBits(Bits: NO_READS); |
7772 | break; |
7773 | default: |
7774 | llvm_unreachable("Unexpected attribute!" ); |
7775 | } |
7776 | } |
7777 | |
7778 | if (auto *I = dyn_cast<Instruction>(Val: &IRP.getAnchorValue())) { |
7779 | if (!I->mayReadFromMemory()) |
7780 | State.addKnownBits(Bits: NO_READS); |
7781 | if (!I->mayWriteToMemory()) |
7782 | State.addKnownBits(Bits: NO_WRITES); |
7783 | } |
7784 | } |
7785 | |
7786 | /// See AbstractAttribute::getDeducedAttributes(...). |
7787 | void getDeducedAttributes(Attributor &A, LLVMContext &Ctx, |
7788 | SmallVectorImpl<Attribute> &Attrs) const override { |
7789 | assert(Attrs.size() == 0); |
7790 | if (isAssumedReadNone()) |
7791 | Attrs.push_back(Attribute::get(Ctx, Attribute::ReadNone)); |
7792 | else if (isAssumedReadOnly()) |
7793 | Attrs.push_back(Attribute::get(Ctx, Attribute::ReadOnly)); |
7794 | else if (isAssumedWriteOnly()) |
7795 | Attrs.push_back(Attribute::get(Ctx, Attribute::WriteOnly)); |
7796 | assert(Attrs.size() <= 1); |
7797 | } |
7798 | |
7799 | /// See AbstractAttribute::manifest(...). |
7800 | ChangeStatus manifest(Attributor &A) override { |
7801 | const IRPosition &IRP = getIRPosition(); |
7802 | |
7803 | if (A.hasAttr(IRP, Attribute::ReadNone, |
7804 | /* IgnoreSubsumingPositions */ true)) |
7805 | return ChangeStatus::UNCHANGED; |
7806 | |
7807 | // Check if we would improve the existing attributes first. |
7808 | SmallVector<Attribute, 4> DeducedAttrs; |
7809 | getDeducedAttributes(A, Ctx&: IRP.getAnchorValue().getContext(), Attrs&: DeducedAttrs); |
7810 | if (llvm::all_of(Range&: DeducedAttrs, P: [&](const Attribute &Attr) { |
7811 | return A.hasAttr(IRP, AKs: Attr.getKindAsEnum(), |
7812 | /* IgnoreSubsumingPositions */ true); |
7813 | })) |
7814 | return ChangeStatus::UNCHANGED; |
7815 | |
7816 | // Clear existing attributes. |
7817 | A.removeAttrs(IRP, AttrKinds); |
7818 | // Clear conflicting writable attribute. |
7819 | if (isAssumedReadOnly()) |
7820 | A.removeAttrs(IRP, Attribute::Writable); |
7821 | |
7822 | // Use the generic manifest method. |
7823 | return IRAttribute::manifest(A); |
7824 | } |
7825 | |
7826 | /// See AbstractState::getAsStr(). |
7827 | const std::string getAsStr(Attributor *A) const override { |
7828 | if (isAssumedReadNone()) |
7829 | return "readnone" ; |
7830 | if (isAssumedReadOnly()) |
7831 | return "readonly" ; |
7832 | if (isAssumedWriteOnly()) |
7833 | return "writeonly" ; |
7834 | return "may-read/write" ; |
7835 | } |
7836 | |
7837 | /// The set of IR attributes AAMemoryBehavior deals with. |
7838 | static const Attribute::AttrKind AttrKinds[3]; |
7839 | }; |
7840 | |
7841 | const Attribute::AttrKind AAMemoryBehaviorImpl::AttrKinds[] = { |
7842 | Attribute::ReadNone, Attribute::ReadOnly, Attribute::WriteOnly}; |
7843 | |
7844 | /// Memory behavior attribute for a floating value. |
7845 | struct AAMemoryBehaviorFloating : AAMemoryBehaviorImpl { |
7846 | AAMemoryBehaviorFloating(const IRPosition &IRP, Attributor &A) |
7847 | : AAMemoryBehaviorImpl(IRP, A) {} |
7848 | |
7849 | /// See AbstractAttribute::updateImpl(...). |
7850 | ChangeStatus updateImpl(Attributor &A) override; |
7851 | |
7852 | /// See AbstractAttribute::trackStatistics() |
7853 | void trackStatistics() const override { |
7854 | if (isAssumedReadNone()) |
7855 | STATS_DECLTRACK_FLOATING_ATTR(readnone) |
7856 | else if (isAssumedReadOnly()) |
7857 | STATS_DECLTRACK_FLOATING_ATTR(readonly) |
7858 | else if (isAssumedWriteOnly()) |
7859 | STATS_DECLTRACK_FLOATING_ATTR(writeonly) |
7860 | } |
7861 | |
7862 | private: |
7863 | /// Return true if users of \p UserI might access the underlying |
7864 | /// variable/location described by \p U and should therefore be analyzed. |
7865 | bool followUsersOfUseIn(Attributor &A, const Use &U, |
7866 | const Instruction *UserI); |
7867 | |
7868 | /// Update the state according to the effect of use \p U in \p UserI. |
7869 | void analyzeUseIn(Attributor &A, const Use &U, const Instruction *UserI); |
7870 | }; |
7871 | |
7872 | /// Memory behavior attribute for function argument. |
7873 | struct AAMemoryBehaviorArgument : AAMemoryBehaviorFloating { |
7874 | AAMemoryBehaviorArgument(const IRPosition &IRP, Attributor &A) |
7875 | : AAMemoryBehaviorFloating(IRP, A) {} |
7876 | |
7877 | /// See AbstractAttribute::initialize(...). |
7878 | void initialize(Attributor &A) override { |
7879 | intersectAssumedBits(BitsEncoding: BEST_STATE); |
7880 | const IRPosition &IRP = getIRPosition(); |
7881 | // TODO: Make IgnoreSubsumingPositions a property of an IRAttribute so we |
7882 | // can query it when we use has/getAttr. That would allow us to reuse the |
7883 | // initialize of the base class here. |
7884 | bool HasByVal = A.hasAttr(IRP, {Attribute::ByVal}, |
7885 | /* IgnoreSubsumingPositions */ true); |
7886 | getKnownStateFromValue(A, IRP, State&: getState(), |
7887 | /* IgnoreSubsumingPositions */ HasByVal); |
7888 | } |
7889 | |
7890 | ChangeStatus manifest(Attributor &A) override { |
7891 | // TODO: Pointer arguments are not supported on vectors of pointers yet. |
7892 | if (!getAssociatedValue().getType()->isPointerTy()) |
7893 | return ChangeStatus::UNCHANGED; |
7894 | |
7895 | // TODO: From readattrs.ll: "inalloca parameters are always |
7896 | // considered written" |
7897 | if (A.hasAttr(getIRPosition(), |
7898 | {Attribute::InAlloca, Attribute::Preallocated})) { |
7899 | removeKnownBits(BitsEncoding: NO_WRITES); |
7900 | removeAssumedBits(BitsEncoding: NO_WRITES); |
7901 | } |
7902 | A.removeAttrs(IRP: getIRPosition(), AttrKinds); |
7903 | return AAMemoryBehaviorFloating::manifest(A); |
7904 | } |
7905 | |
7906 | /// See AbstractAttribute::trackStatistics() |
7907 | void trackStatistics() const override { |
7908 | if (isAssumedReadNone()) |
7909 | STATS_DECLTRACK_ARG_ATTR(readnone) |
7910 | else if (isAssumedReadOnly()) |
7911 | STATS_DECLTRACK_ARG_ATTR(readonly) |
7912 | else if (isAssumedWriteOnly()) |
7913 | STATS_DECLTRACK_ARG_ATTR(writeonly) |
7914 | } |
7915 | }; |
7916 | |
7917 | struct AAMemoryBehaviorCallSiteArgument final : AAMemoryBehaviorArgument { |
7918 | AAMemoryBehaviorCallSiteArgument(const IRPosition &IRP, Attributor &A) |
7919 | : AAMemoryBehaviorArgument(IRP, A) {} |
7920 | |
7921 | /// See AbstractAttribute::initialize(...). |
7922 | void initialize(Attributor &A) override { |
7923 | // If we don't have an associated attribute this is either a variadic call |
7924 | // or an indirect call, either way, nothing to do here. |
7925 | Argument *Arg = getAssociatedArgument(); |
7926 | if (!Arg) { |
7927 | indicatePessimisticFixpoint(); |
7928 | return; |
7929 | } |
7930 | if (Arg->hasByValAttr()) { |
7931 | addKnownBits(Bits: NO_WRITES); |
7932 | removeKnownBits(BitsEncoding: NO_READS); |
7933 | removeAssumedBits(BitsEncoding: NO_READS); |
7934 | } |
7935 | AAMemoryBehaviorArgument::initialize(A); |
7936 | if (getAssociatedFunction()->isDeclaration()) |
7937 | indicatePessimisticFixpoint(); |
7938 | } |
7939 | |
7940 | /// See AbstractAttribute::updateImpl(...). |
7941 | ChangeStatus updateImpl(Attributor &A) override { |
7942 | // TODO: Once we have call site specific value information we can provide |
7943 | // call site specific liveness liveness information and then it makes |
7944 | // sense to specialize attributes for call sites arguments instead of |
7945 | // redirecting requests to the callee argument. |
7946 | Argument *Arg = getAssociatedArgument(); |
7947 | const IRPosition &ArgPos = IRPosition::argument(Arg: *Arg); |
7948 | auto *ArgAA = |
7949 | A.getAAFor<AAMemoryBehavior>(QueryingAA: *this, IRP: ArgPos, DepClass: DepClassTy::REQUIRED); |
7950 | if (!ArgAA) |
7951 | return indicatePessimisticFixpoint(); |
7952 | return clampStateAndIndicateChange(S&: getState(), R: ArgAA->getState()); |
7953 | } |
7954 | |
7955 | /// See AbstractAttribute::trackStatistics() |
7956 | void trackStatistics() const override { |
7957 | if (isAssumedReadNone()) |
7958 | STATS_DECLTRACK_CSARG_ATTR(readnone) |
7959 | else if (isAssumedReadOnly()) |
7960 | STATS_DECLTRACK_CSARG_ATTR(readonly) |
7961 | else if (isAssumedWriteOnly()) |
7962 | STATS_DECLTRACK_CSARG_ATTR(writeonly) |
7963 | } |
7964 | }; |
7965 | |
7966 | /// Memory behavior attribute for a call site return position. |
7967 | struct AAMemoryBehaviorCallSiteReturned final : AAMemoryBehaviorFloating { |
7968 | AAMemoryBehaviorCallSiteReturned(const IRPosition &IRP, Attributor &A) |
7969 | : AAMemoryBehaviorFloating(IRP, A) {} |
7970 | |
7971 | /// See AbstractAttribute::initialize(...). |
7972 | void initialize(Attributor &A) override { |
7973 | AAMemoryBehaviorImpl::initialize(A); |
7974 | } |
7975 | /// See AbstractAttribute::manifest(...). |
7976 | ChangeStatus manifest(Attributor &A) override { |
7977 | // We do not annotate returned values. |
7978 | return ChangeStatus::UNCHANGED; |
7979 | } |
7980 | |
7981 | /// See AbstractAttribute::trackStatistics() |
7982 | void trackStatistics() const override {} |
7983 | }; |
7984 | |
7985 | /// An AA to represent the memory behavior function attributes. |
7986 | struct AAMemoryBehaviorFunction final : public AAMemoryBehaviorImpl { |
7987 | AAMemoryBehaviorFunction(const IRPosition &IRP, Attributor &A) |
7988 | : AAMemoryBehaviorImpl(IRP, A) {} |
7989 | |
7990 | /// See AbstractAttribute::updateImpl(Attributor &A). |
7991 | ChangeStatus updateImpl(Attributor &A) override; |
7992 | |
7993 | /// See AbstractAttribute::manifest(...). |
7994 | ChangeStatus manifest(Attributor &A) override { |
7995 | // TODO: It would be better to merge this with AAMemoryLocation, so that |
7996 | // we could determine read/write per location. This would also have the |
7997 | // benefit of only one place trying to manifest the memory attribute. |
7998 | Function &F = cast<Function>(Val&: getAnchorValue()); |
7999 | MemoryEffects ME = MemoryEffects::unknown(); |
8000 | if (isAssumedReadNone()) |
8001 | ME = MemoryEffects::none(); |
8002 | else if (isAssumedReadOnly()) |
8003 | ME = MemoryEffects::readOnly(); |
8004 | else if (isAssumedWriteOnly()) |
8005 | ME = MemoryEffects::writeOnly(); |
8006 | |
8007 | A.removeAttrs(IRP: getIRPosition(), AttrKinds); |
8008 | // Clear conflicting writable attribute. |
8009 | if (ME.onlyReadsMemory()) |
8010 | for (Argument &Arg : F.args()) |
8011 | A.removeAttrs(IRPosition::argument(Arg), Attribute::Writable); |
8012 | return A.manifestAttrs(IRP: getIRPosition(), |
8013 | DeducedAttrs: Attribute::getWithMemoryEffects(Context&: F.getContext(), ME)); |
8014 | } |
8015 | |
8016 | /// See AbstractAttribute::trackStatistics() |
8017 | void trackStatistics() const override { |
8018 | if (isAssumedReadNone()) |
8019 | STATS_DECLTRACK_FN_ATTR(readnone) |
8020 | else if (isAssumedReadOnly()) |
8021 | STATS_DECLTRACK_FN_ATTR(readonly) |
8022 | else if (isAssumedWriteOnly()) |
8023 | STATS_DECLTRACK_FN_ATTR(writeonly) |
8024 | } |
8025 | }; |
8026 | |
8027 | /// AAMemoryBehavior attribute for call sites. |
8028 | struct AAMemoryBehaviorCallSite final |
8029 | : AACalleeToCallSite<AAMemoryBehavior, AAMemoryBehaviorImpl> { |
8030 | AAMemoryBehaviorCallSite(const IRPosition &IRP, Attributor &A) |
8031 | : AACalleeToCallSite<AAMemoryBehavior, AAMemoryBehaviorImpl>(IRP, A) {} |
8032 | |
8033 | /// See AbstractAttribute::manifest(...). |
8034 | ChangeStatus manifest(Attributor &A) override { |
8035 | // TODO: Deduplicate this with AAMemoryBehaviorFunction. |
8036 | CallBase &CB = cast<CallBase>(Val&: getAnchorValue()); |
8037 | MemoryEffects ME = MemoryEffects::unknown(); |
8038 | if (isAssumedReadNone()) |
8039 | ME = MemoryEffects::none(); |
8040 | else if (isAssumedReadOnly()) |
8041 | ME = MemoryEffects::readOnly(); |
8042 | else if (isAssumedWriteOnly()) |
8043 | ME = MemoryEffects::writeOnly(); |
8044 | |
8045 | A.removeAttrs(IRP: getIRPosition(), AttrKinds); |
8046 | // Clear conflicting writable attribute. |
8047 | if (ME.onlyReadsMemory()) |
8048 | for (Use &U : CB.args()) |
8049 | A.removeAttrs(IRPosition::callsite_argument(CB, U.getOperandNo()), |
8050 | Attribute::Writable); |
8051 | return A.manifestAttrs( |
8052 | IRP: getIRPosition(), DeducedAttrs: Attribute::getWithMemoryEffects(Context&: CB.getContext(), ME)); |
8053 | } |
8054 | |
8055 | /// See AbstractAttribute::trackStatistics() |
8056 | void trackStatistics() const override { |
8057 | if (isAssumedReadNone()) |
8058 | STATS_DECLTRACK_CS_ATTR(readnone) |
8059 | else if (isAssumedReadOnly()) |
8060 | STATS_DECLTRACK_CS_ATTR(readonly) |
8061 | else if (isAssumedWriteOnly()) |
8062 | STATS_DECLTRACK_CS_ATTR(writeonly) |
8063 | } |
8064 | }; |
8065 | |
8066 | ChangeStatus AAMemoryBehaviorFunction::updateImpl(Attributor &A) { |
8067 | |
8068 | // The current assumed state used to determine a change. |
8069 | auto AssumedState = getAssumed(); |
8070 | |
8071 | auto CheckRWInst = [&](Instruction &I) { |
8072 | // If the instruction has an own memory behavior state, use it to restrict |
8073 | // the local state. No further analysis is required as the other memory |
8074 | // state is as optimistic as it gets. |
8075 | if (const auto *CB = dyn_cast<CallBase>(Val: &I)) { |
8076 | const auto *MemBehaviorAA = A.getAAFor<AAMemoryBehavior>( |
8077 | QueryingAA: *this, IRP: IRPosition::callsite_function(CB: *CB), DepClass: DepClassTy::REQUIRED); |
8078 | if (MemBehaviorAA) { |
8079 | intersectAssumedBits(BitsEncoding: MemBehaviorAA->getAssumed()); |
8080 | return !isAtFixpoint(); |
8081 | } |
8082 | } |
8083 | |
8084 | // Remove access kind modifiers if necessary. |
8085 | if (I.mayReadFromMemory()) |
8086 | removeAssumedBits(BitsEncoding: NO_READS); |
8087 | if (I.mayWriteToMemory()) |
8088 | removeAssumedBits(BitsEncoding: NO_WRITES); |
8089 | return !isAtFixpoint(); |
8090 | }; |
8091 | |
8092 | bool UsedAssumedInformation = false; |
8093 | if (!A.checkForAllReadWriteInstructions(Pred: CheckRWInst, QueryingAA&: *this, |
8094 | UsedAssumedInformation)) |
8095 | return indicatePessimisticFixpoint(); |
8096 | |
8097 | return (AssumedState != getAssumed()) ? ChangeStatus::CHANGED |
8098 | : ChangeStatus::UNCHANGED; |
8099 | } |
8100 | |
8101 | ChangeStatus AAMemoryBehaviorFloating::updateImpl(Attributor &A) { |
8102 | |
8103 | const IRPosition &IRP = getIRPosition(); |
8104 | const IRPosition &FnPos = IRPosition::function_scope(IRP); |
8105 | AAMemoryBehavior::StateType &S = getState(); |
8106 | |
8107 | // First, check the function scope. We take the known information and we avoid |
8108 | // work if the assumed information implies the current assumed information for |
8109 | // this attribute. This is a valid for all but byval arguments. |
8110 | Argument *Arg = IRP.getAssociatedArgument(); |
8111 | AAMemoryBehavior::base_t FnMemAssumedState = |
8112 | AAMemoryBehavior::StateType::getWorstState(); |
8113 | if (!Arg || !Arg->hasByValAttr()) { |
8114 | const auto *FnMemAA = |
8115 | A.getAAFor<AAMemoryBehavior>(QueryingAA: *this, IRP: FnPos, DepClass: DepClassTy::OPTIONAL); |
8116 | if (FnMemAA) { |
8117 | FnMemAssumedState = FnMemAA->getAssumed(); |
8118 | S.addKnownBits(Bits: FnMemAA->getKnown()); |
8119 | if ((S.getAssumed() & FnMemAA->getAssumed()) == S.getAssumed()) |
8120 | return ChangeStatus::UNCHANGED; |
8121 | } |
8122 | } |
8123 | |
8124 | // The current assumed state used to determine a change. |
8125 | auto AssumedState = S.getAssumed(); |
8126 | |
8127 | // Make sure the value is not captured (except through "return"), if |
8128 | // it is, any information derived would be irrelevant anyway as we cannot |
8129 | // check the potential aliases introduced by the capture. However, no need |
8130 | // to fall back to anythign less optimistic than the function state. |
8131 | bool IsKnownNoCapture; |
8132 | const AANoCapture *ArgNoCaptureAA = nullptr; |
8133 | bool IsAssumedNoCapture = AA::hasAssumedIRAttr<Attribute::NoCapture>( |
8134 | A, this, IRP, DepClassTy::OPTIONAL, IsKnownNoCapture, false, |
8135 | &ArgNoCaptureAA); |
8136 | |
8137 | if (!IsAssumedNoCapture && |
8138 | (!ArgNoCaptureAA || !ArgNoCaptureAA->isAssumedNoCaptureMaybeReturned())) { |
8139 | S.intersectAssumedBits(BitsEncoding: FnMemAssumedState); |
8140 | return (AssumedState != getAssumed()) ? ChangeStatus::CHANGED |
8141 | : ChangeStatus::UNCHANGED; |
8142 | } |
8143 | |
8144 | // Visit and expand uses until all are analyzed or a fixpoint is reached. |
8145 | auto UsePred = [&](const Use &U, bool &Follow) -> bool { |
8146 | Instruction *UserI = cast<Instruction>(Val: U.getUser()); |
8147 | LLVM_DEBUG(dbgs() << "[AAMemoryBehavior] Use: " << *U << " in " << *UserI |
8148 | << " \n" ); |
8149 | |
8150 | // Droppable users, e.g., llvm::assume does not actually perform any action. |
8151 | if (UserI->isDroppable()) |
8152 | return true; |
8153 | |
8154 | // Check if the users of UserI should also be visited. |
8155 | Follow = followUsersOfUseIn(A, U, UserI); |
8156 | |
8157 | // If UserI might touch memory we analyze the use in detail. |
8158 | if (UserI->mayReadOrWriteMemory()) |
8159 | analyzeUseIn(A, U, UserI); |
8160 | |
8161 | return !isAtFixpoint(); |
8162 | }; |
8163 | |
8164 | if (!A.checkForAllUses(Pred: UsePred, QueryingAA: *this, V: getAssociatedValue())) |
8165 | return indicatePessimisticFixpoint(); |
8166 | |
8167 | return (AssumedState != getAssumed()) ? ChangeStatus::CHANGED |
8168 | : ChangeStatus::UNCHANGED; |
8169 | } |
8170 | |
8171 | bool AAMemoryBehaviorFloating::followUsersOfUseIn(Attributor &A, const Use &U, |
8172 | const Instruction *UserI) { |
8173 | // The loaded value is unrelated to the pointer argument, no need to |
8174 | // follow the users of the load. |
8175 | if (isa<LoadInst>(Val: UserI) || isa<ReturnInst>(Val: UserI)) |
8176 | return false; |
8177 | |
8178 | // By default we follow all uses assuming UserI might leak information on U, |
8179 | // we have special handling for call sites operands though. |
8180 | const auto *CB = dyn_cast<CallBase>(Val: UserI); |
8181 | if (!CB || !CB->isArgOperand(U: &U)) |
8182 | return true; |
8183 | |
8184 | // If the use is a call argument known not to be captured, the users of |
8185 | // the call do not need to be visited because they have to be unrelated to |
8186 | // the input. Note that this check is not trivial even though we disallow |
8187 | // general capturing of the underlying argument. The reason is that the |
8188 | // call might the argument "through return", which we allow and for which we |
8189 | // need to check call users. |
8190 | if (U.get()->getType()->isPointerTy()) { |
8191 | unsigned ArgNo = CB->getArgOperandNo(U: &U); |
8192 | bool IsKnownNoCapture; |
8193 | return !AA::hasAssumedIRAttr<Attribute::NoCapture>( |
8194 | A, this, IRPosition::callsite_argument(*CB, ArgNo), |
8195 | DepClassTy::OPTIONAL, IsKnownNoCapture); |
8196 | } |
8197 | |
8198 | return true; |
8199 | } |
8200 | |
8201 | void AAMemoryBehaviorFloating::analyzeUseIn(Attributor &A, const Use &U, |
8202 | const Instruction *UserI) { |
8203 | assert(UserI->mayReadOrWriteMemory()); |
8204 | |
8205 | switch (UserI->getOpcode()) { |
8206 | default: |
8207 | // TODO: Handle all atomics and other side-effect operations we know of. |
8208 | break; |
8209 | case Instruction::Load: |
8210 | // Loads cause the NO_READS property to disappear. |
8211 | removeAssumedBits(BitsEncoding: NO_READS); |
8212 | return; |
8213 | |
8214 | case Instruction::Store: |
8215 | // Stores cause the NO_WRITES property to disappear if the use is the |
8216 | // pointer operand. Note that while capturing was taken care of somewhere |
8217 | // else we need to deal with stores of the value that is not looked through. |
8218 | if (cast<StoreInst>(Val: UserI)->getPointerOperand() == U.get()) |
8219 | removeAssumedBits(BitsEncoding: NO_WRITES); |
8220 | else |
8221 | indicatePessimisticFixpoint(); |
8222 | return; |
8223 | |
8224 | case Instruction::Call: |
8225 | case Instruction::CallBr: |
8226 | case Instruction::Invoke: { |
8227 | // For call sites we look at the argument memory behavior attribute (this |
8228 | // could be recursive!) in order to restrict our own state. |
8229 | const auto *CB = cast<CallBase>(Val: UserI); |
8230 | |
8231 | // Give up on operand bundles. |
8232 | if (CB->isBundleOperand(U: &U)) { |
8233 | indicatePessimisticFixpoint(); |
8234 | return; |
8235 | } |
8236 | |
8237 | // Calling a function does read the function pointer, maybe write it if the |
8238 | // function is self-modifying. |
8239 | if (CB->isCallee(U: &U)) { |
8240 | removeAssumedBits(BitsEncoding: NO_READS); |
8241 | break; |
8242 | } |
8243 | |
8244 | // Adjust the possible access behavior based on the information on the |
8245 | // argument. |
8246 | IRPosition Pos; |
8247 | if (U.get()->getType()->isPointerTy()) |
8248 | Pos = IRPosition::callsite_argument(CB: *CB, ArgNo: CB->getArgOperandNo(U: &U)); |
8249 | else |
8250 | Pos = IRPosition::callsite_function(CB: *CB); |
8251 | const auto *MemBehaviorAA = |
8252 | A.getAAFor<AAMemoryBehavior>(QueryingAA: *this, IRP: Pos, DepClass: DepClassTy::OPTIONAL); |
8253 | if (!MemBehaviorAA) |
8254 | break; |
8255 | // "assumed" has at most the same bits as the MemBehaviorAA assumed |
8256 | // and at least "known". |
8257 | intersectAssumedBits(BitsEncoding: MemBehaviorAA->getAssumed()); |
8258 | return; |
8259 | } |
8260 | }; |
8261 | |
8262 | // Generally, look at the "may-properties" and adjust the assumed state if we |
8263 | // did not trigger special handling before. |
8264 | if (UserI->mayReadFromMemory()) |
8265 | removeAssumedBits(BitsEncoding: NO_READS); |
8266 | if (UserI->mayWriteToMemory()) |
8267 | removeAssumedBits(BitsEncoding: NO_WRITES); |
8268 | } |
8269 | } // namespace |
8270 | |
8271 | /// -------------------- Memory Locations Attributes --------------------------- |
8272 | /// Includes read-none, argmemonly, inaccessiblememonly, |
8273 | /// inaccessiblememorargmemonly |
8274 | /// ---------------------------------------------------------------------------- |
8275 | |
8276 | std::string AAMemoryLocation::getMemoryLocationsAsStr( |
8277 | AAMemoryLocation::MemoryLocationsKind MLK) { |
8278 | if (0 == (MLK & AAMemoryLocation::NO_LOCATIONS)) |
8279 | return "all memory" ; |
8280 | if (MLK == AAMemoryLocation::NO_LOCATIONS) |
8281 | return "no memory" ; |
8282 | std::string S = "memory:" ; |
8283 | if (0 == (MLK & AAMemoryLocation::NO_LOCAL_MEM)) |
8284 | S += "stack," ; |
8285 | if (0 == (MLK & AAMemoryLocation::NO_CONST_MEM)) |
8286 | S += "constant," ; |
8287 | if (0 == (MLK & AAMemoryLocation::NO_GLOBAL_INTERNAL_MEM)) |
8288 | S += "internal global," ; |
8289 | if (0 == (MLK & AAMemoryLocation::NO_GLOBAL_EXTERNAL_MEM)) |
8290 | S += "external global," ; |
8291 | if (0 == (MLK & AAMemoryLocation::NO_ARGUMENT_MEM)) |
8292 | S += "argument," ; |
8293 | if (0 == (MLK & AAMemoryLocation::NO_INACCESSIBLE_MEM)) |
8294 | S += "inaccessible," ; |
8295 | if (0 == (MLK & AAMemoryLocation::NO_MALLOCED_MEM)) |
8296 | S += "malloced," ; |
8297 | if (0 == (MLK & AAMemoryLocation::NO_UNKOWN_MEM)) |
8298 | S += "unknown," ; |
8299 | S.pop_back(); |
8300 | return S; |
8301 | } |
8302 | |
8303 | namespace { |
8304 | struct AAMemoryLocationImpl : public AAMemoryLocation { |
8305 | |
8306 | AAMemoryLocationImpl(const IRPosition &IRP, Attributor &A) |
8307 | : AAMemoryLocation(IRP, A), Allocator(A.Allocator) { |
8308 | AccessKind2Accesses.fill(u: nullptr); |
8309 | } |
8310 | |
8311 | ~AAMemoryLocationImpl() { |
8312 | // The AccessSets are allocated via a BumpPtrAllocator, we call |
8313 | // the destructor manually. |
8314 | for (AccessSet *AS : AccessKind2Accesses) |
8315 | if (AS) |
8316 | AS->~AccessSet(); |
8317 | } |
8318 | |
8319 | /// See AbstractAttribute::initialize(...). |
8320 | void initialize(Attributor &A) override { |
8321 | intersectAssumedBits(BitsEncoding: BEST_STATE); |
8322 | getKnownStateFromValue(A, IRP: getIRPosition(), State&: getState()); |
8323 | AAMemoryLocation::initialize(A); |
8324 | } |
8325 | |
8326 | /// Return the memory behavior information encoded in the IR for \p IRP. |
8327 | static void getKnownStateFromValue(Attributor &A, const IRPosition &IRP, |
8328 | BitIntegerState &State, |
8329 | bool IgnoreSubsumingPositions = false) { |
8330 | // For internal functions we ignore `argmemonly` and |
8331 | // `inaccessiblememorargmemonly` as we might break it via interprocedural |
8332 | // constant propagation. It is unclear if this is the best way but it is |
8333 | // unlikely this will cause real performance problems. If we are deriving |
8334 | // attributes for the anchor function we even remove the attribute in |
8335 | // addition to ignoring it. |
8336 | // TODO: A better way to handle this would be to add ~NO_GLOBAL_MEM / |
8337 | // MemoryEffects::Other as a possible location. |
8338 | bool UseArgMemOnly = true; |
8339 | Function *AnchorFn = IRP.getAnchorScope(); |
8340 | if (AnchorFn && A.isRunOn(Fn&: *AnchorFn)) |
8341 | UseArgMemOnly = !AnchorFn->hasLocalLinkage(); |
8342 | |
8343 | SmallVector<Attribute, 2> Attrs; |
8344 | A.getAttrs(IRP, {Attribute::Memory}, Attrs, IgnoreSubsumingPositions); |
8345 | for (const Attribute &Attr : Attrs) { |
8346 | // TODO: We can map MemoryEffects to Attributor locations more precisely. |
8347 | MemoryEffects ME = Attr.getMemoryEffects(); |
8348 | if (ME.doesNotAccessMemory()) { |
8349 | State.addKnownBits(Bits: NO_LOCAL_MEM | NO_CONST_MEM); |
8350 | continue; |
8351 | } |
8352 | if (ME.onlyAccessesInaccessibleMem()) { |
8353 | State.addKnownBits(Bits: inverseLocation(Loc: NO_INACCESSIBLE_MEM, AndLocalMem: true, AndConstMem: true)); |
8354 | continue; |
8355 | } |
8356 | if (ME.onlyAccessesArgPointees()) { |
8357 | if (UseArgMemOnly) |
8358 | State.addKnownBits(Bits: inverseLocation(Loc: NO_ARGUMENT_MEM, AndLocalMem: true, AndConstMem: true)); |
8359 | else { |
8360 | // Remove location information, only keep read/write info. |
8361 | ME = MemoryEffects(ME.getModRef()); |
8362 | A.manifestAttrs(IRP, |
8363 | DeducedAttrs: Attribute::getWithMemoryEffects( |
8364 | Context&: IRP.getAnchorValue().getContext(), ME), |
8365 | /*ForceReplace*/ true); |
8366 | } |
8367 | continue; |
8368 | } |
8369 | if (ME.onlyAccessesInaccessibleOrArgMem()) { |
8370 | if (UseArgMemOnly) |
8371 | State.addKnownBits(Bits: inverseLocation( |
8372 | Loc: NO_INACCESSIBLE_MEM | NO_ARGUMENT_MEM, AndLocalMem: true, AndConstMem: true)); |
8373 | else { |
8374 | // Remove location information, only keep read/write info. |
8375 | ME = MemoryEffects(ME.getModRef()); |
8376 | A.manifestAttrs(IRP, |
8377 | DeducedAttrs: Attribute::getWithMemoryEffects( |
8378 | Context&: IRP.getAnchorValue().getContext(), ME), |
8379 | /*ForceReplace*/ true); |
8380 | } |
8381 | continue; |
8382 | } |
8383 | } |
8384 | } |
8385 | |
8386 | /// See AbstractAttribute::getDeducedAttributes(...). |
8387 | void getDeducedAttributes(Attributor &A, LLVMContext &Ctx, |
8388 | SmallVectorImpl<Attribute> &Attrs) const override { |
8389 | // TODO: We can map Attributor locations to MemoryEffects more precisely. |
8390 | assert(Attrs.size() == 0); |
8391 | if (getIRPosition().getPositionKind() == IRPosition::IRP_FUNCTION) { |
8392 | if (isAssumedReadNone()) |
8393 | Attrs.push_back( |
8394 | Elt: Attribute::getWithMemoryEffects(Context&: Ctx, ME: MemoryEffects::none())); |
8395 | else if (isAssumedInaccessibleMemOnly()) |
8396 | Attrs.push_back(Elt: Attribute::getWithMemoryEffects( |
8397 | Context&: Ctx, ME: MemoryEffects::inaccessibleMemOnly())); |
8398 | else if (isAssumedArgMemOnly()) |
8399 | Attrs.push_back( |
8400 | Elt: Attribute::getWithMemoryEffects(Context&: Ctx, ME: MemoryEffects::argMemOnly())); |
8401 | else if (isAssumedInaccessibleOrArgMemOnly()) |
8402 | Attrs.push_back(Elt: Attribute::getWithMemoryEffects( |
8403 | Context&: Ctx, ME: MemoryEffects::inaccessibleOrArgMemOnly())); |
8404 | } |
8405 | assert(Attrs.size() <= 1); |
8406 | } |
8407 | |
8408 | /// See AbstractAttribute::manifest(...). |
8409 | ChangeStatus manifest(Attributor &A) override { |
8410 | // TODO: If AAMemoryLocation and AAMemoryBehavior are merged, we could |
8411 | // provide per-location modref information here. |
8412 | const IRPosition &IRP = getIRPosition(); |
8413 | |
8414 | SmallVector<Attribute, 1> DeducedAttrs; |
8415 | getDeducedAttributes(A, Ctx&: IRP.getAnchorValue().getContext(), Attrs&: DeducedAttrs); |
8416 | if (DeducedAttrs.size() != 1) |
8417 | return ChangeStatus::UNCHANGED; |
8418 | MemoryEffects ME = DeducedAttrs[0].getMemoryEffects(); |
8419 | |
8420 | return A.manifestAttrs(IRP, DeducedAttrs: Attribute::getWithMemoryEffects( |
8421 | Context&: IRP.getAnchorValue().getContext(), ME)); |
8422 | } |
8423 | |
8424 | /// See AAMemoryLocation::checkForAllAccessesToMemoryKind(...). |
8425 | bool checkForAllAccessesToMemoryKind( |
8426 | function_ref<bool(const Instruction *, const Value *, AccessKind, |
8427 | MemoryLocationsKind)> |
8428 | Pred, |
8429 | MemoryLocationsKind RequestedMLK) const override { |
8430 | if (!isValidState()) |
8431 | return false; |
8432 | |
8433 | MemoryLocationsKind AssumedMLK = getAssumedNotAccessedLocation(); |
8434 | if (AssumedMLK == NO_LOCATIONS) |
8435 | return true; |
8436 | |
8437 | unsigned Idx = 0; |
8438 | for (MemoryLocationsKind CurMLK = 1; CurMLK < NO_LOCATIONS; |
8439 | CurMLK *= 2, ++Idx) { |
8440 | if (CurMLK & RequestedMLK) |
8441 | continue; |
8442 | |
8443 | if (const AccessSet *Accesses = AccessKind2Accesses[Idx]) |
8444 | for (const AccessInfo &AI : *Accesses) |
8445 | if (!Pred(AI.I, AI.Ptr, AI.Kind, CurMLK)) |
8446 | return false; |
8447 | } |
8448 | |
8449 | return true; |
8450 | } |
8451 | |
8452 | ChangeStatus indicatePessimisticFixpoint() override { |
8453 | // If we give up and indicate a pessimistic fixpoint this instruction will |
8454 | // become an access for all potential access kinds: |
8455 | // TODO: Add pointers for argmemonly and globals to improve the results of |
8456 | // checkForAllAccessesToMemoryKind. |
8457 | bool Changed = false; |
8458 | MemoryLocationsKind KnownMLK = getKnown(); |
8459 | Instruction *I = dyn_cast<Instruction>(Val: &getAssociatedValue()); |
8460 | for (MemoryLocationsKind CurMLK = 1; CurMLK < NO_LOCATIONS; CurMLK *= 2) |
8461 | if (!(CurMLK & KnownMLK)) |
8462 | updateStateAndAccessesMap(State&: getState(), MLK: CurMLK, I, Ptr: nullptr, Changed, |
8463 | AK: getAccessKindFromInst(I)); |
8464 | return AAMemoryLocation::indicatePessimisticFixpoint(); |
8465 | } |
8466 | |
8467 | protected: |
8468 | /// Helper struct to tie together an instruction that has a read or write |
8469 | /// effect with the pointer it accesses (if any). |
8470 | struct AccessInfo { |
8471 | |
8472 | /// The instruction that caused the access. |
8473 | const Instruction *I; |
8474 | |
8475 | /// The base pointer that is accessed, or null if unknown. |
8476 | const Value *Ptr; |
8477 | |
8478 | /// The kind of access (read/write/read+write). |
8479 | AccessKind Kind; |
8480 | |
8481 | bool operator==(const AccessInfo &RHS) const { |
8482 | return I == RHS.I && Ptr == RHS.Ptr && Kind == RHS.Kind; |
8483 | } |
8484 | bool operator()(const AccessInfo &LHS, const AccessInfo &RHS) const { |
8485 | if (LHS.I != RHS.I) |
8486 | return LHS.I < RHS.I; |
8487 | if (LHS.Ptr != RHS.Ptr) |
8488 | return LHS.Ptr < RHS.Ptr; |
8489 | if (LHS.Kind != RHS.Kind) |
8490 | return LHS.Kind < RHS.Kind; |
8491 | return false; |
8492 | } |
8493 | }; |
8494 | |
8495 | /// Mapping from *single* memory location kinds, e.g., LOCAL_MEM with the |
8496 | /// value of NO_LOCAL_MEM, to the accesses encountered for this memory kind. |
8497 | using AccessSet = SmallSet<AccessInfo, 2, AccessInfo>; |
8498 | std::array<AccessSet *, llvm::CTLog2<VALID_STATE>()> AccessKind2Accesses; |
8499 | |
8500 | /// Categorize the pointer arguments of CB that might access memory in |
8501 | /// AccessedLoc and update the state and access map accordingly. |
8502 | void |
8503 | categorizeArgumentPointerLocations(Attributor &A, CallBase &CB, |
8504 | AAMemoryLocation::StateType &AccessedLocs, |
8505 | bool &Changed); |
8506 | |
8507 | /// Return the kind(s) of location that may be accessed by \p V. |
8508 | AAMemoryLocation::MemoryLocationsKind |
8509 | categorizeAccessedLocations(Attributor &A, Instruction &I, bool &Changed); |
8510 | |
8511 | /// Return the access kind as determined by \p I. |
8512 | AccessKind getAccessKindFromInst(const Instruction *I) { |
8513 | AccessKind AK = READ_WRITE; |
8514 | if (I) { |
8515 | AK = I->mayReadFromMemory() ? READ : NONE; |
8516 | AK = AccessKind(AK | (I->mayWriteToMemory() ? WRITE : NONE)); |
8517 | } |
8518 | return AK; |
8519 | } |
8520 | |
8521 | /// Update the state \p State and the AccessKind2Accesses given that \p I is |
8522 | /// an access of kind \p AK to a \p MLK memory location with the access |
8523 | /// pointer \p Ptr. |
8524 | void updateStateAndAccessesMap(AAMemoryLocation::StateType &State, |
8525 | MemoryLocationsKind MLK, const Instruction *I, |
8526 | const Value *Ptr, bool &Changed, |
8527 | AccessKind AK = READ_WRITE) { |
8528 | |
8529 | assert(isPowerOf2_32(MLK) && "Expected a single location set!" ); |
8530 | auto *&Accesses = AccessKind2Accesses[llvm::Log2_32(Value: MLK)]; |
8531 | if (!Accesses) |
8532 | Accesses = new (Allocator) AccessSet(); |
8533 | Changed |= Accesses->insert(V: AccessInfo{.I: I, .Ptr: Ptr, .Kind: AK}).second; |
8534 | if (MLK == NO_UNKOWN_MEM) |
8535 | MLK = NO_LOCATIONS; |
8536 | State.removeAssumedBits(BitsEncoding: MLK); |
8537 | } |
8538 | |
8539 | /// Determine the underlying locations kinds for \p Ptr, e.g., globals or |
8540 | /// arguments, and update the state and access map accordingly. |
8541 | void categorizePtrValue(Attributor &A, const Instruction &I, const Value &Ptr, |
8542 | AAMemoryLocation::StateType &State, bool &Changed, |
8543 | unsigned AccessAS = 0); |
8544 | |
8545 | /// Used to allocate access sets. |
8546 | BumpPtrAllocator &Allocator; |
8547 | }; |
8548 | |
8549 | void AAMemoryLocationImpl::categorizePtrValue( |
8550 | Attributor &A, const Instruction &I, const Value &Ptr, |
8551 | AAMemoryLocation::StateType &State, bool &Changed, unsigned AccessAS) { |
8552 | LLVM_DEBUG(dbgs() << "[AAMemoryLocation] Categorize pointer locations for " |
8553 | << Ptr << " [" |
8554 | << getMemoryLocationsAsStr(State.getAssumed()) << "]\n" ); |
8555 | |
8556 | auto Pred = [&](Value &Obj) { |
8557 | unsigned ObjectAS = Obj.getType()->getPointerAddressSpace(); |
8558 | // TODO: recognize the TBAA used for constant accesses. |
8559 | MemoryLocationsKind MLK = NO_LOCATIONS; |
8560 | |
8561 | // Filter accesses to constant (GPU) memory if we have an AS at the access |
8562 | // site or the object is known to actually have the associated AS. |
8563 | if ((AccessAS == (unsigned)AA::GPUAddressSpace::Constant || |
8564 | (ObjectAS == (unsigned)AA::GPUAddressSpace::Constant && |
8565 | isIdentifiedObject(V: &Obj))) && |
8566 | AA::isGPU(M: *I.getModule())) |
8567 | return true; |
8568 | |
8569 | if (isa<UndefValue>(Val: &Obj)) |
8570 | return true; |
8571 | if (isa<Argument>(Val: &Obj)) { |
8572 | // TODO: For now we do not treat byval arguments as local copies performed |
8573 | // on the call edge, though, we should. To make that happen we need to |
8574 | // teach various passes, e.g., DSE, about the copy effect of a byval. That |
8575 | // would also allow us to mark functions only accessing byval arguments as |
8576 | // readnone again, arguably their accesses have no effect outside of the |
8577 | // function, like accesses to allocas. |
8578 | MLK = NO_ARGUMENT_MEM; |
8579 | } else if (auto *GV = dyn_cast<GlobalValue>(Val: &Obj)) { |
8580 | // Reading constant memory is not treated as a read "effect" by the |
8581 | // function attr pass so we won't neither. Constants defined by TBAA are |
8582 | // similar. (We know we do not write it because it is constant.) |
8583 | if (auto *GVar = dyn_cast<GlobalVariable>(Val: GV)) |
8584 | if (GVar->isConstant()) |
8585 | return true; |
8586 | |
8587 | if (GV->hasLocalLinkage()) |
8588 | MLK = NO_GLOBAL_INTERNAL_MEM; |
8589 | else |
8590 | MLK = NO_GLOBAL_EXTERNAL_MEM; |
8591 | } else if (isa<ConstantPointerNull>(Val: &Obj) && |
8592 | (!NullPointerIsDefined(F: getAssociatedFunction(), AS: AccessAS) || |
8593 | !NullPointerIsDefined(F: getAssociatedFunction(), AS: ObjectAS))) { |
8594 | return true; |
8595 | } else if (isa<AllocaInst>(Val: &Obj)) { |
8596 | MLK = NO_LOCAL_MEM; |
8597 | } else if (const auto *CB = dyn_cast<CallBase>(Val: &Obj)) { |
8598 | bool IsKnownNoAlias; |
8599 | if (AA::hasAssumedIRAttr<Attribute::NoAlias>( |
8600 | A, this, IRPosition::callsite_returned(*CB), DepClassTy::OPTIONAL, |
8601 | IsKnownNoAlias)) |
8602 | MLK = NO_MALLOCED_MEM; |
8603 | else |
8604 | MLK = NO_UNKOWN_MEM; |
8605 | } else { |
8606 | MLK = NO_UNKOWN_MEM; |
8607 | } |
8608 | |
8609 | assert(MLK != NO_LOCATIONS && "No location specified!" ); |
8610 | LLVM_DEBUG(dbgs() << "[AAMemoryLocation] Ptr value can be categorized: " |
8611 | << Obj << " -> " << getMemoryLocationsAsStr(MLK) << "\n" ); |
8612 | updateStateAndAccessesMap(State, MLK, I: &I, Ptr: &Obj, Changed, |
8613 | AK: getAccessKindFromInst(I: &I)); |
8614 | |
8615 | return true; |
8616 | }; |
8617 | |
8618 | const auto *AA = A.getAAFor<AAUnderlyingObjects>( |
8619 | QueryingAA: *this, IRP: IRPosition::value(V: Ptr), DepClass: DepClassTy::OPTIONAL); |
8620 | if (!AA || !AA->forallUnderlyingObjects(Pred, Scope: AA::Intraprocedural)) { |
8621 | LLVM_DEBUG( |
8622 | dbgs() << "[AAMemoryLocation] Pointer locations not categorized\n" ); |
8623 | updateStateAndAccessesMap(State, MLK: NO_UNKOWN_MEM, I: &I, Ptr: nullptr, Changed, |
8624 | AK: getAccessKindFromInst(I: &I)); |
8625 | return; |
8626 | } |
8627 | |
8628 | LLVM_DEBUG( |
8629 | dbgs() << "[AAMemoryLocation] Accessed locations with pointer locations: " |
8630 | << getMemoryLocationsAsStr(State.getAssumed()) << "\n" ); |
8631 | } |
8632 | |
8633 | void AAMemoryLocationImpl::categorizeArgumentPointerLocations( |
8634 | Attributor &A, CallBase &CB, AAMemoryLocation::StateType &AccessedLocs, |
8635 | bool &Changed) { |
8636 | for (unsigned ArgNo = 0, E = CB.arg_size(); ArgNo < E; ++ArgNo) { |
8637 | |
8638 | // Skip non-pointer arguments. |
8639 | const Value *ArgOp = CB.getArgOperand(i: ArgNo); |
8640 | if (!ArgOp->getType()->isPtrOrPtrVectorTy()) |
8641 | continue; |
8642 | |
8643 | // Skip readnone arguments. |
8644 | const IRPosition &ArgOpIRP = IRPosition::callsite_argument(CB, ArgNo); |
8645 | const auto *ArgOpMemLocationAA = |
8646 | A.getAAFor<AAMemoryBehavior>(QueryingAA: *this, IRP: ArgOpIRP, DepClass: DepClassTy::OPTIONAL); |
8647 | |
8648 | if (ArgOpMemLocationAA && ArgOpMemLocationAA->isAssumedReadNone()) |
8649 | continue; |
8650 | |
8651 | // Categorize potentially accessed pointer arguments as if there was an |
8652 | // access instruction with them as pointer. |
8653 | categorizePtrValue(A, I: CB, Ptr: *ArgOp, State&: AccessedLocs, Changed); |
8654 | } |
8655 | } |
8656 | |
8657 | AAMemoryLocation::MemoryLocationsKind |
8658 | AAMemoryLocationImpl::categorizeAccessedLocations(Attributor &A, Instruction &I, |
8659 | bool &Changed) { |
8660 | LLVM_DEBUG(dbgs() << "[AAMemoryLocation] Categorize accessed locations for " |
8661 | << I << "\n" ); |
8662 | |
8663 | AAMemoryLocation::StateType AccessedLocs; |
8664 | AccessedLocs.intersectAssumedBits(BitsEncoding: NO_LOCATIONS); |
8665 | |
8666 | if (auto *CB = dyn_cast<CallBase>(Val: &I)) { |
8667 | |
8668 | // First check if we assume any memory is access is visible. |
8669 | const auto *CBMemLocationAA = A.getAAFor<AAMemoryLocation>( |
8670 | QueryingAA: *this, IRP: IRPosition::callsite_function(CB: *CB), DepClass: DepClassTy::OPTIONAL); |
8671 | LLVM_DEBUG(dbgs() << "[AAMemoryLocation] Categorize call site: " << I |
8672 | << " [" << CBMemLocationAA << "]\n" ); |
8673 | if (!CBMemLocationAA) { |
8674 | updateStateAndAccessesMap(State&: AccessedLocs, MLK: NO_UNKOWN_MEM, I: &I, Ptr: nullptr, |
8675 | Changed, AK: getAccessKindFromInst(I: &I)); |
8676 | return NO_UNKOWN_MEM; |
8677 | } |
8678 | |
8679 | if (CBMemLocationAA->isAssumedReadNone()) |
8680 | return NO_LOCATIONS; |
8681 | |
8682 | if (CBMemLocationAA->isAssumedInaccessibleMemOnly()) { |
8683 | updateStateAndAccessesMap(State&: AccessedLocs, MLK: NO_INACCESSIBLE_MEM, I: &I, Ptr: nullptr, |
8684 | Changed, AK: getAccessKindFromInst(I: &I)); |
8685 | return AccessedLocs.getAssumed(); |
8686 | } |
8687 | |
8688 | uint32_t CBAssumedNotAccessedLocs = |
8689 | CBMemLocationAA->getAssumedNotAccessedLocation(); |
8690 | |
8691 | // Set the argmemonly and global bit as we handle them separately below. |
8692 | uint32_t CBAssumedNotAccessedLocsNoArgMem = |
8693 | CBAssumedNotAccessedLocs | NO_ARGUMENT_MEM | NO_GLOBAL_MEM; |
8694 | |
8695 | for (MemoryLocationsKind CurMLK = 1; CurMLK < NO_LOCATIONS; CurMLK *= 2) { |
8696 | if (CBAssumedNotAccessedLocsNoArgMem & CurMLK) |
8697 | continue; |
8698 | updateStateAndAccessesMap(State&: AccessedLocs, MLK: CurMLK, I: &I, Ptr: nullptr, Changed, |
8699 | AK: getAccessKindFromInst(I: &I)); |
8700 | } |
8701 | |
8702 | // Now handle global memory if it might be accessed. This is slightly tricky |
8703 | // as NO_GLOBAL_MEM has multiple bits set. |
8704 | bool HasGlobalAccesses = ((~CBAssumedNotAccessedLocs) & NO_GLOBAL_MEM); |
8705 | if (HasGlobalAccesses) { |
8706 | auto AccessPred = [&](const Instruction *, const Value *Ptr, |
8707 | AccessKind Kind, MemoryLocationsKind MLK) { |
8708 | updateStateAndAccessesMap(State&: AccessedLocs, MLK, I: &I, Ptr, Changed, |
8709 | AK: getAccessKindFromInst(I: &I)); |
8710 | return true; |
8711 | }; |
8712 | if (!CBMemLocationAA->checkForAllAccessesToMemoryKind( |
8713 | Pred: AccessPred, MLK: inverseLocation(Loc: NO_GLOBAL_MEM, AndLocalMem: false, AndConstMem: false))) |
8714 | return AccessedLocs.getWorstState(); |
8715 | } |
8716 | |
8717 | LLVM_DEBUG( |
8718 | dbgs() << "[AAMemoryLocation] Accessed state before argument handling: " |
8719 | << getMemoryLocationsAsStr(AccessedLocs.getAssumed()) << "\n" ); |
8720 | |
8721 | // Now handle argument memory if it might be accessed. |
8722 | bool HasArgAccesses = ((~CBAssumedNotAccessedLocs) & NO_ARGUMENT_MEM); |
8723 | if (HasArgAccesses) |
8724 | categorizeArgumentPointerLocations(A, CB&: *CB, AccessedLocs, Changed); |
8725 | |
8726 | LLVM_DEBUG( |
8727 | dbgs() << "[AAMemoryLocation] Accessed state after argument handling: " |
8728 | << getMemoryLocationsAsStr(AccessedLocs.getAssumed()) << "\n" ); |
8729 | |
8730 | return AccessedLocs.getAssumed(); |
8731 | } |
8732 | |
8733 | if (const Value *Ptr = getPointerOperand(I: &I, /* AllowVolatile */ true)) { |
8734 | LLVM_DEBUG( |
8735 | dbgs() << "[AAMemoryLocation] Categorize memory access with pointer: " |
8736 | << I << " [" << *Ptr << "]\n" ); |
8737 | categorizePtrValue(A, I, Ptr: *Ptr, State&: AccessedLocs, Changed, |
8738 | AccessAS: Ptr->getType()->getPointerAddressSpace()); |
8739 | return AccessedLocs.getAssumed(); |
8740 | } |
8741 | |
8742 | LLVM_DEBUG(dbgs() << "[AAMemoryLocation] Failed to categorize instruction: " |
8743 | << I << "\n" ); |
8744 | updateStateAndAccessesMap(State&: AccessedLocs, MLK: NO_UNKOWN_MEM, I: &I, Ptr: nullptr, Changed, |
8745 | AK: getAccessKindFromInst(I: &I)); |
8746 | return AccessedLocs.getAssumed(); |
8747 | } |
8748 | |
8749 | /// An AA to represent the memory behavior function attributes. |
8750 | struct AAMemoryLocationFunction final : public AAMemoryLocationImpl { |
8751 | AAMemoryLocationFunction(const IRPosition &IRP, Attributor &A) |
8752 | : AAMemoryLocationImpl(IRP, A) {} |
8753 | |
8754 | /// See AbstractAttribute::updateImpl(Attributor &A). |
8755 | ChangeStatus updateImpl(Attributor &A) override { |
8756 | |
8757 | const auto *MemBehaviorAA = |
8758 | A.getAAFor<AAMemoryBehavior>(QueryingAA: *this, IRP: getIRPosition(), DepClass: DepClassTy::NONE); |
8759 | if (MemBehaviorAA && MemBehaviorAA->isAssumedReadNone()) { |
8760 | if (MemBehaviorAA->isKnownReadNone()) |
8761 | return indicateOptimisticFixpoint(); |
8762 | assert(isAssumedReadNone() && |
8763 | "AAMemoryLocation was not read-none but AAMemoryBehavior was!" ); |
8764 | A.recordDependence(FromAA: *MemBehaviorAA, ToAA: *this, DepClass: DepClassTy::OPTIONAL); |
8765 | return ChangeStatus::UNCHANGED; |
8766 | } |
8767 | |
8768 | // The current assumed state used to determine a change. |
8769 | auto AssumedState = getAssumed(); |
8770 | bool Changed = false; |
8771 | |
8772 | auto CheckRWInst = [&](Instruction &I) { |
8773 | MemoryLocationsKind MLK = categorizeAccessedLocations(A, I, Changed); |
8774 | LLVM_DEBUG(dbgs() << "[AAMemoryLocation] Accessed locations for " << I |
8775 | << ": " << getMemoryLocationsAsStr(MLK) << "\n" ); |
8776 | removeAssumedBits(BitsEncoding: inverseLocation(Loc: MLK, AndLocalMem: false, AndConstMem: false)); |
8777 | // Stop once only the valid bit set in the *not assumed location*, thus |
8778 | // once we don't actually exclude any memory locations in the state. |
8779 | return getAssumedNotAccessedLocation() != VALID_STATE; |
8780 | }; |
8781 | |
8782 | bool UsedAssumedInformation = false; |
8783 | if (!A.checkForAllReadWriteInstructions(Pred: CheckRWInst, QueryingAA&: *this, |
8784 | UsedAssumedInformation)) |
8785 | return indicatePessimisticFixpoint(); |
8786 | |
8787 | Changed |= AssumedState != getAssumed(); |
8788 | return Changed ? ChangeStatus::CHANGED : ChangeStatus::UNCHANGED; |
8789 | } |
8790 | |
8791 | /// See AbstractAttribute::trackStatistics() |
8792 | void trackStatistics() const override { |
8793 | if (isAssumedReadNone()) |
8794 | STATS_DECLTRACK_FN_ATTR(readnone) |
8795 | else if (isAssumedArgMemOnly()) |
8796 | STATS_DECLTRACK_FN_ATTR(argmemonly) |
8797 | else if (isAssumedInaccessibleMemOnly()) |
8798 | STATS_DECLTRACK_FN_ATTR(inaccessiblememonly) |
8799 | else if (isAssumedInaccessibleOrArgMemOnly()) |
8800 | STATS_DECLTRACK_FN_ATTR(inaccessiblememorargmemonly) |
8801 | } |
8802 | }; |
8803 | |
8804 | /// AAMemoryLocation attribute for call sites. |
8805 | struct AAMemoryLocationCallSite final : AAMemoryLocationImpl { |
8806 | AAMemoryLocationCallSite(const IRPosition &IRP, Attributor &A) |
8807 | : AAMemoryLocationImpl(IRP, A) {} |
8808 | |
8809 | /// See AbstractAttribute::updateImpl(...). |
8810 | ChangeStatus updateImpl(Attributor &A) override { |
8811 | // TODO: Once we have call site specific value information we can provide |
8812 | // call site specific liveness liveness information and then it makes |
8813 | // sense to specialize attributes for call sites arguments instead of |
8814 | // redirecting requests to the callee argument. |
8815 | Function *F = getAssociatedFunction(); |
8816 | const IRPosition &FnPos = IRPosition::function(F: *F); |
8817 | auto *FnAA = |
8818 | A.getAAFor<AAMemoryLocation>(QueryingAA: *this, IRP: FnPos, DepClass: DepClassTy::REQUIRED); |
8819 | if (!FnAA) |
8820 | return indicatePessimisticFixpoint(); |
8821 | bool Changed = false; |
8822 | auto AccessPred = [&](const Instruction *I, const Value *Ptr, |
8823 | AccessKind Kind, MemoryLocationsKind MLK) { |
8824 | updateStateAndAccessesMap(State&: getState(), MLK, I, Ptr, Changed, |
8825 | AK: getAccessKindFromInst(I)); |
8826 | return true; |
8827 | }; |
8828 | if (!FnAA->checkForAllAccessesToMemoryKind(Pred: AccessPred, MLK: ALL_LOCATIONS)) |
8829 | return indicatePessimisticFixpoint(); |
8830 | return Changed ? ChangeStatus::CHANGED : ChangeStatus::UNCHANGED; |
8831 | } |
8832 | |
8833 | /// See AbstractAttribute::trackStatistics() |
8834 | void trackStatistics() const override { |
8835 | if (isAssumedReadNone()) |
8836 | STATS_DECLTRACK_CS_ATTR(readnone) |
8837 | } |
8838 | }; |
8839 | } // namespace |
8840 | |
8841 | /// ------------------ denormal-fp-math Attribute ------------------------- |
8842 | |
8843 | namespace { |
8844 | struct AADenormalFPMathImpl : public AADenormalFPMath { |
8845 | AADenormalFPMathImpl(const IRPosition &IRP, Attributor &A) |
8846 | : AADenormalFPMath(IRP, A) {} |
8847 | |
8848 | const std::string getAsStr(Attributor *A) const override { |
8849 | std::string Str("AADenormalFPMath[" ); |
8850 | raw_string_ostream OS(Str); |
8851 | |
8852 | DenormalState Known = getKnown(); |
8853 | if (Known.Mode.isValid()) |
8854 | OS << "denormal-fp-math=" << Known.Mode; |
8855 | else |
8856 | OS << "invalid" ; |
8857 | |
8858 | if (Known.ModeF32.isValid()) |
8859 | OS << " denormal-fp-math-f32=" << Known.ModeF32; |
8860 | OS << ']'; |
8861 | return OS.str(); |
8862 | } |
8863 | }; |
8864 | |
8865 | struct AADenormalFPMathFunction final : AADenormalFPMathImpl { |
8866 | AADenormalFPMathFunction(const IRPosition &IRP, Attributor &A) |
8867 | : AADenormalFPMathImpl(IRP, A) {} |
8868 | |
8869 | void initialize(Attributor &A) override { |
8870 | const Function *F = getAnchorScope(); |
8871 | DenormalMode Mode = F->getDenormalModeRaw(); |
8872 | DenormalMode ModeF32 = F->getDenormalModeF32Raw(); |
8873 | |
8874 | // TODO: Handling this here prevents handling the case where a callee has a |
8875 | // fixed denormal-fp-math with dynamic denormal-fp-math-f32, but called from |
8876 | // a function with a fully fixed mode. |
8877 | if (ModeF32 == DenormalMode::getInvalid()) |
8878 | ModeF32 = Mode; |
8879 | Known = DenormalState{.Mode: Mode, .ModeF32: ModeF32}; |
8880 | if (isModeFixed()) |
8881 | indicateFixpoint(); |
8882 | } |
8883 | |
8884 | ChangeStatus updateImpl(Attributor &A) override { |
8885 | ChangeStatus Change = ChangeStatus::UNCHANGED; |
8886 | |
8887 | auto CheckCallSite = [=, &Change, &A](AbstractCallSite CS) { |
8888 | Function *Caller = CS.getInstruction()->getFunction(); |
8889 | LLVM_DEBUG(dbgs() << "[AADenormalFPMath] Call " << Caller->getName() |
8890 | << "->" << getAssociatedFunction()->getName() << '\n'); |
8891 | |
8892 | const auto *CallerInfo = A.getAAFor<AADenormalFPMath>( |
8893 | QueryingAA: *this, IRP: IRPosition::function(F: *Caller), DepClass: DepClassTy::REQUIRED); |
8894 | if (!CallerInfo) |
8895 | return false; |
8896 | |
8897 | Change = Change | clampStateAndIndicateChange(S&: this->getState(), |
8898 | R: CallerInfo->getState()); |
8899 | return true; |
8900 | }; |
8901 | |
8902 | bool AllCallSitesKnown = true; |
8903 | if (!A.checkForAllCallSites(Pred: CheckCallSite, QueryingAA: *this, RequireAllCallSites: true, UsedAssumedInformation&: AllCallSitesKnown)) |
8904 | return indicatePessimisticFixpoint(); |
8905 | |
8906 | if (Change == ChangeStatus::CHANGED && isModeFixed()) |
8907 | indicateFixpoint(); |
8908 | return Change; |
8909 | } |
8910 | |
8911 | ChangeStatus manifest(Attributor &A) override { |
8912 | LLVMContext &Ctx = getAssociatedFunction()->getContext(); |
8913 | |
8914 | SmallVector<Attribute, 2> AttrToAdd; |
8915 | SmallVector<StringRef, 2> AttrToRemove; |
8916 | if (Known.Mode == DenormalMode::getDefault()) { |
8917 | AttrToRemove.push_back(Elt: "denormal-fp-math" ); |
8918 | } else { |
8919 | AttrToAdd.push_back( |
8920 | Elt: Attribute::get(Context&: Ctx, Kind: "denormal-fp-math" , Val: Known.Mode.str())); |
8921 | } |
8922 | |
8923 | if (Known.ModeF32 != Known.Mode) { |
8924 | AttrToAdd.push_back( |
8925 | Elt: Attribute::get(Context&: Ctx, Kind: "denormal-fp-math-f32" , Val: Known.ModeF32.str())); |
8926 | } else { |
8927 | AttrToRemove.push_back(Elt: "denormal-fp-math-f32" ); |
8928 | } |
8929 | |
8930 | auto &IRP = getIRPosition(); |
8931 | |
8932 | // TODO: There should be a combined add and remove API. |
8933 | return A.removeAttrs(IRP, Attrs: AttrToRemove) | |
8934 | A.manifestAttrs(IRP, DeducedAttrs: AttrToAdd, /*ForceReplace=*/true); |
8935 | } |
8936 | |
8937 | void trackStatistics() const override { |
8938 | STATS_DECLTRACK_FN_ATTR(denormal_fp_math) |
8939 | } |
8940 | }; |
8941 | } // namespace |
8942 | |
8943 | /// ------------------ Value Constant Range Attribute ------------------------- |
8944 | |
8945 | namespace { |
8946 | struct AAValueConstantRangeImpl : AAValueConstantRange { |
8947 | using StateType = IntegerRangeState; |
8948 | AAValueConstantRangeImpl(const IRPosition &IRP, Attributor &A) |
8949 | : AAValueConstantRange(IRP, A) {} |
8950 | |
8951 | /// See AbstractAttribute::initialize(..). |
8952 | void initialize(Attributor &A) override { |
8953 | if (A.hasSimplificationCallback(IRP: getIRPosition())) { |
8954 | indicatePessimisticFixpoint(); |
8955 | return; |
8956 | } |
8957 | |
8958 | // Intersect a range given by SCEV. |
8959 | intersectKnown(R: getConstantRangeFromSCEV(A, I: getCtxI())); |
8960 | |
8961 | // Intersect a range given by LVI. |
8962 | intersectKnown(R: getConstantRangeFromLVI(A, CtxI: getCtxI())); |
8963 | } |
8964 | |
8965 | /// See AbstractAttribute::getAsStr(). |
8966 | const std::string getAsStr(Attributor *A) const override { |
8967 | std::string Str; |
8968 | llvm::raw_string_ostream OS(Str); |
8969 | OS << "range(" << getBitWidth() << ")<" ; |
8970 | getKnown().print(OS); |
8971 | OS << " / " ; |
8972 | getAssumed().print(OS); |
8973 | OS << ">" ; |
8974 | return OS.str(); |
8975 | } |
8976 | |
8977 | /// Helper function to get a SCEV expr for the associated value at program |
8978 | /// point \p I. |
8979 | const SCEV *getSCEV(Attributor &A, const Instruction *I = nullptr) const { |
8980 | if (!getAnchorScope()) |
8981 | return nullptr; |
8982 | |
8983 | ScalarEvolution *SE = |
8984 | A.getInfoCache().getAnalysisResultForFunction<ScalarEvolutionAnalysis>( |
8985 | F: *getAnchorScope()); |
8986 | |
8987 | LoopInfo *LI = A.getInfoCache().getAnalysisResultForFunction<LoopAnalysis>( |
8988 | F: *getAnchorScope()); |
8989 | |
8990 | if (!SE || !LI) |
8991 | return nullptr; |
8992 | |
8993 | const SCEV *S = SE->getSCEV(V: &getAssociatedValue()); |
8994 | if (!I) |
8995 | return S; |
8996 | |
8997 | return SE->getSCEVAtScope(S, L: LI->getLoopFor(BB: I->getParent())); |
8998 | } |
8999 | |
9000 | /// Helper function to get a range from SCEV for the associated value at |
9001 | /// program point \p I. |
9002 | ConstantRange getConstantRangeFromSCEV(Attributor &A, |
9003 | const Instruction *I = nullptr) const { |
9004 | if (!getAnchorScope()) |
9005 | return getWorstState(BitWidth: getBitWidth()); |
9006 | |
9007 | ScalarEvolution *SE = |
9008 | A.getInfoCache().getAnalysisResultForFunction<ScalarEvolutionAnalysis>( |
9009 | F: *getAnchorScope()); |
9010 | |
9011 | const SCEV *S = getSCEV(A, I); |
9012 | if (!SE || !S) |
9013 | return getWorstState(BitWidth: getBitWidth()); |
9014 | |
9015 | return SE->getUnsignedRange(S); |
9016 | } |
9017 | |
9018 | /// Helper function to get a range from LVI for the associated value at |
9019 | /// program point \p I. |
9020 | ConstantRange |
9021 | getConstantRangeFromLVI(Attributor &A, |
9022 | const Instruction *CtxI = nullptr) const { |
9023 | if (!getAnchorScope()) |
9024 | return getWorstState(BitWidth: getBitWidth()); |
9025 | |
9026 | LazyValueInfo *LVI = |
9027 | A.getInfoCache().getAnalysisResultForFunction<LazyValueAnalysis>( |
9028 | F: *getAnchorScope()); |
9029 | |
9030 | if (!LVI || !CtxI) |
9031 | return getWorstState(BitWidth: getBitWidth()); |
9032 | return LVI->getConstantRange(V: &getAssociatedValue(), |
9033 | CxtI: const_cast<Instruction *>(CtxI), |
9034 | /*UndefAllowed*/ false); |
9035 | } |
9036 | |
9037 | /// Return true if \p CtxI is valid for querying outside analyses. |
9038 | /// This basically makes sure we do not ask intra-procedural analysis |
9039 | /// about a context in the wrong function or a context that violates |
9040 | /// dominance assumptions they might have. The \p AllowAACtxI flag indicates |
9041 | /// if the original context of this AA is OK or should be considered invalid. |
9042 | bool isValidCtxInstructionForOutsideAnalysis(Attributor &A, |
9043 | const Instruction *CtxI, |
9044 | bool AllowAACtxI) const { |
9045 | if (!CtxI || (!AllowAACtxI && CtxI == getCtxI())) |
9046 | return false; |
9047 | |
9048 | // Our context might be in a different function, neither intra-procedural |
9049 | // analysis (ScalarEvolution nor LazyValueInfo) can handle that. |
9050 | if (!AA::isValidInScope(V: getAssociatedValue(), Scope: CtxI->getFunction())) |
9051 | return false; |
9052 | |
9053 | // If the context is not dominated by the value there are paths to the |
9054 | // context that do not define the value. This cannot be handled by |
9055 | // LazyValueInfo so we need to bail. |
9056 | if (auto *I = dyn_cast<Instruction>(Val: &getAssociatedValue())) { |
9057 | InformationCache &InfoCache = A.getInfoCache(); |
9058 | const DominatorTree *DT = |
9059 | InfoCache.getAnalysisResultForFunction<DominatorTreeAnalysis>( |
9060 | F: *I->getFunction()); |
9061 | return DT && DT->dominates(Def: I, User: CtxI); |
9062 | } |
9063 | |
9064 | return true; |
9065 | } |
9066 | |
9067 | /// See AAValueConstantRange::getKnownConstantRange(..). |
9068 | ConstantRange |
9069 | getKnownConstantRange(Attributor &A, |
9070 | const Instruction *CtxI = nullptr) const override { |
9071 | if (!isValidCtxInstructionForOutsideAnalysis(A, CtxI, |
9072 | /* AllowAACtxI */ false)) |
9073 | return getKnown(); |
9074 | |
9075 | ConstantRange LVIR = getConstantRangeFromLVI(A, CtxI); |
9076 | ConstantRange SCEVR = getConstantRangeFromSCEV(A, I: CtxI); |
9077 | return getKnown().intersectWith(CR: SCEVR).intersectWith(CR: LVIR); |
9078 | } |
9079 | |
9080 | /// See AAValueConstantRange::getAssumedConstantRange(..). |
9081 | ConstantRange |
9082 | getAssumedConstantRange(Attributor &A, |
9083 | const Instruction *CtxI = nullptr) const override { |
9084 | // TODO: Make SCEV use Attributor assumption. |
9085 | // We may be able to bound a variable range via assumptions in |
9086 | // Attributor. ex.) If x is assumed to be in [1, 3] and y is known to |
9087 | // evolve to x^2 + x, then we can say that y is in [2, 12]. |
9088 | if (!isValidCtxInstructionForOutsideAnalysis(A, CtxI, |
9089 | /* AllowAACtxI */ false)) |
9090 | return getAssumed(); |
9091 | |
9092 | ConstantRange LVIR = getConstantRangeFromLVI(A, CtxI); |
9093 | ConstantRange SCEVR = getConstantRangeFromSCEV(A, I: CtxI); |
9094 | return getAssumed().intersectWith(CR: SCEVR).intersectWith(CR: LVIR); |
9095 | } |
9096 | |
9097 | /// Helper function to create MDNode for range metadata. |
9098 | static MDNode * |
9099 | getMDNodeForConstantRange(Type *Ty, LLVMContext &Ctx, |
9100 | const ConstantRange &AssumedConstantRange) { |
9101 | Metadata *LowAndHigh[] = {ConstantAsMetadata::get(C: ConstantInt::get( |
9102 | Ty, V: AssumedConstantRange.getLower())), |
9103 | ConstantAsMetadata::get(C: ConstantInt::get( |
9104 | Ty, V: AssumedConstantRange.getUpper()))}; |
9105 | return MDNode::get(Context&: Ctx, MDs: LowAndHigh); |
9106 | } |
9107 | |
9108 | /// Return true if \p Assumed is included in \p KnownRanges. |
9109 | static bool isBetterRange(const ConstantRange &Assumed, MDNode *KnownRanges) { |
9110 | |
9111 | if (Assumed.isFullSet()) |
9112 | return false; |
9113 | |
9114 | if (!KnownRanges) |
9115 | return true; |
9116 | |
9117 | // If multiple ranges are annotated in IR, we give up to annotate assumed |
9118 | // range for now. |
9119 | |
9120 | // TODO: If there exists a known range which containts assumed range, we |
9121 | // can say assumed range is better. |
9122 | if (KnownRanges->getNumOperands() > 2) |
9123 | return false; |
9124 | |
9125 | ConstantInt *Lower = |
9126 | mdconst::extract<ConstantInt>(MD: KnownRanges->getOperand(I: 0)); |
9127 | ConstantInt *Upper = |
9128 | mdconst::extract<ConstantInt>(MD: KnownRanges->getOperand(I: 1)); |
9129 | |
9130 | ConstantRange Known(Lower->getValue(), Upper->getValue()); |
9131 | return Known.contains(CR: Assumed) && Known != Assumed; |
9132 | } |
9133 | |
9134 | /// Helper function to set range metadata. |
9135 | static bool |
9136 | setRangeMetadataIfisBetterRange(Instruction *I, |
9137 | const ConstantRange &AssumedConstantRange) { |
9138 | auto *OldRangeMD = I->getMetadata(KindID: LLVMContext::MD_range); |
9139 | if (isBetterRange(Assumed: AssumedConstantRange, KnownRanges: OldRangeMD)) { |
9140 | if (!AssumedConstantRange.isEmptySet()) { |
9141 | I->setMetadata(KindID: LLVMContext::MD_range, |
9142 | Node: getMDNodeForConstantRange(Ty: I->getType(), Ctx&: I->getContext(), |
9143 | AssumedConstantRange)); |
9144 | return true; |
9145 | } |
9146 | } |
9147 | return false; |
9148 | } |
9149 | |
9150 | /// See AbstractAttribute::manifest() |
9151 | ChangeStatus manifest(Attributor &A) override { |
9152 | ChangeStatus Changed = ChangeStatus::UNCHANGED; |
9153 | ConstantRange AssumedConstantRange = getAssumedConstantRange(A); |
9154 | assert(!AssumedConstantRange.isFullSet() && "Invalid state" ); |
9155 | |
9156 | auto &V = getAssociatedValue(); |
9157 | if (!AssumedConstantRange.isEmptySet() && |
9158 | !AssumedConstantRange.isSingleElement()) { |
9159 | if (Instruction *I = dyn_cast<Instruction>(Val: &V)) { |
9160 | assert(I == getCtxI() && "Should not annotate an instruction which is " |
9161 | "not the context instruction" ); |
9162 | if (isa<CallInst>(Val: I) || isa<LoadInst>(Val: I)) |
9163 | if (setRangeMetadataIfisBetterRange(I, AssumedConstantRange)) |
9164 | Changed = ChangeStatus::CHANGED; |
9165 | } |
9166 | } |
9167 | |
9168 | return Changed; |
9169 | } |
9170 | }; |
9171 | |
9172 | struct AAValueConstantRangeArgument final |
9173 | : AAArgumentFromCallSiteArguments< |
9174 | AAValueConstantRange, AAValueConstantRangeImpl, IntegerRangeState, |
9175 | true /* BridgeCallBaseContext */> { |
9176 | using Base = AAArgumentFromCallSiteArguments< |
9177 | AAValueConstantRange, AAValueConstantRangeImpl, IntegerRangeState, |
9178 | true /* BridgeCallBaseContext */>; |
9179 | AAValueConstantRangeArgument(const IRPosition &IRP, Attributor &A) |
9180 | : Base(IRP, A) {} |
9181 | |
9182 | /// See AbstractAttribute::trackStatistics() |
9183 | void trackStatistics() const override { |
9184 | STATS_DECLTRACK_ARG_ATTR(value_range) |
9185 | } |
9186 | }; |
9187 | |
9188 | struct AAValueConstantRangeReturned |
9189 | : AAReturnedFromReturnedValues<AAValueConstantRange, |
9190 | AAValueConstantRangeImpl, |
9191 | AAValueConstantRangeImpl::StateType, |
9192 | /* PropogateCallBaseContext */ true> { |
9193 | using Base = |
9194 | AAReturnedFromReturnedValues<AAValueConstantRange, |
9195 | AAValueConstantRangeImpl, |
9196 | AAValueConstantRangeImpl::StateType, |
9197 | /* PropogateCallBaseContext */ true>; |
9198 | AAValueConstantRangeReturned(const IRPosition &IRP, Attributor &A) |
9199 | : Base(IRP, A) {} |
9200 | |
9201 | /// See AbstractAttribute::initialize(...). |
9202 | void initialize(Attributor &A) override { |
9203 | if (!A.isFunctionIPOAmendable(F: *getAssociatedFunction())) |
9204 | indicatePessimisticFixpoint(); |
9205 | } |
9206 | |
9207 | /// See AbstractAttribute::trackStatistics() |
9208 | void trackStatistics() const override { |
9209 | STATS_DECLTRACK_FNRET_ATTR(value_range) |
9210 | } |
9211 | }; |
9212 | |
9213 | struct AAValueConstantRangeFloating : AAValueConstantRangeImpl { |
9214 | AAValueConstantRangeFloating(const IRPosition &IRP, Attributor &A) |
9215 | : AAValueConstantRangeImpl(IRP, A) {} |
9216 | |
9217 | /// See AbstractAttribute::initialize(...). |
9218 | void initialize(Attributor &A) override { |
9219 | AAValueConstantRangeImpl::initialize(A); |
9220 | if (isAtFixpoint()) |
9221 | return; |
9222 | |
9223 | Value &V = getAssociatedValue(); |
9224 | |
9225 | if (auto *C = dyn_cast<ConstantInt>(Val: &V)) { |
9226 | unionAssumed(R: ConstantRange(C->getValue())); |
9227 | indicateOptimisticFixpoint(); |
9228 | return; |
9229 | } |
9230 | |
9231 | if (isa<UndefValue>(Val: &V)) { |
9232 | // Collapse the undef state to 0. |
9233 | unionAssumed(R: ConstantRange(APInt(getBitWidth(), 0))); |
9234 | indicateOptimisticFixpoint(); |
9235 | return; |
9236 | } |
9237 | |
9238 | if (isa<CallBase>(Val: &V)) |
9239 | return; |
9240 | |
9241 | if (isa<BinaryOperator>(Val: &V) || isa<CmpInst>(Val: &V) || isa<CastInst>(Val: &V)) |
9242 | return; |
9243 | |
9244 | // If it is a load instruction with range metadata, use it. |
9245 | if (LoadInst *LI = dyn_cast<LoadInst>(Val: &V)) |
9246 | if (auto *RangeMD = LI->getMetadata(KindID: LLVMContext::MD_range)) { |
9247 | intersectKnown(R: getConstantRangeFromMetadata(RangeMD: *RangeMD)); |
9248 | return; |
9249 | } |
9250 | |
9251 | // We can work with PHI and select instruction as we traverse their operands |
9252 | // during update. |
9253 | if (isa<SelectInst>(Val: V) || isa<PHINode>(Val: V)) |
9254 | return; |
9255 | |
9256 | // Otherwise we give up. |
9257 | indicatePessimisticFixpoint(); |
9258 | |
9259 | LLVM_DEBUG(dbgs() << "[AAValueConstantRange] We give up: " |
9260 | << getAssociatedValue() << "\n" ); |
9261 | } |
9262 | |
9263 | bool calculateBinaryOperator( |
9264 | Attributor &A, BinaryOperator *BinOp, IntegerRangeState &T, |
9265 | const Instruction *CtxI, |
9266 | SmallVectorImpl<const AAValueConstantRange *> &QuerriedAAs) { |
9267 | Value *LHS = BinOp->getOperand(i_nocapture: 0); |
9268 | Value *RHS = BinOp->getOperand(i_nocapture: 1); |
9269 | |
9270 | // Simplify the operands first. |
9271 | bool UsedAssumedInformation = false; |
9272 | const auto &SimplifiedLHS = A.getAssumedSimplified( |
9273 | IRP: IRPosition::value(V: *LHS, CBContext: getCallBaseContext()), AA: *this, |
9274 | UsedAssumedInformation, S: AA::Interprocedural); |
9275 | if (!SimplifiedLHS.has_value()) |
9276 | return true; |
9277 | if (!*SimplifiedLHS) |
9278 | return false; |
9279 | LHS = *SimplifiedLHS; |
9280 | |
9281 | const auto &SimplifiedRHS = A.getAssumedSimplified( |
9282 | IRP: IRPosition::value(V: *RHS, CBContext: getCallBaseContext()), AA: *this, |
9283 | UsedAssumedInformation, S: AA::Interprocedural); |
9284 | if (!SimplifiedRHS.has_value()) |
9285 | return true; |
9286 | if (!*SimplifiedRHS) |
9287 | return false; |
9288 | RHS = *SimplifiedRHS; |
9289 | |
9290 | // TODO: Allow non integers as well. |
9291 | if (!LHS->getType()->isIntegerTy() || !RHS->getType()->isIntegerTy()) |
9292 | return false; |
9293 | |
9294 | auto *LHSAA = A.getAAFor<AAValueConstantRange>( |
9295 | QueryingAA: *this, IRP: IRPosition::value(V: *LHS, CBContext: getCallBaseContext()), |
9296 | DepClass: DepClassTy::REQUIRED); |
9297 | if (!LHSAA) |
9298 | return false; |
9299 | QuerriedAAs.push_back(Elt: LHSAA); |
9300 | auto LHSAARange = LHSAA->getAssumedConstantRange(A, CtxI); |
9301 | |
9302 | auto *RHSAA = A.getAAFor<AAValueConstantRange>( |
9303 | QueryingAA: *this, IRP: IRPosition::value(V: *RHS, CBContext: getCallBaseContext()), |
9304 | DepClass: DepClassTy::REQUIRED); |
9305 | if (!RHSAA) |
9306 | return false; |
9307 | QuerriedAAs.push_back(Elt: RHSAA); |
9308 | auto RHSAARange = RHSAA->getAssumedConstantRange(A, CtxI); |
9309 | |
9310 | auto AssumedRange = LHSAARange.binaryOp(BinOp: BinOp->getOpcode(), Other: RHSAARange); |
9311 | |
9312 | T.unionAssumed(R: AssumedRange); |
9313 | |
9314 | // TODO: Track a known state too. |
9315 | |
9316 | return T.isValidState(); |
9317 | } |
9318 | |
9319 | bool calculateCastInst( |
9320 | Attributor &A, CastInst *CastI, IntegerRangeState &T, |
9321 | const Instruction *CtxI, |
9322 | SmallVectorImpl<const AAValueConstantRange *> &QuerriedAAs) { |
9323 | assert(CastI->getNumOperands() == 1 && "Expected cast to be unary!" ); |
9324 | // TODO: Allow non integers as well. |
9325 | Value *OpV = CastI->getOperand(i_nocapture: 0); |
9326 | |
9327 | // Simplify the operand first. |
9328 | bool UsedAssumedInformation = false; |
9329 | const auto &SimplifiedOpV = A.getAssumedSimplified( |
9330 | IRP: IRPosition::value(V: *OpV, CBContext: getCallBaseContext()), AA: *this, |
9331 | UsedAssumedInformation, S: AA::Interprocedural); |
9332 | if (!SimplifiedOpV.has_value()) |
9333 | return true; |
9334 | if (!*SimplifiedOpV) |
9335 | return false; |
9336 | OpV = *SimplifiedOpV; |
9337 | |
9338 | if (!OpV->getType()->isIntegerTy()) |
9339 | return false; |
9340 | |
9341 | auto *OpAA = A.getAAFor<AAValueConstantRange>( |
9342 | QueryingAA: *this, IRP: IRPosition::value(V: *OpV, CBContext: getCallBaseContext()), |
9343 | DepClass: DepClassTy::REQUIRED); |
9344 | if (!OpAA) |
9345 | return false; |
9346 | QuerriedAAs.push_back(Elt: OpAA); |
9347 | T.unionAssumed(R: OpAA->getAssumed().castOp(CastOp: CastI->getOpcode(), |
9348 | BitWidth: getState().getBitWidth())); |
9349 | return T.isValidState(); |
9350 | } |
9351 | |
9352 | bool |
9353 | calculateCmpInst(Attributor &A, CmpInst *CmpI, IntegerRangeState &T, |
9354 | const Instruction *CtxI, |
9355 | SmallVectorImpl<const AAValueConstantRange *> &QuerriedAAs) { |
9356 | Value *LHS = CmpI->getOperand(i_nocapture: 0); |
9357 | Value *RHS = CmpI->getOperand(i_nocapture: 1); |
9358 | |
9359 | // Simplify the operands first. |
9360 | bool UsedAssumedInformation = false; |
9361 | const auto &SimplifiedLHS = A.getAssumedSimplified( |
9362 | IRP: IRPosition::value(V: *LHS, CBContext: getCallBaseContext()), AA: *this, |
9363 | UsedAssumedInformation, S: AA::Interprocedural); |
9364 | if (!SimplifiedLHS.has_value()) |
9365 | return true; |
9366 | if (!*SimplifiedLHS) |
9367 | return false; |
9368 | LHS = *SimplifiedLHS; |
9369 | |
9370 | const auto &SimplifiedRHS = A.getAssumedSimplified( |
9371 | IRP: IRPosition::value(V: *RHS, CBContext: getCallBaseContext()), AA: *this, |
9372 | UsedAssumedInformation, S: AA::Interprocedural); |
9373 | if (!SimplifiedRHS.has_value()) |
9374 | return true; |
9375 | if (!*SimplifiedRHS) |
9376 | return false; |
9377 | RHS = *SimplifiedRHS; |
9378 | |
9379 | // TODO: Allow non integers as well. |
9380 | if (!LHS->getType()->isIntegerTy() || !RHS->getType()->isIntegerTy()) |
9381 | return false; |
9382 | |
9383 | auto *LHSAA = A.getAAFor<AAValueConstantRange>( |
9384 | QueryingAA: *this, IRP: IRPosition::value(V: *LHS, CBContext: getCallBaseContext()), |
9385 | DepClass: DepClassTy::REQUIRED); |
9386 | if (!LHSAA) |
9387 | return false; |
9388 | QuerriedAAs.push_back(Elt: LHSAA); |
9389 | auto *RHSAA = A.getAAFor<AAValueConstantRange>( |
9390 | QueryingAA: *this, IRP: IRPosition::value(V: *RHS, CBContext: getCallBaseContext()), |
9391 | DepClass: DepClassTy::REQUIRED); |
9392 | if (!RHSAA) |
9393 | return false; |
9394 | QuerriedAAs.push_back(Elt: RHSAA); |
9395 | auto LHSAARange = LHSAA->getAssumedConstantRange(A, CtxI); |
9396 | auto RHSAARange = RHSAA->getAssumedConstantRange(A, CtxI); |
9397 | |
9398 | // If one of them is empty set, we can't decide. |
9399 | if (LHSAARange.isEmptySet() || RHSAARange.isEmptySet()) |
9400 | return true; |
9401 | |
9402 | bool MustTrue = false, MustFalse = false; |
9403 | |
9404 | auto AllowedRegion = |
9405 | ConstantRange::makeAllowedICmpRegion(Pred: CmpI->getPredicate(), Other: RHSAARange); |
9406 | |
9407 | if (AllowedRegion.intersectWith(CR: LHSAARange).isEmptySet()) |
9408 | MustFalse = true; |
9409 | |
9410 | if (LHSAARange.icmp(Pred: CmpI->getPredicate(), Other: RHSAARange)) |
9411 | MustTrue = true; |
9412 | |
9413 | assert((!MustTrue || !MustFalse) && |
9414 | "Either MustTrue or MustFalse should be false!" ); |
9415 | |
9416 | if (MustTrue) |
9417 | T.unionAssumed(R: ConstantRange(APInt(/* numBits */ 1, /* val */ 1))); |
9418 | else if (MustFalse) |
9419 | T.unionAssumed(R: ConstantRange(APInt(/* numBits */ 1, /* val */ 0))); |
9420 | else |
9421 | T.unionAssumed(R: ConstantRange(/* BitWidth */ 1, /* isFullSet */ true)); |
9422 | |
9423 | LLVM_DEBUG(dbgs() << "[AAValueConstantRange] " << *CmpI << " after " |
9424 | << (MustTrue ? "true" : (MustFalse ? "false" : "unknown" )) |
9425 | << ": " << T << "\n\t" << *LHSAA << "\t<op>\n\t" |
9426 | << *RHSAA); |
9427 | |
9428 | // TODO: Track a known state too. |
9429 | return T.isValidState(); |
9430 | } |
9431 | |
9432 | /// See AbstractAttribute::updateImpl(...). |
9433 | ChangeStatus updateImpl(Attributor &A) override { |
9434 | |
9435 | IntegerRangeState T(getBitWidth()); |
9436 | auto VisitValueCB = [&](Value &V, const Instruction *CtxI) -> bool { |
9437 | Instruction *I = dyn_cast<Instruction>(Val: &V); |
9438 | if (!I || isa<CallBase>(Val: I)) { |
9439 | |
9440 | // Simplify the operand first. |
9441 | bool UsedAssumedInformation = false; |
9442 | const auto &SimplifiedOpV = A.getAssumedSimplified( |
9443 | IRP: IRPosition::value(V, CBContext: getCallBaseContext()), AA: *this, |
9444 | UsedAssumedInformation, S: AA::Interprocedural); |
9445 | if (!SimplifiedOpV.has_value()) |
9446 | return true; |
9447 | if (!*SimplifiedOpV) |
9448 | return false; |
9449 | Value *VPtr = *SimplifiedOpV; |
9450 | |
9451 | // If the value is not instruction, we query AA to Attributor. |
9452 | const auto *AA = A.getAAFor<AAValueConstantRange>( |
9453 | QueryingAA: *this, IRP: IRPosition::value(V: *VPtr, CBContext: getCallBaseContext()), |
9454 | DepClass: DepClassTy::REQUIRED); |
9455 | |
9456 | // Clamp operator is not used to utilize a program point CtxI. |
9457 | if (AA) |
9458 | T.unionAssumed(R: AA->getAssumedConstantRange(A, CtxI)); |
9459 | else |
9460 | return false; |
9461 | |
9462 | return T.isValidState(); |
9463 | } |
9464 | |
9465 | SmallVector<const AAValueConstantRange *, 4> QuerriedAAs; |
9466 | if (auto *BinOp = dyn_cast<BinaryOperator>(Val: I)) { |
9467 | if (!calculateBinaryOperator(A, BinOp, T, CtxI, QuerriedAAs)) |
9468 | return false; |
9469 | } else if (auto *CmpI = dyn_cast<CmpInst>(Val: I)) { |
9470 | if (!calculateCmpInst(A, CmpI, T, CtxI, QuerriedAAs)) |
9471 | return false; |
9472 | } else if (auto *CastI = dyn_cast<CastInst>(Val: I)) { |
9473 | if (!calculateCastInst(A, CastI, T, CtxI, QuerriedAAs)) |
9474 | return false; |
9475 | } else { |
9476 | // Give up with other instructions. |
9477 | // TODO: Add other instructions |
9478 | |
9479 | T.indicatePessimisticFixpoint(); |
9480 | return false; |
9481 | } |
9482 | |
9483 | // Catch circular reasoning in a pessimistic way for now. |
9484 | // TODO: Check how the range evolves and if we stripped anything, see also |
9485 | // AADereferenceable or AAAlign for similar situations. |
9486 | for (const AAValueConstantRange *QueriedAA : QuerriedAAs) { |
9487 | if (QueriedAA != this) |
9488 | continue; |
9489 | // If we are in a stady state we do not need to worry. |
9490 | if (T.getAssumed() == getState().getAssumed()) |
9491 | continue; |
9492 | T.indicatePessimisticFixpoint(); |
9493 | } |
9494 | |
9495 | return T.isValidState(); |
9496 | }; |
9497 | |
9498 | if (!VisitValueCB(getAssociatedValue(), getCtxI())) |
9499 | return indicatePessimisticFixpoint(); |
9500 | |
9501 | // Ensure that long def-use chains can't cause circular reasoning either by |
9502 | // introducing a cutoff below. |
9503 | if (clampStateAndIndicateChange(S&: getState(), R: T) == ChangeStatus::UNCHANGED) |
9504 | return ChangeStatus::UNCHANGED; |
9505 | if (++NumChanges > MaxNumChanges) { |
9506 | LLVM_DEBUG(dbgs() << "[AAValueConstantRange] performed " << NumChanges |
9507 | << " but only " << MaxNumChanges |
9508 | << " are allowed to avoid cyclic reasoning." ); |
9509 | return indicatePessimisticFixpoint(); |
9510 | } |
9511 | return ChangeStatus::CHANGED; |
9512 | } |
9513 | |
9514 | /// See AbstractAttribute::trackStatistics() |
9515 | void trackStatistics() const override { |
9516 | STATS_DECLTRACK_FLOATING_ATTR(value_range) |
9517 | } |
9518 | |
9519 | /// Tracker to bail after too many widening steps of the constant range. |
9520 | int NumChanges = 0; |
9521 | |
9522 | /// Upper bound for the number of allowed changes (=widening steps) for the |
9523 | /// constant range before we give up. |
9524 | static constexpr int MaxNumChanges = 5; |
9525 | }; |
9526 | |
9527 | struct AAValueConstantRangeFunction : AAValueConstantRangeImpl { |
9528 | AAValueConstantRangeFunction(const IRPosition &IRP, Attributor &A) |
9529 | : AAValueConstantRangeImpl(IRP, A) {} |
9530 | |
9531 | /// See AbstractAttribute::initialize(...). |
9532 | ChangeStatus updateImpl(Attributor &A) override { |
9533 | llvm_unreachable("AAValueConstantRange(Function|CallSite)::updateImpl will " |
9534 | "not be called" ); |
9535 | } |
9536 | |
9537 | /// See AbstractAttribute::trackStatistics() |
9538 | void trackStatistics() const override { STATS_DECLTRACK_FN_ATTR(value_range) } |
9539 | }; |
9540 | |
9541 | struct AAValueConstantRangeCallSite : AAValueConstantRangeFunction { |
9542 | AAValueConstantRangeCallSite(const IRPosition &IRP, Attributor &A) |
9543 | : AAValueConstantRangeFunction(IRP, A) {} |
9544 | |
9545 | /// See AbstractAttribute::trackStatistics() |
9546 | void trackStatistics() const override { STATS_DECLTRACK_CS_ATTR(value_range) } |
9547 | }; |
9548 | |
9549 | struct AAValueConstantRangeCallSiteReturned |
9550 | : AACalleeToCallSite<AAValueConstantRange, AAValueConstantRangeImpl, |
9551 | AAValueConstantRangeImpl::StateType, |
9552 | /* IntroduceCallBaseContext */ true> { |
9553 | AAValueConstantRangeCallSiteReturned(const IRPosition &IRP, Attributor &A) |
9554 | : AACalleeToCallSite<AAValueConstantRange, AAValueConstantRangeImpl, |
9555 | AAValueConstantRangeImpl::StateType, |
9556 | /* IntroduceCallBaseContext */ true>(IRP, A) {} |
9557 | |
9558 | /// See AbstractAttribute::initialize(...). |
9559 | void initialize(Attributor &A) override { |
9560 | // If it is a load instruction with range metadata, use the metadata. |
9561 | if (CallInst *CI = dyn_cast<CallInst>(Val: &getAssociatedValue())) |
9562 | if (auto *RangeMD = CI->getMetadata(KindID: LLVMContext::MD_range)) |
9563 | intersectKnown(R: getConstantRangeFromMetadata(RangeMD: *RangeMD)); |
9564 | |
9565 | AAValueConstantRangeImpl::initialize(A); |
9566 | } |
9567 | |
9568 | /// See AbstractAttribute::trackStatistics() |
9569 | void trackStatistics() const override { |
9570 | STATS_DECLTRACK_CSRET_ATTR(value_range) |
9571 | } |
9572 | }; |
9573 | struct AAValueConstantRangeCallSiteArgument : AAValueConstantRangeFloating { |
9574 | AAValueConstantRangeCallSiteArgument(const IRPosition &IRP, Attributor &A) |
9575 | : AAValueConstantRangeFloating(IRP, A) {} |
9576 | |
9577 | /// See AbstractAttribute::manifest() |
9578 | ChangeStatus manifest(Attributor &A) override { |
9579 | return ChangeStatus::UNCHANGED; |
9580 | } |
9581 | |
9582 | /// See AbstractAttribute::trackStatistics() |
9583 | void trackStatistics() const override { |
9584 | STATS_DECLTRACK_CSARG_ATTR(value_range) |
9585 | } |
9586 | }; |
9587 | } // namespace |
9588 | |
9589 | /// ------------------ Potential Values Attribute ------------------------- |
9590 | |
9591 | namespace { |
9592 | struct AAPotentialConstantValuesImpl : AAPotentialConstantValues { |
9593 | using StateType = PotentialConstantIntValuesState; |
9594 | |
9595 | AAPotentialConstantValuesImpl(const IRPosition &IRP, Attributor &A) |
9596 | : AAPotentialConstantValues(IRP, A) {} |
9597 | |
9598 | /// See AbstractAttribute::initialize(..). |
9599 | void initialize(Attributor &A) override { |
9600 | if (A.hasSimplificationCallback(IRP: getIRPosition())) |
9601 | indicatePessimisticFixpoint(); |
9602 | else |
9603 | AAPotentialConstantValues::initialize(A); |
9604 | } |
9605 | |
9606 | bool fillSetWithConstantValues(Attributor &A, const IRPosition &IRP, SetTy &S, |
9607 | bool &ContainsUndef, bool ForSelf) { |
9608 | SmallVector<AA::ValueAndContext> Values; |
9609 | bool UsedAssumedInformation = false; |
9610 | if (!A.getAssumedSimplifiedValues(IRP, AA: *this, Values, S: AA::Interprocedural, |
9611 | UsedAssumedInformation)) { |
9612 | // Avoid recursion when the caller is computing constant values for this |
9613 | // IRP itself. |
9614 | if (ForSelf) |
9615 | return false; |
9616 | if (!IRP.getAssociatedType()->isIntegerTy()) |
9617 | return false; |
9618 | auto *PotentialValuesAA = A.getAAFor<AAPotentialConstantValues>( |
9619 | QueryingAA: *this, IRP, DepClass: DepClassTy::REQUIRED); |
9620 | if (!PotentialValuesAA || !PotentialValuesAA->getState().isValidState()) |
9621 | return false; |
9622 | ContainsUndef = PotentialValuesAA->getState().undefIsContained(); |
9623 | S = PotentialValuesAA->getState().getAssumedSet(); |
9624 | return true; |
9625 | } |
9626 | |
9627 | // Copy all the constant values, except UndefValue. ContainsUndef is true |
9628 | // iff Values contains only UndefValue instances. If there are other known |
9629 | // constants, then UndefValue is dropped. |
9630 | ContainsUndef = false; |
9631 | for (auto &It : Values) { |
9632 | if (isa<UndefValue>(Val: It.getValue())) { |
9633 | ContainsUndef = true; |
9634 | continue; |
9635 | } |
9636 | auto *CI = dyn_cast<ConstantInt>(Val: It.getValue()); |
9637 | if (!CI) |
9638 | return false; |
9639 | S.insert(X: CI->getValue()); |
9640 | } |
9641 | ContainsUndef &= S.empty(); |
9642 | |
9643 | return true; |
9644 | } |
9645 | |
9646 | /// See AbstractAttribute::getAsStr(). |
9647 | const std::string getAsStr(Attributor *A) const override { |
9648 | std::string Str; |
9649 | llvm::raw_string_ostream OS(Str); |
9650 | OS << getState(); |
9651 | return OS.str(); |
9652 | } |
9653 | |
9654 | /// See AbstractAttribute::updateImpl(...). |
9655 | ChangeStatus updateImpl(Attributor &A) override { |
9656 | return indicatePessimisticFixpoint(); |
9657 | } |
9658 | }; |
9659 | |
9660 | struct AAPotentialConstantValuesArgument final |
9661 | : AAArgumentFromCallSiteArguments<AAPotentialConstantValues, |
9662 | AAPotentialConstantValuesImpl, |
9663 | PotentialConstantIntValuesState> { |
9664 | using Base = AAArgumentFromCallSiteArguments<AAPotentialConstantValues, |
9665 | AAPotentialConstantValuesImpl, |
9666 | PotentialConstantIntValuesState>; |
9667 | AAPotentialConstantValuesArgument(const IRPosition &IRP, Attributor &A) |
9668 | : Base(IRP, A) {} |
9669 | |
9670 | /// See AbstractAttribute::trackStatistics() |
9671 | void trackStatistics() const override { |
9672 | STATS_DECLTRACK_ARG_ATTR(potential_values) |
9673 | } |
9674 | }; |
9675 | |
9676 | struct AAPotentialConstantValuesReturned |
9677 | : AAReturnedFromReturnedValues<AAPotentialConstantValues, |
9678 | AAPotentialConstantValuesImpl> { |
9679 | using Base = AAReturnedFromReturnedValues<AAPotentialConstantValues, |
9680 | AAPotentialConstantValuesImpl>; |
9681 | AAPotentialConstantValuesReturned(const IRPosition &IRP, Attributor &A) |
9682 | : Base(IRP, A) {} |
9683 | |
9684 | void initialize(Attributor &A) override { |
9685 | if (!A.isFunctionIPOAmendable(F: *getAssociatedFunction())) |
9686 | indicatePessimisticFixpoint(); |
9687 | Base::initialize(A); |
9688 | } |
9689 | |
9690 | /// See AbstractAttribute::trackStatistics() |
9691 | void trackStatistics() const override { |
9692 | STATS_DECLTRACK_FNRET_ATTR(potential_values) |
9693 | } |
9694 | }; |
9695 | |
9696 | struct AAPotentialConstantValuesFloating : AAPotentialConstantValuesImpl { |
9697 | AAPotentialConstantValuesFloating(const IRPosition &IRP, Attributor &A) |
9698 | : AAPotentialConstantValuesImpl(IRP, A) {} |
9699 | |
9700 | /// See AbstractAttribute::initialize(..). |
9701 | void initialize(Attributor &A) override { |
9702 | AAPotentialConstantValuesImpl::initialize(A); |
9703 | if (isAtFixpoint()) |
9704 | return; |
9705 | |
9706 | Value &V = getAssociatedValue(); |
9707 | |
9708 | if (auto *C = dyn_cast<ConstantInt>(Val: &V)) { |
9709 | unionAssumed(C: C->getValue()); |
9710 | indicateOptimisticFixpoint(); |
9711 | return; |
9712 | } |
9713 | |
9714 | if (isa<UndefValue>(Val: &V)) { |
9715 | unionAssumedWithUndef(); |
9716 | indicateOptimisticFixpoint(); |
9717 | return; |
9718 | } |
9719 | |
9720 | if (isa<BinaryOperator>(Val: &V) || isa<ICmpInst>(Val: &V) || isa<CastInst>(Val: &V)) |
9721 | return; |
9722 | |
9723 | if (isa<SelectInst>(Val: V) || isa<PHINode>(Val: V) || isa<LoadInst>(Val: V)) |
9724 | return; |
9725 | |
9726 | indicatePessimisticFixpoint(); |
9727 | |
9728 | LLVM_DEBUG(dbgs() << "[AAPotentialConstantValues] We give up: " |
9729 | << getAssociatedValue() << "\n" ); |
9730 | } |
9731 | |
9732 | static bool calculateICmpInst(const ICmpInst *ICI, const APInt &LHS, |
9733 | const APInt &RHS) { |
9734 | return ICmpInst::compare(LHS, RHS, Pred: ICI->getPredicate()); |
9735 | } |
9736 | |
9737 | static APInt calculateCastInst(const CastInst *CI, const APInt &Src, |
9738 | uint32_t ResultBitWidth) { |
9739 | Instruction::CastOps CastOp = CI->getOpcode(); |
9740 | switch (CastOp) { |
9741 | default: |
9742 | llvm_unreachable("unsupported or not integer cast" ); |
9743 | case Instruction::Trunc: |
9744 | return Src.trunc(width: ResultBitWidth); |
9745 | case Instruction::SExt: |
9746 | return Src.sext(width: ResultBitWidth); |
9747 | case Instruction::ZExt: |
9748 | return Src.zext(width: ResultBitWidth); |
9749 | case Instruction::BitCast: |
9750 | return Src; |
9751 | } |
9752 | } |
9753 | |
9754 | static APInt calculateBinaryOperator(const BinaryOperator *BinOp, |
9755 | const APInt &LHS, const APInt &RHS, |
9756 | bool &SkipOperation, bool &Unsupported) { |
9757 | Instruction::BinaryOps BinOpcode = BinOp->getOpcode(); |
9758 | // Unsupported is set to true when the binary operator is not supported. |
9759 | // SkipOperation is set to true when UB occur with the given operand pair |
9760 | // (LHS, RHS). |
9761 | // TODO: we should look at nsw and nuw keywords to handle operations |
9762 | // that create poison or undef value. |
9763 | switch (BinOpcode) { |
9764 | default: |
9765 | Unsupported = true; |
9766 | return LHS; |
9767 | case Instruction::Add: |
9768 | return LHS + RHS; |
9769 | case Instruction::Sub: |
9770 | return LHS - RHS; |
9771 | case Instruction::Mul: |
9772 | return LHS * RHS; |
9773 | case Instruction::UDiv: |
9774 | if (RHS.isZero()) { |
9775 | SkipOperation = true; |
9776 | return LHS; |
9777 | } |
9778 | return LHS.udiv(RHS); |
9779 | case Instruction::SDiv: |
9780 | if (RHS.isZero()) { |
9781 | SkipOperation = true; |
9782 | return LHS; |
9783 | } |
9784 | return LHS.sdiv(RHS); |
9785 | case Instruction::URem: |
9786 | if (RHS.isZero()) { |
9787 | SkipOperation = true; |
9788 | return LHS; |
9789 | } |
9790 | return LHS.urem(RHS); |
9791 | case Instruction::SRem: |
9792 | if (RHS.isZero()) { |
9793 | SkipOperation = true; |
9794 | return LHS; |
9795 | } |
9796 | return LHS.srem(RHS); |
9797 | case Instruction::Shl: |
9798 | return LHS.shl(ShiftAmt: RHS); |
9799 | case Instruction::LShr: |
9800 | return LHS.lshr(ShiftAmt: RHS); |
9801 | case Instruction::AShr: |
9802 | return LHS.ashr(ShiftAmt: RHS); |
9803 | case Instruction::And: |
9804 | return LHS & RHS; |
9805 | case Instruction::Or: |
9806 | return LHS | RHS; |
9807 | case Instruction::Xor: |
9808 | return LHS ^ RHS; |
9809 | } |
9810 | } |
9811 | |
9812 | bool calculateBinaryOperatorAndTakeUnion(const BinaryOperator *BinOp, |
9813 | const APInt &LHS, const APInt &RHS) { |
9814 | bool SkipOperation = false; |
9815 | bool Unsupported = false; |
9816 | APInt Result = |
9817 | calculateBinaryOperator(BinOp, LHS, RHS, SkipOperation, Unsupported); |
9818 | if (Unsupported) |
9819 | return false; |
9820 | // If SkipOperation is true, we can ignore this operand pair (L, R). |
9821 | if (!SkipOperation) |
9822 | unionAssumed(C: Result); |
9823 | return isValidState(); |
9824 | } |
9825 | |
9826 | ChangeStatus updateWithICmpInst(Attributor &A, ICmpInst *ICI) { |
9827 | auto AssumedBefore = getAssumed(); |
9828 | Value *LHS = ICI->getOperand(i_nocapture: 0); |
9829 | Value *RHS = ICI->getOperand(i_nocapture: 1); |
9830 | |
9831 | bool LHSContainsUndef = false, RHSContainsUndef = false; |
9832 | SetTy LHSAAPVS, RHSAAPVS; |
9833 | if (!fillSetWithConstantValues(A, IRP: IRPosition::value(V: *LHS), S&: LHSAAPVS, |
9834 | ContainsUndef&: LHSContainsUndef, /* ForSelf */ false) || |
9835 | !fillSetWithConstantValues(A, IRP: IRPosition::value(V: *RHS), S&: RHSAAPVS, |
9836 | ContainsUndef&: RHSContainsUndef, /* ForSelf */ false)) |
9837 | return indicatePessimisticFixpoint(); |
9838 | |
9839 | // TODO: make use of undef flag to limit potential values aggressively. |
9840 | bool MaybeTrue = false, MaybeFalse = false; |
9841 | const APInt Zero(RHS->getType()->getIntegerBitWidth(), 0); |
9842 | if (LHSContainsUndef && RHSContainsUndef) { |
9843 | // The result of any comparison between undefs can be soundly replaced |
9844 | // with undef. |
9845 | unionAssumedWithUndef(); |
9846 | } else if (LHSContainsUndef) { |
9847 | for (const APInt &R : RHSAAPVS) { |
9848 | bool CmpResult = calculateICmpInst(ICI, LHS: Zero, RHS: R); |
9849 | MaybeTrue |= CmpResult; |
9850 | MaybeFalse |= !CmpResult; |
9851 | if (MaybeTrue & MaybeFalse) |
9852 | return indicatePessimisticFixpoint(); |
9853 | } |
9854 | } else if (RHSContainsUndef) { |
9855 | for (const APInt &L : LHSAAPVS) { |
9856 | bool CmpResult = calculateICmpInst(ICI, LHS: L, RHS: Zero); |
9857 | MaybeTrue |= CmpResult; |
9858 | MaybeFalse |= !CmpResult; |
9859 | if (MaybeTrue & MaybeFalse) |
9860 | return indicatePessimisticFixpoint(); |
9861 | } |
9862 | } else { |
9863 | for (const APInt &L : LHSAAPVS) { |
9864 | for (const APInt &R : RHSAAPVS) { |
9865 | bool CmpResult = calculateICmpInst(ICI, LHS: L, RHS: R); |
9866 | MaybeTrue |= CmpResult; |
9867 | MaybeFalse |= !CmpResult; |
9868 | if (MaybeTrue & MaybeFalse) |
9869 | return indicatePessimisticFixpoint(); |
9870 | } |
9871 | } |
9872 | } |
9873 | if (MaybeTrue) |
9874 | unionAssumed(C: APInt(/* numBits */ 1, /* val */ 1)); |
9875 | if (MaybeFalse) |
9876 | unionAssumed(C: APInt(/* numBits */ 1, /* val */ 0)); |
9877 | return AssumedBefore == getAssumed() ? ChangeStatus::UNCHANGED |
9878 | : ChangeStatus::CHANGED; |
9879 | } |
9880 | |
9881 | ChangeStatus updateWithSelectInst(Attributor &A, SelectInst *SI) { |
9882 | auto AssumedBefore = getAssumed(); |
9883 | Value *LHS = SI->getTrueValue(); |
9884 | Value *RHS = SI->getFalseValue(); |
9885 | |
9886 | bool UsedAssumedInformation = false; |
9887 | std::optional<Constant *> C = A.getAssumedConstant( |
9888 | V: *SI->getCondition(), AA: *this, UsedAssumedInformation); |
9889 | |
9890 | // Check if we only need one operand. |
9891 | bool OnlyLeft = false, OnlyRight = false; |
9892 | if (C && *C && (*C)->isOneValue()) |
9893 | OnlyLeft = true; |
9894 | else if (C && *C && (*C)->isZeroValue()) |
9895 | OnlyRight = true; |
9896 | |
9897 | bool LHSContainsUndef = false, RHSContainsUndef = false; |
9898 | SetTy LHSAAPVS, RHSAAPVS; |
9899 | if (!OnlyRight && |
9900 | !fillSetWithConstantValues(A, IRP: IRPosition::value(V: *LHS), S&: LHSAAPVS, |
9901 | ContainsUndef&: LHSContainsUndef, /* ForSelf */ false)) |
9902 | return indicatePessimisticFixpoint(); |
9903 | |
9904 | if (!OnlyLeft && |
9905 | !fillSetWithConstantValues(A, IRP: IRPosition::value(V: *RHS), S&: RHSAAPVS, |
9906 | ContainsUndef&: RHSContainsUndef, /* ForSelf */ false)) |
9907 | return indicatePessimisticFixpoint(); |
9908 | |
9909 | if (OnlyLeft || OnlyRight) { |
9910 | // select (true/false), lhs, rhs |
9911 | auto *OpAA = OnlyLeft ? &LHSAAPVS : &RHSAAPVS; |
9912 | auto Undef = OnlyLeft ? LHSContainsUndef : RHSContainsUndef; |
9913 | |
9914 | if (Undef) |
9915 | unionAssumedWithUndef(); |
9916 | else { |
9917 | for (const auto &It : *OpAA) |
9918 | unionAssumed(C: It); |
9919 | } |
9920 | |
9921 | } else if (LHSContainsUndef && RHSContainsUndef) { |
9922 | // select i1 *, undef , undef => undef |
9923 | unionAssumedWithUndef(); |
9924 | } else { |
9925 | for (const auto &It : LHSAAPVS) |
9926 | unionAssumed(C: It); |
9927 | for (const auto &It : RHSAAPVS) |
9928 | unionAssumed(C: It); |
9929 | } |
9930 | return AssumedBefore == getAssumed() ? ChangeStatus::UNCHANGED |
9931 | : ChangeStatus::CHANGED; |
9932 | } |
9933 | |
9934 | ChangeStatus updateWithCastInst(Attributor &A, CastInst *CI) { |
9935 | auto AssumedBefore = getAssumed(); |
9936 | if (!CI->isIntegerCast()) |
9937 | return indicatePessimisticFixpoint(); |
9938 | assert(CI->getNumOperands() == 1 && "Expected cast to be unary!" ); |
9939 | uint32_t ResultBitWidth = CI->getDestTy()->getIntegerBitWidth(); |
9940 | Value *Src = CI->getOperand(i_nocapture: 0); |
9941 | |
9942 | bool SrcContainsUndef = false; |
9943 | SetTy SrcPVS; |
9944 | if (!fillSetWithConstantValues(A, IRP: IRPosition::value(V: *Src), S&: SrcPVS, |
9945 | ContainsUndef&: SrcContainsUndef, /* ForSelf */ false)) |
9946 | return indicatePessimisticFixpoint(); |
9947 | |
9948 | if (SrcContainsUndef) |
9949 | unionAssumedWithUndef(); |
9950 | else { |
9951 | for (const APInt &S : SrcPVS) { |
9952 | APInt T = calculateCastInst(CI, Src: S, ResultBitWidth); |
9953 | unionAssumed(C: T); |
9954 | } |
9955 | } |
9956 | return AssumedBefore == getAssumed() ? ChangeStatus::UNCHANGED |
9957 | : ChangeStatus::CHANGED; |
9958 | } |
9959 | |
9960 | ChangeStatus updateWithBinaryOperator(Attributor &A, BinaryOperator *BinOp) { |
9961 | auto AssumedBefore = getAssumed(); |
9962 | Value *LHS = BinOp->getOperand(i_nocapture: 0); |
9963 | Value *RHS = BinOp->getOperand(i_nocapture: 1); |
9964 | |
9965 | bool LHSContainsUndef = false, RHSContainsUndef = false; |
9966 | SetTy LHSAAPVS, RHSAAPVS; |
9967 | if (!fillSetWithConstantValues(A, IRP: IRPosition::value(V: *LHS), S&: LHSAAPVS, |
9968 | ContainsUndef&: LHSContainsUndef, /* ForSelf */ false) || |
9969 | !fillSetWithConstantValues(A, IRP: IRPosition::value(V: *RHS), S&: RHSAAPVS, |
9970 | ContainsUndef&: RHSContainsUndef, /* ForSelf */ false)) |
9971 | return indicatePessimisticFixpoint(); |
9972 | |
9973 | const APInt Zero = APInt(LHS->getType()->getIntegerBitWidth(), 0); |
9974 | |
9975 | // TODO: make use of undef flag to limit potential values aggressively. |
9976 | if (LHSContainsUndef && RHSContainsUndef) { |
9977 | if (!calculateBinaryOperatorAndTakeUnion(BinOp, LHS: Zero, RHS: Zero)) |
9978 | return indicatePessimisticFixpoint(); |
9979 | } else if (LHSContainsUndef) { |
9980 | for (const APInt &R : RHSAAPVS) { |
9981 | if (!calculateBinaryOperatorAndTakeUnion(BinOp, LHS: Zero, RHS: R)) |
9982 | return indicatePessimisticFixpoint(); |
9983 | } |
9984 | } else if (RHSContainsUndef) { |
9985 | for (const APInt &L : LHSAAPVS) { |
9986 | if (!calculateBinaryOperatorAndTakeUnion(BinOp, LHS: L, RHS: Zero)) |
9987 | return indicatePessimisticFixpoint(); |
9988 | } |
9989 | } else { |
9990 | for (const APInt &L : LHSAAPVS) { |
9991 | for (const APInt &R : RHSAAPVS) { |
9992 | if (!calculateBinaryOperatorAndTakeUnion(BinOp, LHS: L, RHS: R)) |
9993 | return indicatePessimisticFixpoint(); |
9994 | } |
9995 | } |
9996 | } |
9997 | return AssumedBefore == getAssumed() ? ChangeStatus::UNCHANGED |
9998 | : ChangeStatus::CHANGED; |
9999 | } |
10000 | |
10001 | ChangeStatus updateWithInstruction(Attributor &A, Instruction *Inst) { |
10002 | auto AssumedBefore = getAssumed(); |
10003 | SetTy Incoming; |
10004 | bool ContainsUndef; |
10005 | if (!fillSetWithConstantValues(A, IRP: IRPosition::value(V: *Inst), S&: Incoming, |
10006 | ContainsUndef, /* ForSelf */ true)) |
10007 | return indicatePessimisticFixpoint(); |
10008 | if (ContainsUndef) { |
10009 | unionAssumedWithUndef(); |
10010 | } else { |
10011 | for (const auto &It : Incoming) |
10012 | unionAssumed(C: It); |
10013 | } |
10014 | return AssumedBefore == getAssumed() ? ChangeStatus::UNCHANGED |
10015 | : ChangeStatus::CHANGED; |
10016 | } |
10017 | |
10018 | /// See AbstractAttribute::updateImpl(...). |
10019 | ChangeStatus updateImpl(Attributor &A) override { |
10020 | Value &V = getAssociatedValue(); |
10021 | Instruction *I = dyn_cast<Instruction>(Val: &V); |
10022 | |
10023 | if (auto *ICI = dyn_cast<ICmpInst>(Val: I)) |
10024 | return updateWithICmpInst(A, ICI); |
10025 | |
10026 | if (auto *SI = dyn_cast<SelectInst>(Val: I)) |
10027 | return updateWithSelectInst(A, SI); |
10028 | |
10029 | if (auto *CI = dyn_cast<CastInst>(Val: I)) |
10030 | return updateWithCastInst(A, CI); |
10031 | |
10032 | if (auto *BinOp = dyn_cast<BinaryOperator>(Val: I)) |
10033 | return updateWithBinaryOperator(A, BinOp); |
10034 | |
10035 | if (isa<PHINode>(Val: I) || isa<LoadInst>(Val: I)) |
10036 | return updateWithInstruction(A, Inst: I); |
10037 | |
10038 | return indicatePessimisticFixpoint(); |
10039 | } |
10040 | |
10041 | /// See AbstractAttribute::trackStatistics() |
10042 | void trackStatistics() const override { |
10043 | STATS_DECLTRACK_FLOATING_ATTR(potential_values) |
10044 | } |
10045 | }; |
10046 | |
10047 | struct AAPotentialConstantValuesFunction : AAPotentialConstantValuesImpl { |
10048 | AAPotentialConstantValuesFunction(const IRPosition &IRP, Attributor &A) |
10049 | : AAPotentialConstantValuesImpl(IRP, A) {} |
10050 | |
10051 | /// See AbstractAttribute::initialize(...). |
10052 | ChangeStatus updateImpl(Attributor &A) override { |
10053 | llvm_unreachable( |
10054 | "AAPotentialConstantValues(Function|CallSite)::updateImpl will " |
10055 | "not be called" ); |
10056 | } |
10057 | |
10058 | /// See AbstractAttribute::trackStatistics() |
10059 | void trackStatistics() const override { |
10060 | STATS_DECLTRACK_FN_ATTR(potential_values) |
10061 | } |
10062 | }; |
10063 | |
10064 | struct AAPotentialConstantValuesCallSite : AAPotentialConstantValuesFunction { |
10065 | AAPotentialConstantValuesCallSite(const IRPosition &IRP, Attributor &A) |
10066 | : AAPotentialConstantValuesFunction(IRP, A) {} |
10067 | |
10068 | /// See AbstractAttribute::trackStatistics() |
10069 | void trackStatistics() const override { |
10070 | STATS_DECLTRACK_CS_ATTR(potential_values) |
10071 | } |
10072 | }; |
10073 | |
10074 | struct AAPotentialConstantValuesCallSiteReturned |
10075 | : AACalleeToCallSite<AAPotentialConstantValues, |
10076 | AAPotentialConstantValuesImpl> { |
10077 | AAPotentialConstantValuesCallSiteReturned(const IRPosition &IRP, |
10078 | Attributor &A) |
10079 | : AACalleeToCallSite<AAPotentialConstantValues, |
10080 | AAPotentialConstantValuesImpl>(IRP, A) {} |
10081 | |
10082 | /// See AbstractAttribute::trackStatistics() |
10083 | void trackStatistics() const override { |
10084 | STATS_DECLTRACK_CSRET_ATTR(potential_values) |
10085 | } |
10086 | }; |
10087 | |
10088 | struct AAPotentialConstantValuesCallSiteArgument |
10089 | : AAPotentialConstantValuesFloating { |
10090 | AAPotentialConstantValuesCallSiteArgument(const IRPosition &IRP, |
10091 | Attributor &A) |
10092 | : AAPotentialConstantValuesFloating(IRP, A) {} |
10093 | |
10094 | /// See AbstractAttribute::initialize(..). |
10095 | void initialize(Attributor &A) override { |
10096 | AAPotentialConstantValuesImpl::initialize(A); |
10097 | if (isAtFixpoint()) |
10098 | return; |
10099 | |
10100 | Value &V = getAssociatedValue(); |
10101 | |
10102 | if (auto *C = dyn_cast<ConstantInt>(Val: &V)) { |
10103 | unionAssumed(C: C->getValue()); |
10104 | indicateOptimisticFixpoint(); |
10105 | return; |
10106 | } |
10107 | |
10108 | if (isa<UndefValue>(Val: &V)) { |
10109 | unionAssumedWithUndef(); |
10110 | indicateOptimisticFixpoint(); |
10111 | return; |
10112 | } |
10113 | } |
10114 | |
10115 | /// See AbstractAttribute::updateImpl(...). |
10116 | ChangeStatus updateImpl(Attributor &A) override { |
10117 | Value &V = getAssociatedValue(); |
10118 | auto AssumedBefore = getAssumed(); |
10119 | auto *AA = A.getAAFor<AAPotentialConstantValues>( |
10120 | QueryingAA: *this, IRP: IRPosition::value(V), DepClass: DepClassTy::REQUIRED); |
10121 | if (!AA) |
10122 | return indicatePessimisticFixpoint(); |
10123 | const auto &S = AA->getAssumed(); |
10124 | unionAssumed(PVS: S); |
10125 | return AssumedBefore == getAssumed() ? ChangeStatus::UNCHANGED |
10126 | : ChangeStatus::CHANGED; |
10127 | } |
10128 | |
10129 | /// See AbstractAttribute::trackStatistics() |
10130 | void trackStatistics() const override { |
10131 | STATS_DECLTRACK_CSARG_ATTR(potential_values) |
10132 | } |
10133 | }; |
10134 | } // namespace |
10135 | |
10136 | /// ------------------------ NoUndef Attribute --------------------------------- |
10137 | bool AANoUndef::isImpliedByIR(Attributor &A, const IRPosition &IRP, |
10138 | Attribute::AttrKind ImpliedAttributeKind, |
10139 | bool IgnoreSubsumingPositions) { |
10140 | assert(ImpliedAttributeKind == Attribute::NoUndef && |
10141 | "Unexpected attribute kind" ); |
10142 | if (A.hasAttr(IRP, {Attribute::NoUndef}, IgnoreSubsumingPositions, |
10143 | Attribute::NoUndef)) |
10144 | return true; |
10145 | |
10146 | Value &Val = IRP.getAssociatedValue(); |
10147 | if (IRP.getPositionKind() != IRPosition::IRP_RETURNED && |
10148 | isGuaranteedNotToBeUndefOrPoison(V: &Val)) { |
10149 | LLVMContext &Ctx = Val.getContext(); |
10150 | A.manifestAttrs(IRP, Attribute::get(Ctx, Attribute::NoUndef)); |
10151 | return true; |
10152 | } |
10153 | |
10154 | return false; |
10155 | } |
10156 | |
10157 | namespace { |
10158 | struct AANoUndefImpl : AANoUndef { |
10159 | AANoUndefImpl(const IRPosition &IRP, Attributor &A) : AANoUndef(IRP, A) {} |
10160 | |
10161 | /// See AbstractAttribute::initialize(...). |
10162 | void initialize(Attributor &A) override { |
10163 | Value &V = getAssociatedValue(); |
10164 | if (isa<UndefValue>(Val: V)) |
10165 | indicatePessimisticFixpoint(); |
10166 | assert(!isImpliedByIR(A, getIRPosition(), Attribute::NoUndef)); |
10167 | } |
10168 | |
10169 | /// See followUsesInMBEC |
10170 | bool followUseInMBEC(Attributor &A, const Use *U, const Instruction *I, |
10171 | AANoUndef::StateType &State) { |
10172 | const Value *UseV = U->get(); |
10173 | const DominatorTree *DT = nullptr; |
10174 | AssumptionCache *AC = nullptr; |
10175 | InformationCache &InfoCache = A.getInfoCache(); |
10176 | if (Function *F = getAnchorScope()) { |
10177 | DT = InfoCache.getAnalysisResultForFunction<DominatorTreeAnalysis>(F: *F); |
10178 | AC = InfoCache.getAnalysisResultForFunction<AssumptionAnalysis>(F: *F); |
10179 | } |
10180 | State.setKnown(isGuaranteedNotToBeUndefOrPoison(V: UseV, AC, CtxI: I, DT)); |
10181 | bool TrackUse = false; |
10182 | // Track use for instructions which must produce undef or poison bits when |
10183 | // at least one operand contains such bits. |
10184 | if (isa<CastInst>(Val: *I) || isa<GetElementPtrInst>(Val: *I)) |
10185 | TrackUse = true; |
10186 | return TrackUse; |
10187 | } |
10188 | |
10189 | /// See AbstractAttribute::getAsStr(). |
10190 | const std::string getAsStr(Attributor *A) const override { |
10191 | return getAssumed() ? "noundef" : "may-undef-or-poison" ; |
10192 | } |
10193 | |
10194 | ChangeStatus manifest(Attributor &A) override { |
10195 | // We don't manifest noundef attribute for dead positions because the |
10196 | // associated values with dead positions would be replaced with undef |
10197 | // values. |
10198 | bool UsedAssumedInformation = false; |
10199 | if (A.isAssumedDead(getIRPosition(), nullptr, nullptr, |
10200 | UsedAssumedInformation)) |
10201 | return ChangeStatus::UNCHANGED; |
10202 | // A position whose simplified value does not have any value is |
10203 | // considered to be dead. We don't manifest noundef in such positions for |
10204 | // the same reason above. |
10205 | if (!A.getAssumedSimplified(getIRPosition(), *this, UsedAssumedInformation, |
10206 | AA::Interprocedural) |
10207 | .has_value()) |
10208 | return ChangeStatus::UNCHANGED; |
10209 | return AANoUndef::manifest(A); |
10210 | } |
10211 | }; |
10212 | |
10213 | struct AANoUndefFloating : public AANoUndefImpl { |
10214 | AANoUndefFloating(const IRPosition &IRP, Attributor &A) |
10215 | : AANoUndefImpl(IRP, A) {} |
10216 | |
10217 | /// See AbstractAttribute::initialize(...). |
10218 | void initialize(Attributor &A) override { |
10219 | AANoUndefImpl::initialize(A); |
10220 | if (!getState().isAtFixpoint() && getAnchorScope() && |
10221 | !getAnchorScope()->isDeclaration()) |
10222 | if (Instruction *CtxI = getCtxI()) |
10223 | followUsesInMBEC(*this, A, getState(), *CtxI); |
10224 | } |
10225 | |
10226 | /// See AbstractAttribute::updateImpl(...). |
10227 | ChangeStatus updateImpl(Attributor &A) override { |
10228 | auto VisitValueCB = [&](const IRPosition &IRP) -> bool { |
10229 | bool IsKnownNoUndef; |
10230 | return AA::hasAssumedIRAttr<Attribute::NoUndef>( |
10231 | A, this, IRP, DepClassTy::REQUIRED, IsKnownNoUndef); |
10232 | }; |
10233 | |
10234 | bool Stripped; |
10235 | bool UsedAssumedInformation = false; |
10236 | Value *AssociatedValue = &getAssociatedValue(); |
10237 | SmallVector<AA::ValueAndContext> Values; |
10238 | if (!A.getAssumedSimplifiedValues(IRP: getIRPosition(), AA: *this, Values, |
10239 | S: AA::AnyScope, UsedAssumedInformation)) |
10240 | Stripped = false; |
10241 | else |
10242 | Stripped = |
10243 | Values.size() != 1 || Values.front().getValue() != AssociatedValue; |
10244 | |
10245 | if (!Stripped) { |
10246 | // If we haven't stripped anything we might still be able to use a |
10247 | // different AA, but only if the IRP changes. Effectively when we |
10248 | // interpret this not as a call site value but as a floating/argument |
10249 | // value. |
10250 | const IRPosition AVIRP = IRPosition::value(V: *AssociatedValue); |
10251 | if (AVIRP == getIRPosition() || !VisitValueCB(AVIRP)) |
10252 | return indicatePessimisticFixpoint(); |
10253 | return ChangeStatus::UNCHANGED; |
10254 | } |
10255 | |
10256 | for (const auto &VAC : Values) |
10257 | if (!VisitValueCB(IRPosition::value(V: *VAC.getValue()))) |
10258 | return indicatePessimisticFixpoint(); |
10259 | |
10260 | return ChangeStatus::UNCHANGED; |
10261 | } |
10262 | |
10263 | /// See AbstractAttribute::trackStatistics() |
10264 | void trackStatistics() const override { STATS_DECLTRACK_FNRET_ATTR(noundef) } |
10265 | }; |
10266 | |
10267 | struct AANoUndefReturned final |
10268 | : AAReturnedFromReturnedValues<AANoUndef, AANoUndefImpl> { |
10269 | AANoUndefReturned(const IRPosition &IRP, Attributor &A) |
10270 | : AAReturnedFromReturnedValues<AANoUndef, AANoUndefImpl>(IRP, A) {} |
10271 | |
10272 | /// See AbstractAttribute::trackStatistics() |
10273 | void trackStatistics() const override { STATS_DECLTRACK_FNRET_ATTR(noundef) } |
10274 | }; |
10275 | |
10276 | struct AANoUndefArgument final |
10277 | : AAArgumentFromCallSiteArguments<AANoUndef, AANoUndefImpl> { |
10278 | AANoUndefArgument(const IRPosition &IRP, Attributor &A) |
10279 | : AAArgumentFromCallSiteArguments<AANoUndef, AANoUndefImpl>(IRP, A) {} |
10280 | |
10281 | /// See AbstractAttribute::trackStatistics() |
10282 | void trackStatistics() const override { STATS_DECLTRACK_ARG_ATTR(noundef) } |
10283 | }; |
10284 | |
10285 | struct AANoUndefCallSiteArgument final : AANoUndefFloating { |
10286 | AANoUndefCallSiteArgument(const IRPosition &IRP, Attributor &A) |
10287 | : AANoUndefFloating(IRP, A) {} |
10288 | |
10289 | /// See AbstractAttribute::trackStatistics() |
10290 | void trackStatistics() const override { STATS_DECLTRACK_CSARG_ATTR(noundef) } |
10291 | }; |
10292 | |
10293 | struct AANoUndefCallSiteReturned final |
10294 | : AACalleeToCallSite<AANoUndef, AANoUndefImpl> { |
10295 | AANoUndefCallSiteReturned(const IRPosition &IRP, Attributor &A) |
10296 | : AACalleeToCallSite<AANoUndef, AANoUndefImpl>(IRP, A) {} |
10297 | |
10298 | /// See AbstractAttribute::trackStatistics() |
10299 | void trackStatistics() const override { STATS_DECLTRACK_CSRET_ATTR(noundef) } |
10300 | }; |
10301 | |
10302 | /// ------------------------ NoFPClass Attribute ------------------------------- |
10303 | |
10304 | struct AANoFPClassImpl : AANoFPClass { |
10305 | AANoFPClassImpl(const IRPosition &IRP, Attributor &A) : AANoFPClass(IRP, A) {} |
10306 | |
10307 | void initialize(Attributor &A) override { |
10308 | const IRPosition &IRP = getIRPosition(); |
10309 | |
10310 | Value &V = IRP.getAssociatedValue(); |
10311 | if (isa<UndefValue>(Val: V)) { |
10312 | indicateOptimisticFixpoint(); |
10313 | return; |
10314 | } |
10315 | |
10316 | SmallVector<Attribute> Attrs; |
10317 | A.getAttrs(getIRPosition(), {Attribute::NoFPClass}, Attrs, false); |
10318 | for (const auto &Attr : Attrs) { |
10319 | addKnownBits(Attr.getNoFPClass()); |
10320 | } |
10321 | |
10322 | const DataLayout &DL = A.getDataLayout(); |
10323 | if (getPositionKind() != IRPosition::IRP_RETURNED) { |
10324 | KnownFPClass KnownFPClass = computeKnownFPClass(V: &V, DL); |
10325 | addKnownBits(~KnownFPClass.KnownFPClasses); |
10326 | } |
10327 | |
10328 | if (Instruction *CtxI = getCtxI()) |
10329 | followUsesInMBEC(*this, A, getState(), *CtxI); |
10330 | } |
10331 | |
10332 | /// See followUsesInMBEC |
10333 | bool followUseInMBEC(Attributor &A, const Use *U, const Instruction *I, |
10334 | AANoFPClass::StateType &State) { |
10335 | const Value *UseV = U->get(); |
10336 | const DominatorTree *DT = nullptr; |
10337 | AssumptionCache *AC = nullptr; |
10338 | const TargetLibraryInfo *TLI = nullptr; |
10339 | InformationCache &InfoCache = A.getInfoCache(); |
10340 | |
10341 | if (Function *F = getAnchorScope()) { |
10342 | DT = InfoCache.getAnalysisResultForFunction<DominatorTreeAnalysis>(F: *F); |
10343 | AC = InfoCache.getAnalysisResultForFunction<AssumptionAnalysis>(F: *F); |
10344 | TLI = InfoCache.getTargetLibraryInfoForFunction(F: *F); |
10345 | } |
10346 | |
10347 | const DataLayout &DL = A.getDataLayout(); |
10348 | |
10349 | KnownFPClass KnownFPClass = |
10350 | computeKnownFPClass(V: UseV, DL, |
10351 | /*InterestedClasses=*/fcAllFlags, |
10352 | /*Depth=*/0, TLI, AC, CxtI: I, DT); |
10353 | State.addKnownBits(~KnownFPClass.KnownFPClasses); |
10354 | |
10355 | if (auto *CI = dyn_cast<CallInst>(Val: UseV)) { |
10356 | // Special case FP intrinsic with struct return type. |
10357 | switch (CI->getIntrinsicID()) { |
10358 | case Intrinsic::frexp: |
10359 | return true; |
10360 | case Intrinsic::not_intrinsic: |
10361 | // TODO: Could recognize math libcalls |
10362 | return false; |
10363 | default: |
10364 | break; |
10365 | } |
10366 | } |
10367 | |
10368 | if (!UseV->getType()->isFPOrFPVectorTy()) |
10369 | return false; |
10370 | return !isa<LoadInst, AtomicRMWInst>(Val: UseV); |
10371 | } |
10372 | |
10373 | const std::string getAsStr(Attributor *A) const override { |
10374 | std::string Result = "nofpclass" ; |
10375 | raw_string_ostream OS(Result); |
10376 | OS << getAssumedNoFPClass(); |
10377 | return Result; |
10378 | } |
10379 | |
10380 | void getDeducedAttributes(Attributor &A, LLVMContext &Ctx, |
10381 | SmallVectorImpl<Attribute> &Attrs) const override { |
10382 | Attrs.emplace_back(Attribute::getWithNoFPClass(Context&: Ctx, Mask: getAssumedNoFPClass())); |
10383 | } |
10384 | }; |
10385 | |
10386 | struct AANoFPClassFloating : public AANoFPClassImpl { |
10387 | AANoFPClassFloating(const IRPosition &IRP, Attributor &A) |
10388 | : AANoFPClassImpl(IRP, A) {} |
10389 | |
10390 | /// See AbstractAttribute::updateImpl(...). |
10391 | ChangeStatus updateImpl(Attributor &A) override { |
10392 | SmallVector<AA::ValueAndContext> Values; |
10393 | bool UsedAssumedInformation = false; |
10394 | if (!A.getAssumedSimplifiedValues(IRP: getIRPosition(), AA: *this, Values, |
10395 | S: AA::AnyScope, UsedAssumedInformation)) { |
10396 | Values.push_back(Elt: {getAssociatedValue(), getCtxI()}); |
10397 | } |
10398 | |
10399 | StateType T; |
10400 | auto VisitValueCB = [&](Value &V, const Instruction *CtxI) -> bool { |
10401 | const auto *AA = A.getAAFor<AANoFPClass>(*this, IRPosition::value(V), |
10402 | DepClassTy::REQUIRED); |
10403 | if (!AA || this == AA) { |
10404 | T.indicatePessimisticFixpoint(); |
10405 | } else { |
10406 | const AANoFPClass::StateType &S = |
10407 | static_cast<const AANoFPClass::StateType &>(AA->getState()); |
10408 | T ^= S; |
10409 | } |
10410 | return T.isValidState(); |
10411 | }; |
10412 | |
10413 | for (const auto &VAC : Values) |
10414 | if (!VisitValueCB(*VAC.getValue(), VAC.getCtxI())) |
10415 | return indicatePessimisticFixpoint(); |
10416 | |
10417 | return clampStateAndIndicateChange(getState(), T); |
10418 | } |
10419 | |
10420 | /// See AbstractAttribute::trackStatistics() |
10421 | void trackStatistics() const override { |
10422 | STATS_DECLTRACK_FNRET_ATTR(nofpclass) |
10423 | } |
10424 | }; |
10425 | |
10426 | struct AANoFPClassReturned final |
10427 | : AAReturnedFromReturnedValues<AANoFPClass, AANoFPClassImpl, |
10428 | AANoFPClassImpl::StateType, false, Attribute::None, false> { |
10429 | AANoFPClassReturned(const IRPosition &IRP, Attributor &A) |
10430 | : AAReturnedFromReturnedValues<AANoFPClass, AANoFPClassImpl, |
10431 | AANoFPClassImpl::StateType, false, Attribute::None, false>( |
10432 | IRP, A) {} |
10433 | |
10434 | /// See AbstractAttribute::trackStatistics() |
10435 | void trackStatistics() const override { |
10436 | STATS_DECLTRACK_FNRET_ATTR(nofpclass) |
10437 | } |
10438 | }; |
10439 | |
10440 | struct AANoFPClassArgument final |
10441 | : AAArgumentFromCallSiteArguments<AANoFPClass, AANoFPClassImpl> { |
10442 | AANoFPClassArgument(const IRPosition &IRP, Attributor &A) |
10443 | : AAArgumentFromCallSiteArguments<AANoFPClass, AANoFPClassImpl>(IRP, A) {} |
10444 | |
10445 | /// See AbstractAttribute::trackStatistics() |
10446 | void trackStatistics() const override { STATS_DECLTRACK_ARG_ATTR(nofpclass) } |
10447 | }; |
10448 | |
10449 | struct AANoFPClassCallSiteArgument final : AANoFPClassFloating { |
10450 | AANoFPClassCallSiteArgument(const IRPosition &IRP, Attributor &A) |
10451 | : AANoFPClassFloating(IRP, A) {} |
10452 | |
10453 | /// See AbstractAttribute::trackStatistics() |
10454 | void trackStatistics() const override { |
10455 | STATS_DECLTRACK_CSARG_ATTR(nofpclass) |
10456 | } |
10457 | }; |
10458 | |
10459 | struct AANoFPClassCallSiteReturned final |
10460 | : AACalleeToCallSite<AANoFPClass, AANoFPClassImpl> { |
10461 | AANoFPClassCallSiteReturned(const IRPosition &IRP, Attributor &A) |
10462 | : AACalleeToCallSite<AANoFPClass, AANoFPClassImpl>(IRP, A) {} |
10463 | |
10464 | /// See AbstractAttribute::trackStatistics() |
10465 | void trackStatistics() const override { |
10466 | STATS_DECLTRACK_CSRET_ATTR(nofpclass) |
10467 | } |
10468 | }; |
10469 | |
10470 | struct AACallEdgesImpl : public AACallEdges { |
10471 | AACallEdgesImpl(const IRPosition &IRP, Attributor &A) : AACallEdges(IRP, A) {} |
10472 | |
10473 | const SetVector<Function *> &getOptimisticEdges() const override { |
10474 | return CalledFunctions; |
10475 | } |
10476 | |
10477 | bool hasUnknownCallee() const override { return HasUnknownCallee; } |
10478 | |
10479 | bool hasNonAsmUnknownCallee() const override { |
10480 | return HasUnknownCalleeNonAsm; |
10481 | } |
10482 | |
10483 | const std::string getAsStr(Attributor *A) const override { |
10484 | return "CallEdges[" + std::to_string(val: HasUnknownCallee) + "," + |
10485 | std::to_string(val: CalledFunctions.size()) + "]" ; |
10486 | } |
10487 | |
10488 | void trackStatistics() const override {} |
10489 | |
10490 | protected: |
10491 | void addCalledFunction(Function *Fn, ChangeStatus &Change) { |
10492 | if (CalledFunctions.insert(X: Fn)) { |
10493 | Change = ChangeStatus::CHANGED; |
10494 | LLVM_DEBUG(dbgs() << "[AACallEdges] New call edge: " << Fn->getName() |
10495 | << "\n" ); |
10496 | } |
10497 | } |
10498 | |
10499 | void setHasUnknownCallee(bool NonAsm, ChangeStatus &Change) { |
10500 | if (!HasUnknownCallee) |
10501 | Change = ChangeStatus::CHANGED; |
10502 | if (NonAsm && !HasUnknownCalleeNonAsm) |
10503 | Change = ChangeStatus::CHANGED; |
10504 | HasUnknownCalleeNonAsm |= NonAsm; |
10505 | HasUnknownCallee = true; |
10506 | } |
10507 | |
10508 | private: |
10509 | /// Optimistic set of functions that might be called by this position. |
10510 | SetVector<Function *> CalledFunctions; |
10511 | |
10512 | /// Is there any call with a unknown callee. |
10513 | bool HasUnknownCallee = false; |
10514 | |
10515 | /// Is there any call with a unknown callee, excluding any inline asm. |
10516 | bool HasUnknownCalleeNonAsm = false; |
10517 | }; |
10518 | |
10519 | struct AACallEdgesCallSite : public AACallEdgesImpl { |
10520 | AACallEdgesCallSite(const IRPosition &IRP, Attributor &A) |
10521 | : AACallEdgesImpl(IRP, A) {} |
10522 | /// See AbstractAttribute::updateImpl(...). |
10523 | ChangeStatus updateImpl(Attributor &A) override { |
10524 | ChangeStatus Change = ChangeStatus::UNCHANGED; |
10525 | |
10526 | auto VisitValue = [&](Value &V, const Instruction *CtxI) -> bool { |
10527 | if (Function *Fn = dyn_cast<Function>(Val: &V)) { |
10528 | addCalledFunction(Fn, Change); |
10529 | } else { |
10530 | LLVM_DEBUG(dbgs() << "[AACallEdges] Unrecognized value: " << V << "\n" ); |
10531 | setHasUnknownCallee(NonAsm: true, Change); |
10532 | } |
10533 | |
10534 | // Explore all values. |
10535 | return true; |
10536 | }; |
10537 | |
10538 | SmallVector<AA::ValueAndContext> Values; |
10539 | // Process any value that we might call. |
10540 | auto ProcessCalledOperand = [&](Value *V, Instruction *CtxI) { |
10541 | if (isa<Constant>(Val: V)) { |
10542 | VisitValue(*V, CtxI); |
10543 | return; |
10544 | } |
10545 | |
10546 | bool UsedAssumedInformation = false; |
10547 | Values.clear(); |
10548 | if (!A.getAssumedSimplifiedValues(IRP: IRPosition::value(V: *V), AA: *this, Values, |
10549 | S: AA::AnyScope, UsedAssumedInformation)) { |
10550 | Values.push_back(Elt: {*V, CtxI}); |
10551 | } |
10552 | for (auto &VAC : Values) |
10553 | VisitValue(*VAC.getValue(), VAC.getCtxI()); |
10554 | }; |
10555 | |
10556 | CallBase *CB = cast<CallBase>(Val: getCtxI()); |
10557 | |
10558 | if (auto *IA = dyn_cast<InlineAsm>(Val: CB->getCalledOperand())) { |
10559 | if (IA->hasSideEffects() && |
10560 | !hasAssumption(F: *CB->getCaller(), AssumptionStr: "ompx_no_call_asm" ) && |
10561 | !hasAssumption(CB: *CB, AssumptionStr: "ompx_no_call_asm" )) { |
10562 | setHasUnknownCallee(NonAsm: false, Change); |
10563 | } |
10564 | return Change; |
10565 | } |
10566 | |
10567 | if (CB->isIndirectCall()) |
10568 | if (auto *IndirectCallAA = A.getAAFor<AAIndirectCallInfo>( |
10569 | QueryingAA: *this, IRP: getIRPosition(), DepClass: DepClassTy::OPTIONAL)) |
10570 | if (IndirectCallAA->foreachCallee( |
10571 | CB: [&](Function *Fn) { return VisitValue(*Fn, CB); })) |
10572 | return Change; |
10573 | |
10574 | // The most simple case. |
10575 | ProcessCalledOperand(CB->getCalledOperand(), CB); |
10576 | |
10577 | // Process callback functions. |
10578 | SmallVector<const Use *, 4u> CallbackUses; |
10579 | AbstractCallSite::getCallbackUses(CB: *CB, CallbackUses); |
10580 | for (const Use *U : CallbackUses) |
10581 | ProcessCalledOperand(U->get(), CB); |
10582 | |
10583 | return Change; |
10584 | } |
10585 | }; |
10586 | |
10587 | struct AACallEdgesFunction : public AACallEdgesImpl { |
10588 | AACallEdgesFunction(const IRPosition &IRP, Attributor &A) |
10589 | : AACallEdgesImpl(IRP, A) {} |
10590 | |
10591 | /// See AbstractAttribute::updateImpl(...). |
10592 | ChangeStatus updateImpl(Attributor &A) override { |
10593 | ChangeStatus Change = ChangeStatus::UNCHANGED; |
10594 | |
10595 | auto ProcessCallInst = [&](Instruction &Inst) { |
10596 | CallBase &CB = cast<CallBase>(Val&: Inst); |
10597 | |
10598 | auto *CBEdges = A.getAAFor<AACallEdges>( |
10599 | QueryingAA: *this, IRP: IRPosition::callsite_function(CB), DepClass: DepClassTy::REQUIRED); |
10600 | if (!CBEdges) |
10601 | return false; |
10602 | if (CBEdges->hasNonAsmUnknownCallee()) |
10603 | setHasUnknownCallee(NonAsm: true, Change); |
10604 | if (CBEdges->hasUnknownCallee()) |
10605 | setHasUnknownCallee(NonAsm: false, Change); |
10606 | |
10607 | for (Function *F : CBEdges->getOptimisticEdges()) |
10608 | addCalledFunction(Fn: F, Change); |
10609 | |
10610 | return true; |
10611 | }; |
10612 | |
10613 | // Visit all callable instructions. |
10614 | bool UsedAssumedInformation = false; |
10615 | if (!A.checkForAllCallLikeInstructions(Pred: ProcessCallInst, QueryingAA: *this, |
10616 | UsedAssumedInformation, |
10617 | /* CheckBBLivenessOnly */ true)) { |
10618 | // If we haven't looked at all call like instructions, assume that there |
10619 | // are unknown callees. |
10620 | setHasUnknownCallee(NonAsm: true, Change); |
10621 | } |
10622 | |
10623 | return Change; |
10624 | } |
10625 | }; |
10626 | |
10627 | /// -------------------AAInterFnReachability Attribute-------------------------- |
10628 | |
10629 | struct AAInterFnReachabilityFunction |
10630 | : public CachedReachabilityAA<AAInterFnReachability, Function> { |
10631 | using Base = CachedReachabilityAA<AAInterFnReachability, Function>; |
10632 | AAInterFnReachabilityFunction(const IRPosition &IRP, Attributor &A) |
10633 | : Base(IRP, A) {} |
10634 | |
10635 | bool instructionCanReach( |
10636 | Attributor &A, const Instruction &From, const Function &To, |
10637 | const AA::InstExclusionSetTy *ExclusionSet) const override { |
10638 | assert(From.getFunction() == getAnchorScope() && "Queried the wrong AA!" ); |
10639 | auto *NonConstThis = const_cast<AAInterFnReachabilityFunction *>(this); |
10640 | |
10641 | RQITy StackRQI(A, From, To, ExclusionSet, false); |
10642 | typename RQITy::Reachable Result; |
10643 | if (!NonConstThis->checkQueryCache(A, StackRQI, Result)) |
10644 | return NonConstThis->isReachableImpl(A, RQI&: StackRQI, |
10645 | /*IsTemporaryRQI=*/true); |
10646 | return Result == RQITy::Reachable::Yes; |
10647 | } |
10648 | |
10649 | bool isReachableImpl(Attributor &A, RQITy &RQI, |
10650 | bool IsTemporaryRQI) override { |
10651 | const Instruction *EntryI = |
10652 | &RQI.From->getFunction()->getEntryBlock().front(); |
10653 | if (EntryI != RQI.From && |
10654 | !instructionCanReach(A, From: *EntryI, To: *RQI.To, ExclusionSet: nullptr)) |
10655 | return rememberResult(A, Result: RQITy::Reachable::No, RQI, UsedExclusionSet: false, |
10656 | IsTemporaryRQI); |
10657 | |
10658 | auto CheckReachableCallBase = [&](CallBase *CB) { |
10659 | auto *CBEdges = A.getAAFor<AACallEdges>( |
10660 | QueryingAA: *this, IRP: IRPosition::callsite_function(CB: *CB), DepClass: DepClassTy::OPTIONAL); |
10661 | if (!CBEdges || !CBEdges->getState().isValidState()) |
10662 | return false; |
10663 | // TODO Check To backwards in this case. |
10664 | if (CBEdges->hasUnknownCallee()) |
10665 | return false; |
10666 | |
10667 | for (Function *Fn : CBEdges->getOptimisticEdges()) { |
10668 | if (Fn == RQI.To) |
10669 | return false; |
10670 | |
10671 | if (Fn->isDeclaration()) { |
10672 | if (Fn->hasFnAttribute(Attribute::NoCallback)) |
10673 | continue; |
10674 | // TODO Check To backwards in this case. |
10675 | return false; |
10676 | } |
10677 | |
10678 | if (Fn == getAnchorScope()) { |
10679 | if (EntryI == RQI.From) |
10680 | continue; |
10681 | return false; |
10682 | } |
10683 | |
10684 | const AAInterFnReachability *InterFnReachability = |
10685 | A.getAAFor<AAInterFnReachability>(QueryingAA: *this, IRP: IRPosition::function(F: *Fn), |
10686 | DepClass: DepClassTy::OPTIONAL); |
10687 | |
10688 | const Instruction &FnFirstInst = Fn->getEntryBlock().front(); |
10689 | if (!InterFnReachability || |
10690 | InterFnReachability->instructionCanReach(A, Inst: FnFirstInst, Fn: *RQI.To, |
10691 | ExclusionSet: RQI.ExclusionSet)) |
10692 | return false; |
10693 | } |
10694 | return true; |
10695 | }; |
10696 | |
10697 | const auto *IntraFnReachability = A.getAAFor<AAIntraFnReachability>( |
10698 | QueryingAA: *this, IRP: IRPosition::function(F: *RQI.From->getFunction()), |
10699 | DepClass: DepClassTy::OPTIONAL); |
10700 | |
10701 | // Determine call like instructions that we can reach from the inst. |
10702 | auto CheckCallBase = [&](Instruction &CBInst) { |
10703 | // There are usually less nodes in the call graph, check inter function |
10704 | // reachability first. |
10705 | if (CheckReachableCallBase(cast<CallBase>(Val: &CBInst))) |
10706 | return true; |
10707 | return IntraFnReachability && !IntraFnReachability->isAssumedReachable( |
10708 | A, From: *RQI.From, To: CBInst, ExclusionSet: RQI.ExclusionSet); |
10709 | }; |
10710 | |
10711 | bool UsedExclusionSet = /* conservative */ true; |
10712 | bool UsedAssumedInformation = false; |
10713 | if (!A.checkForAllCallLikeInstructions(Pred: CheckCallBase, QueryingAA: *this, |
10714 | UsedAssumedInformation, |
10715 | /* CheckBBLivenessOnly */ true)) |
10716 | return rememberResult(A, Result: RQITy::Reachable::Yes, RQI, UsedExclusionSet, |
10717 | IsTemporaryRQI); |
10718 | |
10719 | return rememberResult(A, Result: RQITy::Reachable::No, RQI, UsedExclusionSet, |
10720 | IsTemporaryRQI); |
10721 | } |
10722 | |
10723 | void trackStatistics() const override {} |
10724 | }; |
10725 | } // namespace |
10726 | |
10727 | template <typename AAType> |
10728 | static std::optional<Constant *> |
10729 | askForAssumedConstant(Attributor &A, const AbstractAttribute &QueryingAA, |
10730 | const IRPosition &IRP, Type &Ty) { |
10731 | if (!Ty.isIntegerTy()) |
10732 | return nullptr; |
10733 | |
10734 | // This will also pass the call base context. |
10735 | const auto *AA = A.getAAFor<AAType>(QueryingAA, IRP, DepClassTy::NONE); |
10736 | if (!AA) |
10737 | return nullptr; |
10738 | |
10739 | std::optional<Constant *> COpt = AA->getAssumedConstant(A); |
10740 | |
10741 | if (!COpt.has_value()) { |
10742 | A.recordDependence(FromAA: *AA, ToAA: QueryingAA, DepClass: DepClassTy::OPTIONAL); |
10743 | return std::nullopt; |
10744 | } |
10745 | if (auto *C = *COpt) { |
10746 | A.recordDependence(FromAA: *AA, ToAA: QueryingAA, DepClass: DepClassTy::OPTIONAL); |
10747 | return C; |
10748 | } |
10749 | return nullptr; |
10750 | } |
10751 | |
10752 | Value *AAPotentialValues::getSingleValue( |
10753 | Attributor &A, const AbstractAttribute &AA, const IRPosition &IRP, |
10754 | SmallVectorImpl<AA::ValueAndContext> &Values) { |
10755 | Type &Ty = *IRP.getAssociatedType(); |
10756 | std::optional<Value *> V; |
10757 | for (auto &It : Values) { |
10758 | V = AA::combineOptionalValuesInAAValueLatice(A: V, B: It.getValue(), Ty: &Ty); |
10759 | if (V.has_value() && !*V) |
10760 | break; |
10761 | } |
10762 | if (!V.has_value()) |
10763 | return UndefValue::get(T: &Ty); |
10764 | return *V; |
10765 | } |
10766 | |
10767 | namespace { |
10768 | struct AAPotentialValuesImpl : AAPotentialValues { |
10769 | using StateType = PotentialLLVMValuesState; |
10770 | |
10771 | AAPotentialValuesImpl(const IRPosition &IRP, Attributor &A) |
10772 | : AAPotentialValues(IRP, A) {} |
10773 | |
10774 | /// See AbstractAttribute::initialize(..). |
10775 | void initialize(Attributor &A) override { |
10776 | if (A.hasSimplificationCallback(IRP: getIRPosition())) { |
10777 | indicatePessimisticFixpoint(); |
10778 | return; |
10779 | } |
10780 | Value *Stripped = getAssociatedValue().stripPointerCasts(); |
10781 | auto *CE = dyn_cast<ConstantExpr>(Val: Stripped); |
10782 | if (isa<Constant>(Val: Stripped) && |
10783 | (!CE || CE->getOpcode() != Instruction::ICmp)) { |
10784 | addValue(A, State&: getState(), V&: *Stripped, CtxI: getCtxI(), S: AA::AnyScope, |
10785 | AnchorScope: getAnchorScope()); |
10786 | indicateOptimisticFixpoint(); |
10787 | return; |
10788 | } |
10789 | AAPotentialValues::initialize(A); |
10790 | } |
10791 | |
10792 | /// See AbstractAttribute::getAsStr(). |
10793 | const std::string getAsStr(Attributor *A) const override { |
10794 | std::string Str; |
10795 | llvm::raw_string_ostream OS(Str); |
10796 | OS << getState(); |
10797 | return OS.str(); |
10798 | } |
10799 | |
10800 | template <typename AAType> |
10801 | static std::optional<Value *> askOtherAA(Attributor &A, |
10802 | const AbstractAttribute &AA, |
10803 | const IRPosition &IRP, Type &Ty) { |
10804 | if (isa<Constant>(Val: IRP.getAssociatedValue())) |
10805 | return &IRP.getAssociatedValue(); |
10806 | std::optional<Constant *> C = askForAssumedConstant<AAType>(A, AA, IRP, Ty); |
10807 | if (!C) |
10808 | return std::nullopt; |
10809 | if (*C) |
10810 | if (auto *CC = AA::getWithType(V&: **C, Ty)) |
10811 | return CC; |
10812 | return nullptr; |
10813 | } |
10814 | |
10815 | virtual void addValue(Attributor &A, StateType &State, Value &V, |
10816 | const Instruction *CtxI, AA::ValueScope S, |
10817 | Function *AnchorScope) const { |
10818 | |
10819 | IRPosition ValIRP = IRPosition::value(V); |
10820 | if (auto *CB = dyn_cast_or_null<CallBase>(Val: CtxI)) { |
10821 | for (const auto &U : CB->args()) { |
10822 | if (U.get() != &V) |
10823 | continue; |
10824 | ValIRP = IRPosition::callsite_argument(CB: *CB, ArgNo: CB->getArgOperandNo(U: &U)); |
10825 | break; |
10826 | } |
10827 | } |
10828 | |
10829 | Value *VPtr = &V; |
10830 | if (ValIRP.getAssociatedType()->isIntegerTy()) { |
10831 | Type &Ty = *getAssociatedType(); |
10832 | std::optional<Value *> SimpleV = |
10833 | askOtherAA<AAValueConstantRange>(A, AA: *this, IRP: ValIRP, Ty); |
10834 | if (SimpleV.has_value() && !*SimpleV) { |
10835 | auto *PotentialConstantsAA = A.getAAFor<AAPotentialConstantValues>( |
10836 | QueryingAA: *this, IRP: ValIRP, DepClass: DepClassTy::OPTIONAL); |
10837 | if (PotentialConstantsAA && PotentialConstantsAA->isValidState()) { |
10838 | for (const auto &It : PotentialConstantsAA->getAssumedSet()) |
10839 | State.unionAssumed(C: {{*ConstantInt::get(Ty: &Ty, V: It), nullptr}, S}); |
10840 | if (PotentialConstantsAA->undefIsContained()) |
10841 | State.unionAssumed(C: {{*UndefValue::get(T: &Ty), nullptr}, S}); |
10842 | return; |
10843 | } |
10844 | } |
10845 | if (!SimpleV.has_value()) |
10846 | return; |
10847 | |
10848 | if (*SimpleV) |
10849 | VPtr = *SimpleV; |
10850 | } |
10851 | |
10852 | if (isa<ConstantInt>(Val: VPtr)) |
10853 | CtxI = nullptr; |
10854 | if (!AA::isValidInScope(V: *VPtr, Scope: AnchorScope)) |
10855 | S = AA::ValueScope(S | AA::Interprocedural); |
10856 | |
10857 | State.unionAssumed(C: {{*VPtr, CtxI}, S}); |
10858 | } |
10859 | |
10860 | /// Helper struct to tie a value+context pair together with the scope for |
10861 | /// which this is the simplified version. |
10862 | struct ItemInfo { |
10863 | AA::ValueAndContext I; |
10864 | AA::ValueScope S; |
10865 | |
10866 | bool operator==(const ItemInfo &II) const { |
10867 | return II.I == I && II.S == S; |
10868 | }; |
10869 | bool operator<(const ItemInfo &II) const { |
10870 | if (I == II.I) |
10871 | return S < II.S; |
10872 | return I < II.I; |
10873 | }; |
10874 | }; |
10875 | |
10876 | bool recurseForValue(Attributor &A, const IRPosition &IRP, AA::ValueScope S) { |
10877 | SmallMapVector<AA::ValueAndContext, int, 8> ValueScopeMap; |
10878 | for (auto CS : {AA::Intraprocedural, AA::Interprocedural}) { |
10879 | if (!(CS & S)) |
10880 | continue; |
10881 | |
10882 | bool UsedAssumedInformation = false; |
10883 | SmallVector<AA::ValueAndContext> Values; |
10884 | if (!A.getAssumedSimplifiedValues(IRP, AA: this, Values, S: CS, |
10885 | UsedAssumedInformation)) |
10886 | return false; |
10887 | |
10888 | for (auto &It : Values) |
10889 | ValueScopeMap[It] += CS; |
10890 | } |
10891 | for (auto &It : ValueScopeMap) |
10892 | addValue(A, State&: getState(), V&: *It.first.getValue(), CtxI: It.first.getCtxI(), |
10893 | S: AA::ValueScope(It.second), AnchorScope: getAnchorScope()); |
10894 | |
10895 | return true; |
10896 | } |
10897 | |
10898 | void giveUpOnIntraprocedural(Attributor &A) { |
10899 | auto NewS = StateType::getBestState(PVS: getState()); |
10900 | for (const auto &It : getAssumedSet()) { |
10901 | if (It.second == AA::Intraprocedural) |
10902 | continue; |
10903 | addValue(A, State&: NewS, V&: *It.first.getValue(), CtxI: It.first.getCtxI(), |
10904 | S: AA::Interprocedural, AnchorScope: getAnchorScope()); |
10905 | } |
10906 | assert(!undefIsContained() && "Undef should be an explicit value!" ); |
10907 | addValue(A, State&: NewS, V&: getAssociatedValue(), CtxI: getCtxI(), S: AA::Intraprocedural, |
10908 | AnchorScope: getAnchorScope()); |
10909 | getState() = NewS; |
10910 | } |
10911 | |
10912 | /// See AbstractState::indicatePessimisticFixpoint(...). |
10913 | ChangeStatus indicatePessimisticFixpoint() override { |
10914 | getState() = StateType::getBestState(PVS: getState()); |
10915 | getState().unionAssumed(C: {{getAssociatedValue(), getCtxI()}, AA::AnyScope}); |
10916 | AAPotentialValues::indicateOptimisticFixpoint(); |
10917 | return ChangeStatus::CHANGED; |
10918 | } |
10919 | |
10920 | /// See AbstractAttribute::updateImpl(...). |
10921 | ChangeStatus updateImpl(Attributor &A) override { |
10922 | return indicatePessimisticFixpoint(); |
10923 | } |
10924 | |
10925 | /// See AbstractAttribute::manifest(...). |
10926 | ChangeStatus manifest(Attributor &A) override { |
10927 | SmallVector<AA::ValueAndContext> Values; |
10928 | for (AA::ValueScope S : {AA::Interprocedural, AA::Intraprocedural}) { |
10929 | Values.clear(); |
10930 | if (!getAssumedSimplifiedValues(A, Values, S)) |
10931 | continue; |
10932 | Value &OldV = getAssociatedValue(); |
10933 | if (isa<UndefValue>(Val: OldV)) |
10934 | continue; |
10935 | Value *NewV = getSingleValue(A, AA: *this, IRP: getIRPosition(), Values); |
10936 | if (!NewV || NewV == &OldV) |
10937 | continue; |
10938 | if (getCtxI() && |
10939 | !AA::isValidAtPosition(VAC: {*NewV, *getCtxI()}, InfoCache&: A.getInfoCache())) |
10940 | continue; |
10941 | if (A.changeAfterManifest(IRP: getIRPosition(), NV&: *NewV)) |
10942 | return ChangeStatus::CHANGED; |
10943 | } |
10944 | return ChangeStatus::UNCHANGED; |
10945 | } |
10946 | |
10947 | bool getAssumedSimplifiedValues( |
10948 | Attributor &A, SmallVectorImpl<AA::ValueAndContext> &Values, |
10949 | AA::ValueScope S, bool RecurseForSelectAndPHI = false) const override { |
10950 | if (!isValidState()) |
10951 | return false; |
10952 | bool UsedAssumedInformation = false; |
10953 | for (const auto &It : getAssumedSet()) |
10954 | if (It.second & S) { |
10955 | if (RecurseForSelectAndPHI && (isa<PHINode>(Val: It.first.getValue()) || |
10956 | isa<SelectInst>(Val: It.first.getValue()))) { |
10957 | if (A.getAssumedSimplifiedValues( |
10958 | IRP: IRPosition::inst(I: *cast<Instruction>(Val: It.first.getValue())), |
10959 | AA: this, Values, S, UsedAssumedInformation)) |
10960 | continue; |
10961 | } |
10962 | Values.push_back(Elt: It.first); |
10963 | } |
10964 | assert(!undefIsContained() && "Undef should be an explicit value!" ); |
10965 | return true; |
10966 | } |
10967 | }; |
10968 | |
10969 | struct AAPotentialValuesFloating : AAPotentialValuesImpl { |
10970 | AAPotentialValuesFloating(const IRPosition &IRP, Attributor &A) |
10971 | : AAPotentialValuesImpl(IRP, A) {} |
10972 | |
10973 | /// See AbstractAttribute::updateImpl(...). |
10974 | ChangeStatus updateImpl(Attributor &A) override { |
10975 | auto AssumedBefore = getAssumed(); |
10976 | |
10977 | genericValueTraversal(A, InitialV: &getAssociatedValue()); |
10978 | |
10979 | return (AssumedBefore == getAssumed()) ? ChangeStatus::UNCHANGED |
10980 | : ChangeStatus::CHANGED; |
10981 | } |
10982 | |
10983 | /// Helper struct to remember which AAIsDead instances we actually used. |
10984 | struct LivenessInfo { |
10985 | const AAIsDead *LivenessAA = nullptr; |
10986 | bool AnyDead = false; |
10987 | }; |
10988 | |
10989 | /// Check if \p Cmp is a comparison we can simplify. |
10990 | /// |
10991 | /// We handle multiple cases, one in which at least one operand is an |
10992 | /// (assumed) nullptr. If so, try to simplify it using AANonNull on the other |
10993 | /// operand. Return true if successful, in that case Worklist will be updated. |
10994 | bool handleCmp(Attributor &A, Value &Cmp, Value *LHS, Value *RHS, |
10995 | CmpInst::Predicate Pred, ItemInfo II, |
10996 | SmallVectorImpl<ItemInfo> &Worklist) { |
10997 | |
10998 | // Simplify the operands first. |
10999 | bool UsedAssumedInformation = false; |
11000 | SmallVector<AA::ValueAndContext> LHSValues, RHSValues; |
11001 | auto GetSimplifiedValues = [&](Value &V, |
11002 | SmallVector<AA::ValueAndContext> &Values) { |
11003 | if (!A.getAssumedSimplifiedValues( |
11004 | IRP: IRPosition::value(V, CBContext: getCallBaseContext()), AA: this, Values, |
11005 | S: AA::Intraprocedural, UsedAssumedInformation)) { |
11006 | Values.clear(); |
11007 | Values.push_back(Elt: AA::ValueAndContext{V, II.I.getCtxI()}); |
11008 | } |
11009 | return Values.empty(); |
11010 | }; |
11011 | if (GetSimplifiedValues(*LHS, LHSValues)) |
11012 | return true; |
11013 | if (GetSimplifiedValues(*RHS, RHSValues)) |
11014 | return true; |
11015 | |
11016 | LLVMContext &Ctx = LHS->getContext(); |
11017 | |
11018 | InformationCache &InfoCache = A.getInfoCache(); |
11019 | Instruction *CmpI = dyn_cast<Instruction>(Val: &Cmp); |
11020 | Function *F = CmpI ? CmpI->getFunction() : nullptr; |
11021 | const auto *DT = |
11022 | F ? InfoCache.getAnalysisResultForFunction<DominatorTreeAnalysis>(F: *F) |
11023 | : nullptr; |
11024 | const auto *TLI = |
11025 | F ? A.getInfoCache().getTargetLibraryInfoForFunction(F: *F) : nullptr; |
11026 | auto *AC = |
11027 | F ? InfoCache.getAnalysisResultForFunction<AssumptionAnalysis>(F: *F) |
11028 | : nullptr; |
11029 | |
11030 | const DataLayout &DL = A.getDataLayout(); |
11031 | SimplifyQuery Q(DL, TLI, DT, AC, CmpI); |
11032 | |
11033 | auto CheckPair = [&](Value &LHSV, Value &RHSV) { |
11034 | if (isa<UndefValue>(Val: LHSV) || isa<UndefValue>(Val: RHSV)) { |
11035 | addValue(A, State&: getState(), V&: *UndefValue::get(T: Cmp.getType()), |
11036 | /* CtxI */ nullptr, S: II.S, AnchorScope: getAnchorScope()); |
11037 | return true; |
11038 | } |
11039 | |
11040 | // Handle the trivial case first in which we don't even need to think |
11041 | // about null or non-null. |
11042 | if (&LHSV == &RHSV && |
11043 | (CmpInst::isTrueWhenEqual(predicate: Pred) || CmpInst::isFalseWhenEqual(predicate: Pred))) { |
11044 | Constant *NewV = ConstantInt::get(Ty: Type::getInt1Ty(C&: Ctx), |
11045 | V: CmpInst::isTrueWhenEqual(predicate: Pred)); |
11046 | addValue(A, State&: getState(), V&: *NewV, /* CtxI */ nullptr, S: II.S, |
11047 | AnchorScope: getAnchorScope()); |
11048 | return true; |
11049 | } |
11050 | |
11051 | auto *TypedLHS = AA::getWithType(V&: LHSV, Ty&: *LHS->getType()); |
11052 | auto *TypedRHS = AA::getWithType(V&: RHSV, Ty&: *RHS->getType()); |
11053 | if (TypedLHS && TypedRHS) { |
11054 | Value *NewV = simplifyCmpInst(Predicate: Pred, LHS: TypedLHS, RHS: TypedRHS, Q); |
11055 | if (NewV && NewV != &Cmp) { |
11056 | addValue(A, State&: getState(), V&: *NewV, /* CtxI */ nullptr, S: II.S, |
11057 | AnchorScope: getAnchorScope()); |
11058 | return true; |
11059 | } |
11060 | } |
11061 | |
11062 | // From now on we only handle equalities (==, !=). |
11063 | if (!CmpInst::isEquality(pred: Pred)) |
11064 | return false; |
11065 | |
11066 | bool LHSIsNull = isa<ConstantPointerNull>(Val: LHSV); |
11067 | bool RHSIsNull = isa<ConstantPointerNull>(Val: RHSV); |
11068 | if (!LHSIsNull && !RHSIsNull) |
11069 | return false; |
11070 | |
11071 | // Left is the nullptr ==/!= non-nullptr case. We'll use AANonNull on the |
11072 | // non-nullptr operand and if we assume it's non-null we can conclude the |
11073 | // result of the comparison. |
11074 | assert((LHSIsNull || RHSIsNull) && |
11075 | "Expected nullptr versus non-nullptr comparison at this point" ); |
11076 | |
11077 | // The index is the operand that we assume is not null. |
11078 | unsigned PtrIdx = LHSIsNull; |
11079 | bool IsKnownNonNull; |
11080 | bool IsAssumedNonNull = AA::hasAssumedIRAttr<Attribute::NonNull>( |
11081 | A, this, IRPosition::value(*(PtrIdx ? &RHSV : &LHSV)), |
11082 | DepClassTy::REQUIRED, IsKnownNonNull); |
11083 | if (!IsAssumedNonNull) |
11084 | return false; |
11085 | |
11086 | // The new value depends on the predicate, true for != and false for ==. |
11087 | Constant *NewV = |
11088 | ConstantInt::get(Ty: Type::getInt1Ty(C&: Ctx), V: Pred == CmpInst::ICMP_NE); |
11089 | addValue(A, State&: getState(), V&: *NewV, /* CtxI */ nullptr, S: II.S, |
11090 | AnchorScope: getAnchorScope()); |
11091 | return true; |
11092 | }; |
11093 | |
11094 | for (auto &LHSValue : LHSValues) |
11095 | for (auto &RHSValue : RHSValues) |
11096 | if (!CheckPair(*LHSValue.getValue(), *RHSValue.getValue())) |
11097 | return false; |
11098 | return true; |
11099 | } |
11100 | |
11101 | bool handleSelectInst(Attributor &A, SelectInst &SI, ItemInfo II, |
11102 | SmallVectorImpl<ItemInfo> &Worklist) { |
11103 | const Instruction *CtxI = II.I.getCtxI(); |
11104 | bool UsedAssumedInformation = false; |
11105 | |
11106 | std::optional<Constant *> C = |
11107 | A.getAssumedConstant(V: *SI.getCondition(), AA: *this, UsedAssumedInformation); |
11108 | bool NoValueYet = !C.has_value(); |
11109 | if (NoValueYet || isa_and_nonnull<UndefValue>(Val: *C)) |
11110 | return true; |
11111 | if (auto *CI = dyn_cast_or_null<ConstantInt>(Val: *C)) { |
11112 | if (CI->isZero()) |
11113 | Worklist.push_back(Elt: {.I: {*SI.getFalseValue(), CtxI}, .S: II.S}); |
11114 | else |
11115 | Worklist.push_back(Elt: {.I: {*SI.getTrueValue(), CtxI}, .S: II.S}); |
11116 | } else if (&SI == &getAssociatedValue()) { |
11117 | // We could not simplify the condition, assume both values. |
11118 | Worklist.push_back(Elt: {.I: {*SI.getTrueValue(), CtxI}, .S: II.S}); |
11119 | Worklist.push_back(Elt: {.I: {*SI.getFalseValue(), CtxI}, .S: II.S}); |
11120 | } else { |
11121 | std::optional<Value *> SimpleV = A.getAssumedSimplified( |
11122 | IRP: IRPosition::inst(I: SI), AA: *this, UsedAssumedInformation, S: II.S); |
11123 | if (!SimpleV.has_value()) |
11124 | return true; |
11125 | if (*SimpleV) { |
11126 | addValue(A, State&: getState(), V&: **SimpleV, CtxI, S: II.S, AnchorScope: getAnchorScope()); |
11127 | return true; |
11128 | } |
11129 | return false; |
11130 | } |
11131 | return true; |
11132 | } |
11133 | |
11134 | bool handleLoadInst(Attributor &A, LoadInst &LI, ItemInfo II, |
11135 | SmallVectorImpl<ItemInfo> &Worklist) { |
11136 | SmallSetVector<Value *, 4> PotentialCopies; |
11137 | SmallSetVector<Instruction *, 4> PotentialValueOrigins; |
11138 | bool UsedAssumedInformation = false; |
11139 | if (!AA::getPotentiallyLoadedValues(A, LI, PotentialValues&: PotentialCopies, |
11140 | PotentialValueOrigins, QueryingAA: *this, |
11141 | UsedAssumedInformation, |
11142 | /* OnlyExact */ true)) { |
11143 | LLVM_DEBUG(dbgs() << "[AAPotentialValues] Failed to get potentially " |
11144 | "loaded values for load instruction " |
11145 | << LI << "\n" ); |
11146 | return false; |
11147 | } |
11148 | |
11149 | // Do not simplify loads that are only used in llvm.assume if we cannot also |
11150 | // remove all stores that may feed into the load. The reason is that the |
11151 | // assume is probably worth something as long as the stores are around. |
11152 | InformationCache &InfoCache = A.getInfoCache(); |
11153 | if (InfoCache.isOnlyUsedByAssume(I: LI)) { |
11154 | if (!llvm::all_of(Range&: PotentialValueOrigins, P: [&](Instruction *I) { |
11155 | if (!I || isa<AssumeInst>(Val: I)) |
11156 | return true; |
11157 | if (auto *SI = dyn_cast<StoreInst>(Val: I)) |
11158 | return A.isAssumedDead(U: SI->getOperandUse(i: 0), QueryingAA: this, |
11159 | /* LivenessAA */ FnLivenessAA: nullptr, |
11160 | UsedAssumedInformation, |
11161 | /* CheckBBLivenessOnly */ false); |
11162 | return A.isAssumedDead(I: *I, QueryingAA: this, /* LivenessAA */ nullptr, |
11163 | UsedAssumedInformation, |
11164 | /* CheckBBLivenessOnly */ false); |
11165 | })) { |
11166 | LLVM_DEBUG(dbgs() << "[AAPotentialValues] Load is onl used by assumes " |
11167 | "and we cannot delete all the stores: " |
11168 | << LI << "\n" ); |
11169 | return false; |
11170 | } |
11171 | } |
11172 | |
11173 | // Values have to be dynamically unique or we loose the fact that a |
11174 | // single llvm::Value might represent two runtime values (e.g., |
11175 | // stack locations in different recursive calls). |
11176 | const Instruction *CtxI = II.I.getCtxI(); |
11177 | bool ScopeIsLocal = (II.S & AA::Intraprocedural); |
11178 | bool AllLocal = ScopeIsLocal; |
11179 | bool DynamicallyUnique = llvm::all_of(Range&: PotentialCopies, P: [&](Value *PC) { |
11180 | AllLocal &= AA::isValidInScope(V: *PC, Scope: getAnchorScope()); |
11181 | return AA::isDynamicallyUnique(A, QueryingAA: *this, V: *PC); |
11182 | }); |
11183 | if (!DynamicallyUnique) { |
11184 | LLVM_DEBUG(dbgs() << "[AAPotentialValues] Not all potentially loaded " |
11185 | "values are dynamically unique: " |
11186 | << LI << "\n" ); |
11187 | return false; |
11188 | } |
11189 | |
11190 | for (auto *PotentialCopy : PotentialCopies) { |
11191 | if (AllLocal) { |
11192 | Worklist.push_back(Elt: {.I: {*PotentialCopy, CtxI}, .S: II.S}); |
11193 | } else { |
11194 | Worklist.push_back(Elt: {.I: {*PotentialCopy, CtxI}, .S: AA::Interprocedural}); |
11195 | } |
11196 | } |
11197 | if (!AllLocal && ScopeIsLocal) |
11198 | addValue(A, State&: getState(), V&: LI, CtxI, S: AA::Intraprocedural, AnchorScope: getAnchorScope()); |
11199 | return true; |
11200 | } |
11201 | |
11202 | bool handlePHINode( |
11203 | Attributor &A, PHINode &PHI, ItemInfo II, |
11204 | SmallVectorImpl<ItemInfo> &Worklist, |
11205 | SmallMapVector<const Function *, LivenessInfo, 4> &LivenessAAs) { |
11206 | auto GetLivenessInfo = [&](const Function &F) -> LivenessInfo & { |
11207 | LivenessInfo &LI = LivenessAAs[&F]; |
11208 | if (!LI.LivenessAA) |
11209 | LI.LivenessAA = A.getAAFor<AAIsDead>(QueryingAA: *this, IRP: IRPosition::function(F), |
11210 | DepClass: DepClassTy::NONE); |
11211 | return LI; |
11212 | }; |
11213 | |
11214 | if (&PHI == &getAssociatedValue()) { |
11215 | LivenessInfo &LI = GetLivenessInfo(*PHI.getFunction()); |
11216 | const auto *CI = |
11217 | A.getInfoCache().getAnalysisResultForFunction<CycleAnalysis>( |
11218 | F: *PHI.getFunction()); |
11219 | |
11220 | Cycle *C = nullptr; |
11221 | bool CyclePHI = mayBeInCycle(CI, I: &PHI, /* HeaderOnly */ true, CPtr: &C); |
11222 | for (unsigned u = 0, e = PHI.getNumIncomingValues(); u < e; u++) { |
11223 | BasicBlock *IncomingBB = PHI.getIncomingBlock(i: u); |
11224 | if (LI.LivenessAA && |
11225 | LI.LivenessAA->isEdgeDead(From: IncomingBB, To: PHI.getParent())) { |
11226 | LI.AnyDead = true; |
11227 | continue; |
11228 | } |
11229 | Value *V = PHI.getIncomingValue(i: u); |
11230 | if (V == &PHI) |
11231 | continue; |
11232 | |
11233 | // If the incoming value is not the PHI but an instruction in the same |
11234 | // cycle we might have multiple versions of it flying around. |
11235 | if (CyclePHI && isa<Instruction>(Val: V) && |
11236 | (!C || C->contains(Block: cast<Instruction>(Val: V)->getParent()))) |
11237 | return false; |
11238 | |
11239 | Worklist.push_back(Elt: {.I: {*V, IncomingBB->getTerminator()}, .S: II.S}); |
11240 | } |
11241 | return true; |
11242 | } |
11243 | |
11244 | bool UsedAssumedInformation = false; |
11245 | std::optional<Value *> SimpleV = A.getAssumedSimplified( |
11246 | IRP: IRPosition::inst(I: PHI), AA: *this, UsedAssumedInformation, S: II.S); |
11247 | if (!SimpleV.has_value()) |
11248 | return true; |
11249 | if (!(*SimpleV)) |
11250 | return false; |
11251 | addValue(A, State&: getState(), V&: **SimpleV, CtxI: &PHI, S: II.S, AnchorScope: getAnchorScope()); |
11252 | return true; |
11253 | } |
11254 | |
11255 | /// Use the generic, non-optimistic InstSimplfy functionality if we managed to |
11256 | /// simplify any operand of the instruction \p I. Return true if successful, |
11257 | /// in that case Worklist will be updated. |
11258 | bool handleGenericInst(Attributor &A, Instruction &I, ItemInfo II, |
11259 | SmallVectorImpl<ItemInfo> &Worklist) { |
11260 | bool SomeSimplified = false; |
11261 | bool UsedAssumedInformation = false; |
11262 | |
11263 | SmallVector<Value *, 8> NewOps(I.getNumOperands()); |
11264 | int Idx = 0; |
11265 | for (Value *Op : I.operands()) { |
11266 | const auto &SimplifiedOp = A.getAssumedSimplified( |
11267 | IRP: IRPosition::value(V: *Op, CBContext: getCallBaseContext()), AA: *this, |
11268 | UsedAssumedInformation, S: AA::Intraprocedural); |
11269 | // If we are not sure about any operand we are not sure about the entire |
11270 | // instruction, we'll wait. |
11271 | if (!SimplifiedOp.has_value()) |
11272 | return true; |
11273 | |
11274 | if (*SimplifiedOp) |
11275 | NewOps[Idx] = *SimplifiedOp; |
11276 | else |
11277 | NewOps[Idx] = Op; |
11278 | |
11279 | SomeSimplified |= (NewOps[Idx] != Op); |
11280 | ++Idx; |
11281 | } |
11282 | |
11283 | // We won't bother with the InstSimplify interface if we didn't simplify any |
11284 | // operand ourselves. |
11285 | if (!SomeSimplified) |
11286 | return false; |
11287 | |
11288 | InformationCache &InfoCache = A.getInfoCache(); |
11289 | Function *F = I.getFunction(); |
11290 | const auto *DT = |
11291 | InfoCache.getAnalysisResultForFunction<DominatorTreeAnalysis>(F: *F); |
11292 | const auto *TLI = A.getInfoCache().getTargetLibraryInfoForFunction(F: *F); |
11293 | auto *AC = InfoCache.getAnalysisResultForFunction<AssumptionAnalysis>(F: *F); |
11294 | |
11295 | const DataLayout &DL = I.getModule()->getDataLayout(); |
11296 | SimplifyQuery Q(DL, TLI, DT, AC, &I); |
11297 | Value *NewV = simplifyInstructionWithOperands(I: &I, NewOps, Q); |
11298 | if (!NewV || NewV == &I) |
11299 | return false; |
11300 | |
11301 | LLVM_DEBUG(dbgs() << "Generic inst " << I << " assumed simplified to " |
11302 | << *NewV << "\n" ); |
11303 | Worklist.push_back(Elt: {.I: {*NewV, II.I.getCtxI()}, .S: II.S}); |
11304 | return true; |
11305 | } |
11306 | |
11307 | bool simplifyInstruction( |
11308 | Attributor &A, Instruction &I, ItemInfo II, |
11309 | SmallVectorImpl<ItemInfo> &Worklist, |
11310 | SmallMapVector<const Function *, LivenessInfo, 4> &LivenessAAs) { |
11311 | if (auto *CI = dyn_cast<CmpInst>(Val: &I)) |
11312 | return handleCmp(A, Cmp&: *CI, LHS: CI->getOperand(i_nocapture: 0), RHS: CI->getOperand(i_nocapture: 1), |
11313 | Pred: CI->getPredicate(), II, Worklist); |
11314 | |
11315 | switch (I.getOpcode()) { |
11316 | case Instruction::Select: |
11317 | return handleSelectInst(A, SI&: cast<SelectInst>(Val&: I), II, Worklist); |
11318 | case Instruction::PHI: |
11319 | return handlePHINode(A, PHI&: cast<PHINode>(Val&: I), II, Worklist, LivenessAAs); |
11320 | case Instruction::Load: |
11321 | return handleLoadInst(A, LI&: cast<LoadInst>(Val&: I), II, Worklist); |
11322 | default: |
11323 | return handleGenericInst(A, I, II, Worklist); |
11324 | }; |
11325 | return false; |
11326 | } |
11327 | |
11328 | void genericValueTraversal(Attributor &A, Value *InitialV) { |
11329 | SmallMapVector<const Function *, LivenessInfo, 4> LivenessAAs; |
11330 | |
11331 | SmallSet<ItemInfo, 16> Visited; |
11332 | SmallVector<ItemInfo, 16> Worklist; |
11333 | Worklist.push_back(Elt: {.I: {*InitialV, getCtxI()}, .S: AA::AnyScope}); |
11334 | |
11335 | int Iteration = 0; |
11336 | do { |
11337 | ItemInfo II = Worklist.pop_back_val(); |
11338 | Value *V = II.I.getValue(); |
11339 | assert(V); |
11340 | const Instruction *CtxI = II.I.getCtxI(); |
11341 | AA::ValueScope S = II.S; |
11342 | |
11343 | // Check if we should process the current value. To prevent endless |
11344 | // recursion keep a record of the values we followed! |
11345 | if (!Visited.insert(V: II).second) |
11346 | continue; |
11347 | |
11348 | // Make sure we limit the compile time for complex expressions. |
11349 | if (Iteration++ >= MaxPotentialValuesIterations) { |
11350 | LLVM_DEBUG(dbgs() << "Generic value traversal reached iteration limit: " |
11351 | << Iteration << "!\n" ); |
11352 | addValue(A, State&: getState(), V&: *V, CtxI, S, AnchorScope: getAnchorScope()); |
11353 | continue; |
11354 | } |
11355 | |
11356 | // Explicitly look through calls with a "returned" attribute if we do |
11357 | // not have a pointer as stripPointerCasts only works on them. |
11358 | Value *NewV = nullptr; |
11359 | if (V->getType()->isPointerTy()) { |
11360 | NewV = AA::getWithType(V&: *V->stripPointerCasts(), Ty&: *V->getType()); |
11361 | } else { |
11362 | if (auto *CB = dyn_cast<CallBase>(Val: V)) |
11363 | if (auto *Callee = |
11364 | dyn_cast_if_present<Function>(Val: CB->getCalledOperand())) { |
11365 | for (Argument &Arg : Callee->args()) |
11366 | if (Arg.hasReturnedAttr()) { |
11367 | NewV = CB->getArgOperand(i: Arg.getArgNo()); |
11368 | break; |
11369 | } |
11370 | } |
11371 | } |
11372 | if (NewV && NewV != V) { |
11373 | Worklist.push_back(Elt: {.I: {*NewV, CtxI}, .S: S}); |
11374 | continue; |
11375 | } |
11376 | |
11377 | if (auto *CE = dyn_cast<ConstantExpr>(Val: V)) { |
11378 | if (CE->getOpcode() == Instruction::ICmp) |
11379 | if (handleCmp(A, Cmp&: *CE, LHS: CE->getOperand(i_nocapture: 0), RHS: CE->getOperand(i_nocapture: 1), |
11380 | Pred: CmpInst::Predicate(CE->getPredicate()), II, Worklist)) |
11381 | continue; |
11382 | } |
11383 | |
11384 | if (auto *I = dyn_cast<Instruction>(Val: V)) { |
11385 | if (simplifyInstruction(A, I&: *I, II, Worklist, LivenessAAs)) |
11386 | continue; |
11387 | } |
11388 | |
11389 | if (V != InitialV || isa<Argument>(Val: V)) |
11390 | if (recurseForValue(A, IRP: IRPosition::value(V: *V), S: II.S)) |
11391 | continue; |
11392 | |
11393 | // If we haven't stripped anything we give up. |
11394 | if (V == InitialV && CtxI == getCtxI()) { |
11395 | indicatePessimisticFixpoint(); |
11396 | return; |
11397 | } |
11398 | |
11399 | addValue(A, State&: getState(), V&: *V, CtxI, S, AnchorScope: getAnchorScope()); |
11400 | } while (!Worklist.empty()); |
11401 | |
11402 | // If we actually used liveness information so we have to record a |
11403 | // dependence. |
11404 | for (auto &It : LivenessAAs) |
11405 | if (It.second.AnyDead) |
11406 | A.recordDependence(FromAA: *It.second.LivenessAA, ToAA: *this, DepClass: DepClassTy::OPTIONAL); |
11407 | } |
11408 | |
11409 | /// See AbstractAttribute::trackStatistics() |
11410 | void trackStatistics() const override { |
11411 | STATS_DECLTRACK_FLOATING_ATTR(potential_values) |
11412 | } |
11413 | }; |
11414 | |
11415 | struct AAPotentialValuesArgument final : AAPotentialValuesImpl { |
11416 | using Base = AAPotentialValuesImpl; |
11417 | AAPotentialValuesArgument(const IRPosition &IRP, Attributor &A) |
11418 | : Base(IRP, A) {} |
11419 | |
11420 | /// See AbstractAttribute::initialize(..). |
11421 | void initialize(Attributor &A) override { |
11422 | auto &Arg = cast<Argument>(Val&: getAssociatedValue()); |
11423 | if (Arg.hasPointeeInMemoryValueAttr()) |
11424 | indicatePessimisticFixpoint(); |
11425 | } |
11426 | |
11427 | /// See AbstractAttribute::updateImpl(...). |
11428 | ChangeStatus updateImpl(Attributor &A) override { |
11429 | auto AssumedBefore = getAssumed(); |
11430 | |
11431 | unsigned ArgNo = getCalleeArgNo(); |
11432 | |
11433 | bool UsedAssumedInformation = false; |
11434 | SmallVector<AA::ValueAndContext> Values; |
11435 | auto CallSitePred = [&](AbstractCallSite ACS) { |
11436 | const auto CSArgIRP = IRPosition::callsite_argument(ACS, ArgNo); |
11437 | if (CSArgIRP.getPositionKind() == IRP_INVALID) |
11438 | return false; |
11439 | |
11440 | if (!A.getAssumedSimplifiedValues(IRP: CSArgIRP, AA: this, Values, |
11441 | S: AA::Interprocedural, |
11442 | UsedAssumedInformation)) |
11443 | return false; |
11444 | |
11445 | return isValidState(); |
11446 | }; |
11447 | |
11448 | if (!A.checkForAllCallSites(Pred: CallSitePred, QueryingAA: *this, |
11449 | /* RequireAllCallSites */ true, |
11450 | UsedAssumedInformation)) |
11451 | return indicatePessimisticFixpoint(); |
11452 | |
11453 | Function *Fn = getAssociatedFunction(); |
11454 | bool AnyNonLocal = false; |
11455 | for (auto &It : Values) { |
11456 | if (isa<Constant>(Val: It.getValue())) { |
11457 | addValue(A, State&: getState(), V&: *It.getValue(), CtxI: It.getCtxI(), S: AA::AnyScope, |
11458 | AnchorScope: getAnchorScope()); |
11459 | continue; |
11460 | } |
11461 | if (!AA::isDynamicallyUnique(A, QueryingAA: *this, V: *It.getValue())) |
11462 | return indicatePessimisticFixpoint(); |
11463 | |
11464 | if (auto *Arg = dyn_cast<Argument>(Val: It.getValue())) |
11465 | if (Arg->getParent() == Fn) { |
11466 | addValue(A, State&: getState(), V&: *It.getValue(), CtxI: It.getCtxI(), S: AA::AnyScope, |
11467 | AnchorScope: getAnchorScope()); |
11468 | continue; |
11469 | } |
11470 | addValue(A, State&: getState(), V&: *It.getValue(), CtxI: It.getCtxI(), S: AA::Interprocedural, |
11471 | AnchorScope: getAnchorScope()); |
11472 | AnyNonLocal = true; |
11473 | } |
11474 | assert(!undefIsContained() && "Undef should be an explicit value!" ); |
11475 | if (AnyNonLocal) |
11476 | giveUpOnIntraprocedural(A); |
11477 | |
11478 | return (AssumedBefore == getAssumed()) ? ChangeStatus::UNCHANGED |
11479 | : ChangeStatus::CHANGED; |
11480 | } |
11481 | |
11482 | /// See AbstractAttribute::trackStatistics() |
11483 | void trackStatistics() const override { |
11484 | STATS_DECLTRACK_ARG_ATTR(potential_values) |
11485 | } |
11486 | }; |
11487 | |
11488 | struct AAPotentialValuesReturned : public AAPotentialValuesFloating { |
11489 | using Base = AAPotentialValuesFloating; |
11490 | AAPotentialValuesReturned(const IRPosition &IRP, Attributor &A) |
11491 | : Base(IRP, A) {} |
11492 | |
11493 | /// See AbstractAttribute::initialize(..). |
11494 | void initialize(Attributor &A) override { |
11495 | Function *F = getAssociatedFunction(); |
11496 | if (!F || F->isDeclaration() || F->getReturnType()->isVoidTy()) { |
11497 | indicatePessimisticFixpoint(); |
11498 | return; |
11499 | } |
11500 | |
11501 | for (Argument &Arg : F->args()) |
11502 | if (Arg.hasReturnedAttr()) { |
11503 | addValue(A, State&: getState(), V&: Arg, CtxI: nullptr, S: AA::AnyScope, AnchorScope: F); |
11504 | ReturnedArg = &Arg; |
11505 | break; |
11506 | } |
11507 | if (!A.isFunctionIPOAmendable(F: *F) || |
11508 | A.hasSimplificationCallback(IRP: getIRPosition())) { |
11509 | if (!ReturnedArg) |
11510 | indicatePessimisticFixpoint(); |
11511 | else |
11512 | indicateOptimisticFixpoint(); |
11513 | } |
11514 | } |
11515 | |
11516 | /// See AbstractAttribute::updateImpl(...). |
11517 | ChangeStatus updateImpl(Attributor &A) override { |
11518 | auto AssumedBefore = getAssumed(); |
11519 | bool UsedAssumedInformation = false; |
11520 | |
11521 | SmallVector<AA::ValueAndContext> Values; |
11522 | Function *AnchorScope = getAnchorScope(); |
11523 | auto HandleReturnedValue = [&](Value &V, Instruction *CtxI, |
11524 | bool AddValues) { |
11525 | for (AA::ValueScope S : {AA::Interprocedural, AA::Intraprocedural}) { |
11526 | Values.clear(); |
11527 | if (!A.getAssumedSimplifiedValues(IRP: IRPosition::value(V), AA: this, Values, S, |
11528 | UsedAssumedInformation, |
11529 | /* RecurseForSelectAndPHI */ true)) |
11530 | return false; |
11531 | if (!AddValues) |
11532 | continue; |
11533 | for (const AA::ValueAndContext &VAC : Values) |
11534 | addValue(A, State&: getState(), V&: *VAC.getValue(), |
11535 | CtxI: VAC.getCtxI() ? VAC.getCtxI() : CtxI, S, AnchorScope); |
11536 | } |
11537 | return true; |
11538 | }; |
11539 | |
11540 | if (ReturnedArg) { |
11541 | HandleReturnedValue(*ReturnedArg, nullptr, true); |
11542 | } else { |
11543 | auto RetInstPred = [&](Instruction &RetI) { |
11544 | bool AddValues = true; |
11545 | if (isa<PHINode>(Val: RetI.getOperand(i: 0)) || |
11546 | isa<SelectInst>(Val: RetI.getOperand(i: 0))) { |
11547 | addValue(A, State&: getState(), V&: *RetI.getOperand(i: 0), CtxI: &RetI, S: AA::AnyScope, |
11548 | AnchorScope); |
11549 | AddValues = false; |
11550 | } |
11551 | return HandleReturnedValue(*RetI.getOperand(i: 0), &RetI, AddValues); |
11552 | }; |
11553 | |
11554 | if (!A.checkForAllInstructions(Pred: RetInstPred, QueryingAA: *this, Opcodes: {Instruction::Ret}, |
11555 | UsedAssumedInformation, |
11556 | /* CheckBBLivenessOnly */ true)) |
11557 | return indicatePessimisticFixpoint(); |
11558 | } |
11559 | |
11560 | return (AssumedBefore == getAssumed()) ? ChangeStatus::UNCHANGED |
11561 | : ChangeStatus::CHANGED; |
11562 | } |
11563 | |
11564 | void addValue(Attributor &A, StateType &State, Value &V, |
11565 | const Instruction *CtxI, AA::ValueScope S, |
11566 | Function *AnchorScope) const override { |
11567 | Function *F = getAssociatedFunction(); |
11568 | if (auto *CB = dyn_cast<CallBase>(Val: &V)) |
11569 | if (CB->getCalledOperand() == F) |
11570 | return; |
11571 | Base::addValue(A, State, V, CtxI, S, AnchorScope); |
11572 | } |
11573 | |
11574 | ChangeStatus manifest(Attributor &A) override { |
11575 | if (ReturnedArg) |
11576 | return ChangeStatus::UNCHANGED; |
11577 | SmallVector<AA::ValueAndContext> Values; |
11578 | if (!getAssumedSimplifiedValues(A, Values, S: AA::ValueScope::Intraprocedural, |
11579 | /* RecurseForSelectAndPHI */ true)) |
11580 | return ChangeStatus::UNCHANGED; |
11581 | Value *NewVal = getSingleValue(A, AA: *this, IRP: getIRPosition(), Values); |
11582 | if (!NewVal) |
11583 | return ChangeStatus::UNCHANGED; |
11584 | |
11585 | ChangeStatus Changed = ChangeStatus::UNCHANGED; |
11586 | if (auto *Arg = dyn_cast<Argument>(Val: NewVal)) { |
11587 | STATS_DECLTRACK(UniqueReturnValue, FunctionReturn, |
11588 | "Number of function with unique return" ); |
11589 | Changed |= A.manifestAttrs( |
11590 | IRPosition::argument(*Arg), |
11591 | {Attribute::get(Arg->getContext(), Attribute::Returned)}); |
11592 | STATS_DECLTRACK_ARG_ATTR(returned); |
11593 | } |
11594 | |
11595 | auto RetInstPred = [&](Instruction &RetI) { |
11596 | Value *RetOp = RetI.getOperand(i: 0); |
11597 | if (isa<UndefValue>(Val: RetOp) || RetOp == NewVal) |
11598 | return true; |
11599 | if (AA::isValidAtPosition(VAC: {*NewVal, RetI}, InfoCache&: A.getInfoCache())) |
11600 | if (A.changeUseAfterManifest(U&: RetI.getOperandUse(i: 0), NV&: *NewVal)) |
11601 | Changed = ChangeStatus::CHANGED; |
11602 | return true; |
11603 | }; |
11604 | bool UsedAssumedInformation = false; |
11605 | (void)A.checkForAllInstructions(Pred: RetInstPred, QueryingAA: *this, Opcodes: {Instruction::Ret}, |
11606 | UsedAssumedInformation, |
11607 | /* CheckBBLivenessOnly */ true); |
11608 | return Changed; |
11609 | } |
11610 | |
11611 | ChangeStatus indicatePessimisticFixpoint() override { |
11612 | return AAPotentialValues::indicatePessimisticFixpoint(); |
11613 | } |
11614 | |
11615 | /// See AbstractAttribute::trackStatistics() |
11616 | void trackStatistics() const override{ |
11617 | STATS_DECLTRACK_FNRET_ATTR(potential_values)} |
11618 | |
11619 | /// The argumented with an existing `returned` attribute. |
11620 | Argument *ReturnedArg = nullptr; |
11621 | }; |
11622 | |
11623 | struct AAPotentialValuesFunction : AAPotentialValuesImpl { |
11624 | AAPotentialValuesFunction(const IRPosition &IRP, Attributor &A) |
11625 | : AAPotentialValuesImpl(IRP, A) {} |
11626 | |
11627 | /// See AbstractAttribute::updateImpl(...). |
11628 | ChangeStatus updateImpl(Attributor &A) override { |
11629 | llvm_unreachable("AAPotentialValues(Function|CallSite)::updateImpl will " |
11630 | "not be called" ); |
11631 | } |
11632 | |
11633 | /// See AbstractAttribute::trackStatistics() |
11634 | void trackStatistics() const override { |
11635 | STATS_DECLTRACK_FN_ATTR(potential_values) |
11636 | } |
11637 | }; |
11638 | |
11639 | struct AAPotentialValuesCallSite : AAPotentialValuesFunction { |
11640 | AAPotentialValuesCallSite(const IRPosition &IRP, Attributor &A) |
11641 | : AAPotentialValuesFunction(IRP, A) {} |
11642 | |
11643 | /// See AbstractAttribute::trackStatistics() |
11644 | void trackStatistics() const override { |
11645 | STATS_DECLTRACK_CS_ATTR(potential_values) |
11646 | } |
11647 | }; |
11648 | |
11649 | struct AAPotentialValuesCallSiteReturned : AAPotentialValuesImpl { |
11650 | AAPotentialValuesCallSiteReturned(const IRPosition &IRP, Attributor &A) |
11651 | : AAPotentialValuesImpl(IRP, A) {} |
11652 | |
11653 | /// See AbstractAttribute::updateImpl(...). |
11654 | ChangeStatus updateImpl(Attributor &A) override { |
11655 | auto AssumedBefore = getAssumed(); |
11656 | |
11657 | Function *Callee = getAssociatedFunction(); |
11658 | if (!Callee) |
11659 | return indicatePessimisticFixpoint(); |
11660 | |
11661 | bool UsedAssumedInformation = false; |
11662 | auto *CB = cast<CallBase>(Val: getCtxI()); |
11663 | if (CB->isMustTailCall() && |
11664 | !A.isAssumedDead(IRP: IRPosition::inst(I: *CB), QueryingAA: this, FnLivenessAA: nullptr, |
11665 | UsedAssumedInformation)) |
11666 | return indicatePessimisticFixpoint(); |
11667 | |
11668 | SmallVector<AA::ValueAndContext> Values; |
11669 | if (!A.getAssumedSimplifiedValues(IRP: IRPosition::returned(F: *Callee), AA: this, |
11670 | Values, S: AA::Intraprocedural, |
11671 | UsedAssumedInformation)) |
11672 | return indicatePessimisticFixpoint(); |
11673 | |
11674 | Function *Caller = CB->getCaller(); |
11675 | |
11676 | bool AnyNonLocal = false; |
11677 | for (auto &It : Values) { |
11678 | Value *V = It.getValue(); |
11679 | std::optional<Value *> CallerV = A.translateArgumentToCallSiteContent( |
11680 | V, CB&: *CB, AA: *this, UsedAssumedInformation); |
11681 | if (!CallerV.has_value()) { |
11682 | // Nothing to do as long as no value was determined. |
11683 | continue; |
11684 | } |
11685 | V = *CallerV ? *CallerV : V; |
11686 | if (AA::isDynamicallyUnique(A, QueryingAA: *this, V: *V) && |
11687 | AA::isValidInScope(V: *V, Scope: Caller)) { |
11688 | if (*CallerV) { |
11689 | SmallVector<AA::ValueAndContext> ArgValues; |
11690 | IRPosition IRP = IRPosition::value(V: *V); |
11691 | if (auto *Arg = dyn_cast<Argument>(Val: V)) |
11692 | if (Arg->getParent() == CB->getCalledOperand()) |
11693 | IRP = IRPosition::callsite_argument(CB: *CB, ArgNo: Arg->getArgNo()); |
11694 | if (recurseForValue(A, IRP, S: AA::AnyScope)) |
11695 | continue; |
11696 | } |
11697 | addValue(A, State&: getState(), V&: *V, CtxI: CB, S: AA::AnyScope, AnchorScope: getAnchorScope()); |
11698 | } else { |
11699 | AnyNonLocal = true; |
11700 | break; |
11701 | } |
11702 | } |
11703 | if (AnyNonLocal) { |
11704 | Values.clear(); |
11705 | if (!A.getAssumedSimplifiedValues(IRP: IRPosition::returned(F: *Callee), AA: this, |
11706 | Values, S: AA::Interprocedural, |
11707 | UsedAssumedInformation)) |
11708 | return indicatePessimisticFixpoint(); |
11709 | AnyNonLocal = false; |
11710 | getState() = PotentialLLVMValuesState::getBestState(); |
11711 | for (auto &It : Values) { |
11712 | Value *V = It.getValue(); |
11713 | if (!AA::isDynamicallyUnique(A, QueryingAA: *this, V: *V)) |
11714 | return indicatePessimisticFixpoint(); |
11715 | if (AA::isValidInScope(V: *V, Scope: Caller)) { |
11716 | addValue(A, State&: getState(), V&: *V, CtxI: CB, S: AA::AnyScope, AnchorScope: getAnchorScope()); |
11717 | } else { |
11718 | AnyNonLocal = true; |
11719 | addValue(A, State&: getState(), V&: *V, CtxI: CB, S: AA::Interprocedural, |
11720 | AnchorScope: getAnchorScope()); |
11721 | } |
11722 | } |
11723 | if (AnyNonLocal) |
11724 | giveUpOnIntraprocedural(A); |
11725 | } |
11726 | return (AssumedBefore == getAssumed()) ? ChangeStatus::UNCHANGED |
11727 | : ChangeStatus::CHANGED; |
11728 | } |
11729 | |
11730 | ChangeStatus indicatePessimisticFixpoint() override { |
11731 | return AAPotentialValues::indicatePessimisticFixpoint(); |
11732 | } |
11733 | |
11734 | /// See AbstractAttribute::trackStatistics() |
11735 | void trackStatistics() const override { |
11736 | STATS_DECLTRACK_CSRET_ATTR(potential_values) |
11737 | } |
11738 | }; |
11739 | |
11740 | struct AAPotentialValuesCallSiteArgument : AAPotentialValuesFloating { |
11741 | AAPotentialValuesCallSiteArgument(const IRPosition &IRP, Attributor &A) |
11742 | : AAPotentialValuesFloating(IRP, A) {} |
11743 | |
11744 | /// See AbstractAttribute::trackStatistics() |
11745 | void trackStatistics() const override { |
11746 | STATS_DECLTRACK_CSARG_ATTR(potential_values) |
11747 | } |
11748 | }; |
11749 | } // namespace |
11750 | |
11751 | /// ---------------------- Assumption Propagation ------------------------------ |
11752 | namespace { |
11753 | struct AAAssumptionInfoImpl : public AAAssumptionInfo { |
11754 | AAAssumptionInfoImpl(const IRPosition &IRP, Attributor &A, |
11755 | const DenseSet<StringRef> &Known) |
11756 | : AAAssumptionInfo(IRP, A, Known) {} |
11757 | |
11758 | /// See AbstractAttribute::manifest(...). |
11759 | ChangeStatus manifest(Attributor &A) override { |
11760 | // Don't manifest a universal set if it somehow made it here. |
11761 | if (getKnown().isUniversal()) |
11762 | return ChangeStatus::UNCHANGED; |
11763 | |
11764 | const IRPosition &IRP = getIRPosition(); |
11765 | return A.manifestAttrs( |
11766 | IRP, |
11767 | DeducedAttrs: Attribute::get(Context&: IRP.getAnchorValue().getContext(), Kind: AssumptionAttrKey, |
11768 | Val: llvm::join(R: getAssumed().getSet(), Separator: "," )), |
11769 | /* ForceReplace */ true); |
11770 | } |
11771 | |
11772 | bool hasAssumption(const StringRef Assumption) const override { |
11773 | return isValidState() && setContains(Assumption); |
11774 | } |
11775 | |
11776 | /// See AbstractAttribute::getAsStr() |
11777 | const std::string getAsStr(Attributor *A) const override { |
11778 | const SetContents &Known = getKnown(); |
11779 | const SetContents &Assumed = getAssumed(); |
11780 | |
11781 | const std::string KnownStr = |
11782 | llvm::join(Begin: Known.getSet().begin(), End: Known.getSet().end(), Separator: "," ); |
11783 | const std::string AssumedStr = |
11784 | (Assumed.isUniversal()) |
11785 | ? "Universal" |
11786 | : llvm::join(Begin: Assumed.getSet().begin(), End: Assumed.getSet().end(), Separator: "," ); |
11787 | |
11788 | return "Known [" + KnownStr + "]," + " Assumed [" + AssumedStr + "]" ; |
11789 | } |
11790 | }; |
11791 | |
11792 | /// Propagates assumption information from parent functions to all of their |
11793 | /// successors. An assumption can be propagated if the containing function |
11794 | /// dominates the called function. |
11795 | /// |
11796 | /// We start with a "known" set of assumptions already valid for the associated |
11797 | /// function and an "assumed" set that initially contains all possible |
11798 | /// assumptions. The assumed set is inter-procedurally updated by narrowing its |
11799 | /// contents as concrete values are known. The concrete values are seeded by the |
11800 | /// first nodes that are either entries into the call graph, or contains no |
11801 | /// assumptions. Each node is updated as the intersection of the assumed state |
11802 | /// with all of its predecessors. |
11803 | struct AAAssumptionInfoFunction final : AAAssumptionInfoImpl { |
11804 | AAAssumptionInfoFunction(const IRPosition &IRP, Attributor &A) |
11805 | : AAAssumptionInfoImpl(IRP, A, |
11806 | getAssumptions(F: *IRP.getAssociatedFunction())) {} |
11807 | |
11808 | /// See AbstractAttribute::updateImpl(...). |
11809 | ChangeStatus updateImpl(Attributor &A) override { |
11810 | bool Changed = false; |
11811 | |
11812 | auto CallSitePred = [&](AbstractCallSite ACS) { |
11813 | const auto *AssumptionAA = A.getAAFor<AAAssumptionInfo>( |
11814 | QueryingAA: *this, IRP: IRPosition::callsite_function(CB: *ACS.getInstruction()), |
11815 | DepClass: DepClassTy::REQUIRED); |
11816 | if (!AssumptionAA) |
11817 | return false; |
11818 | // Get the set of assumptions shared by all of this function's callers. |
11819 | Changed |= getIntersection(RHS: AssumptionAA->getAssumed()); |
11820 | return !getAssumed().empty() || !getKnown().empty(); |
11821 | }; |
11822 | |
11823 | bool UsedAssumedInformation = false; |
11824 | // Get the intersection of all assumptions held by this node's predecessors. |
11825 | // If we don't know all the call sites then this is either an entry into the |
11826 | // call graph or an empty node. This node is known to only contain its own |
11827 | // assumptions and can be propagated to its successors. |
11828 | if (!A.checkForAllCallSites(Pred: CallSitePred, QueryingAA: *this, RequireAllCallSites: true, |
11829 | UsedAssumedInformation)) |
11830 | return indicatePessimisticFixpoint(); |
11831 | |
11832 | return Changed ? ChangeStatus::CHANGED : ChangeStatus::UNCHANGED; |
11833 | } |
11834 | |
11835 | void trackStatistics() const override {} |
11836 | }; |
11837 | |
11838 | /// Assumption Info defined for call sites. |
11839 | struct AAAssumptionInfoCallSite final : AAAssumptionInfoImpl { |
11840 | |
11841 | AAAssumptionInfoCallSite(const IRPosition &IRP, Attributor &A) |
11842 | : AAAssumptionInfoImpl(IRP, A, getInitialAssumptions(IRP)) {} |
11843 | |
11844 | /// See AbstractAttribute::initialize(...). |
11845 | void initialize(Attributor &A) override { |
11846 | const IRPosition &FnPos = IRPosition::function(F: *getAnchorScope()); |
11847 | A.getAAFor<AAAssumptionInfo>(QueryingAA: *this, IRP: FnPos, DepClass: DepClassTy::REQUIRED); |
11848 | } |
11849 | |
11850 | /// See AbstractAttribute::updateImpl(...). |
11851 | ChangeStatus updateImpl(Attributor &A) override { |
11852 | const IRPosition &FnPos = IRPosition::function(F: *getAnchorScope()); |
11853 | auto *AssumptionAA = |
11854 | A.getAAFor<AAAssumptionInfo>(QueryingAA: *this, IRP: FnPos, DepClass: DepClassTy::REQUIRED); |
11855 | if (!AssumptionAA) |
11856 | return indicatePessimisticFixpoint(); |
11857 | bool Changed = getIntersection(RHS: AssumptionAA->getAssumed()); |
11858 | return Changed ? ChangeStatus::CHANGED : ChangeStatus::UNCHANGED; |
11859 | } |
11860 | |
11861 | /// See AbstractAttribute::trackStatistics() |
11862 | void trackStatistics() const override {} |
11863 | |
11864 | private: |
11865 | /// Helper to initialized the known set as all the assumptions this call and |
11866 | /// the callee contain. |
11867 | DenseSet<StringRef> getInitialAssumptions(const IRPosition &IRP) { |
11868 | const CallBase &CB = cast<CallBase>(Val&: IRP.getAssociatedValue()); |
11869 | auto Assumptions = getAssumptions(CB); |
11870 | if (const Function *F = CB.getCaller()) |
11871 | set_union(S1&: Assumptions, S2: getAssumptions(F: *F)); |
11872 | if (Function *F = IRP.getAssociatedFunction()) |
11873 | set_union(S1&: Assumptions, S2: getAssumptions(F: *F)); |
11874 | return Assumptions; |
11875 | } |
11876 | }; |
11877 | } // namespace |
11878 | |
11879 | AACallGraphNode *AACallEdgeIterator::operator*() const { |
11880 | return static_cast<AACallGraphNode *>(const_cast<AACallEdges *>( |
11881 | A.getOrCreateAAFor<AACallEdges>(IRP: IRPosition::function(F: **I)))); |
11882 | } |
11883 | |
11884 | void AttributorCallGraph::print() { llvm::WriteGraph(O&: outs(), G: this); } |
11885 | |
11886 | /// ------------------------ UnderlyingObjects --------------------------------- |
11887 | |
11888 | namespace { |
11889 | struct AAUnderlyingObjectsImpl |
11890 | : StateWrapper<BooleanState, AAUnderlyingObjects> { |
11891 | using BaseTy = StateWrapper<BooleanState, AAUnderlyingObjects>; |
11892 | AAUnderlyingObjectsImpl(const IRPosition &IRP, Attributor &A) : BaseTy(IRP) {} |
11893 | |
11894 | /// See AbstractAttribute::getAsStr(). |
11895 | const std::string getAsStr(Attributor *A) const override { |
11896 | return std::string("UnderlyingObjects " ) + |
11897 | (isValidState() |
11898 | ? (std::string("inter #" ) + |
11899 | std::to_string(val: InterAssumedUnderlyingObjects.size()) + |
11900 | " objs" + std::string(", intra #" ) + |
11901 | std::to_string(val: IntraAssumedUnderlyingObjects.size()) + |
11902 | " objs" ) |
11903 | : "<invalid>" ); |
11904 | } |
11905 | |
11906 | /// See AbstractAttribute::trackStatistics() |
11907 | void trackStatistics() const override {} |
11908 | |
11909 | /// See AbstractAttribute::updateImpl(...). |
11910 | ChangeStatus updateImpl(Attributor &A) override { |
11911 | auto &Ptr = getAssociatedValue(); |
11912 | |
11913 | auto DoUpdate = [&](SmallSetVector<Value *, 8> &UnderlyingObjects, |
11914 | AA::ValueScope Scope) { |
11915 | bool UsedAssumedInformation = false; |
11916 | SmallPtrSet<Value *, 8> SeenObjects; |
11917 | SmallVector<AA::ValueAndContext> Values; |
11918 | |
11919 | if (!A.getAssumedSimplifiedValues(IRP: IRPosition::value(V: Ptr), AA: *this, Values, |
11920 | S: Scope, UsedAssumedInformation)) |
11921 | return UnderlyingObjects.insert(X: &Ptr); |
11922 | |
11923 | bool Changed = false; |
11924 | |
11925 | for (unsigned I = 0; I < Values.size(); ++I) { |
11926 | auto &VAC = Values[I]; |
11927 | auto *Obj = VAC.getValue(); |
11928 | Value *UO = getUnderlyingObject(V: Obj); |
11929 | if (UO && UO != VAC.getValue() && SeenObjects.insert(Ptr: UO).second) { |
11930 | const auto *OtherAA = A.getAAFor<AAUnderlyingObjects>( |
11931 | QueryingAA: *this, IRP: IRPosition::value(V: *UO), DepClass: DepClassTy::OPTIONAL); |
11932 | auto Pred = [&Values](Value &V) { |
11933 | Values.emplace_back(Args&: V, Args: nullptr); |
11934 | return true; |
11935 | }; |
11936 | |
11937 | if (!OtherAA || !OtherAA->forallUnderlyingObjects(Pred, Scope)) |
11938 | llvm_unreachable( |
11939 | "The forall call should not return false at this position" ); |
11940 | |
11941 | continue; |
11942 | } |
11943 | |
11944 | if (isa<SelectInst>(Val: Obj)) { |
11945 | Changed |= handleIndirect(A, V&: *Obj, UnderlyingObjects, Scope); |
11946 | continue; |
11947 | } |
11948 | if (auto *PHI = dyn_cast<PHINode>(Val: Obj)) { |
11949 | // Explicitly look through PHIs as we do not care about dynamically |
11950 | // uniqueness. |
11951 | for (unsigned u = 0, e = PHI->getNumIncomingValues(); u < e; u++) { |
11952 | Changed |= handleIndirect(A, V&: *PHI->getIncomingValue(i: u), |
11953 | UnderlyingObjects, Scope); |
11954 | } |
11955 | continue; |
11956 | } |
11957 | |
11958 | Changed |= UnderlyingObjects.insert(X: Obj); |
11959 | } |
11960 | |
11961 | return Changed; |
11962 | }; |
11963 | |
11964 | bool Changed = false; |
11965 | Changed |= DoUpdate(IntraAssumedUnderlyingObjects, AA::Intraprocedural); |
11966 | Changed |= DoUpdate(InterAssumedUnderlyingObjects, AA::Interprocedural); |
11967 | |
11968 | return Changed ? ChangeStatus::CHANGED : ChangeStatus::UNCHANGED; |
11969 | } |
11970 | |
11971 | bool forallUnderlyingObjects( |
11972 | function_ref<bool(Value &)> Pred, |
11973 | AA::ValueScope Scope = AA::Interprocedural) const override { |
11974 | if (!isValidState()) |
11975 | return Pred(getAssociatedValue()); |
11976 | |
11977 | auto &AssumedUnderlyingObjects = Scope == AA::Intraprocedural |
11978 | ? IntraAssumedUnderlyingObjects |
11979 | : InterAssumedUnderlyingObjects; |
11980 | for (Value *Obj : AssumedUnderlyingObjects) |
11981 | if (!Pred(*Obj)) |
11982 | return false; |
11983 | |
11984 | return true; |
11985 | } |
11986 | |
11987 | private: |
11988 | /// Handle the case where the value is not the actual underlying value, such |
11989 | /// as a phi node or a select instruction. |
11990 | bool handleIndirect(Attributor &A, Value &V, |
11991 | SmallSetVector<Value *, 8> &UnderlyingObjects, |
11992 | AA::ValueScope Scope) { |
11993 | bool Changed = false; |
11994 | const auto *AA = A.getAAFor<AAUnderlyingObjects>( |
11995 | QueryingAA: *this, IRP: IRPosition::value(V), DepClass: DepClassTy::OPTIONAL); |
11996 | auto Pred = [&](Value &V) { |
11997 | Changed |= UnderlyingObjects.insert(X: &V); |
11998 | return true; |
11999 | }; |
12000 | if (!AA || !AA->forallUnderlyingObjects(Pred, Scope)) |
12001 | llvm_unreachable( |
12002 | "The forall call should not return false at this position" ); |
12003 | return Changed; |
12004 | } |
12005 | |
12006 | /// All the underlying objects collected so far via intra procedural scope. |
12007 | SmallSetVector<Value *, 8> IntraAssumedUnderlyingObjects; |
12008 | /// All the underlying objects collected so far via inter procedural scope. |
12009 | SmallSetVector<Value *, 8> InterAssumedUnderlyingObjects; |
12010 | }; |
12011 | |
12012 | struct AAUnderlyingObjectsFloating final : AAUnderlyingObjectsImpl { |
12013 | AAUnderlyingObjectsFloating(const IRPosition &IRP, Attributor &A) |
12014 | : AAUnderlyingObjectsImpl(IRP, A) {} |
12015 | }; |
12016 | |
12017 | struct AAUnderlyingObjectsArgument final : AAUnderlyingObjectsImpl { |
12018 | AAUnderlyingObjectsArgument(const IRPosition &IRP, Attributor &A) |
12019 | : AAUnderlyingObjectsImpl(IRP, A) {} |
12020 | }; |
12021 | |
12022 | struct AAUnderlyingObjectsCallSite final : AAUnderlyingObjectsImpl { |
12023 | AAUnderlyingObjectsCallSite(const IRPosition &IRP, Attributor &A) |
12024 | : AAUnderlyingObjectsImpl(IRP, A) {} |
12025 | }; |
12026 | |
12027 | struct AAUnderlyingObjectsCallSiteArgument final : AAUnderlyingObjectsImpl { |
12028 | AAUnderlyingObjectsCallSiteArgument(const IRPosition &IRP, Attributor &A) |
12029 | : AAUnderlyingObjectsImpl(IRP, A) {} |
12030 | }; |
12031 | |
12032 | struct AAUnderlyingObjectsReturned final : AAUnderlyingObjectsImpl { |
12033 | AAUnderlyingObjectsReturned(const IRPosition &IRP, Attributor &A) |
12034 | : AAUnderlyingObjectsImpl(IRP, A) {} |
12035 | }; |
12036 | |
12037 | struct AAUnderlyingObjectsCallSiteReturned final : AAUnderlyingObjectsImpl { |
12038 | AAUnderlyingObjectsCallSiteReturned(const IRPosition &IRP, Attributor &A) |
12039 | : AAUnderlyingObjectsImpl(IRP, A) {} |
12040 | }; |
12041 | |
12042 | struct AAUnderlyingObjectsFunction final : AAUnderlyingObjectsImpl { |
12043 | AAUnderlyingObjectsFunction(const IRPosition &IRP, Attributor &A) |
12044 | : AAUnderlyingObjectsImpl(IRP, A) {} |
12045 | }; |
12046 | } // namespace |
12047 | |
12048 | /// ------------------------ Global Value Info ------------------------------- |
12049 | namespace { |
12050 | struct AAGlobalValueInfoFloating : public AAGlobalValueInfo { |
12051 | AAGlobalValueInfoFloating(const IRPosition &IRP, Attributor &A) |
12052 | : AAGlobalValueInfo(IRP, A) {} |
12053 | |
12054 | /// See AbstractAttribute::initialize(...). |
12055 | void initialize(Attributor &A) override {} |
12056 | |
12057 | bool checkUse(Attributor &A, const Use &U, bool &Follow, |
12058 | SmallVectorImpl<const Value *> &Worklist) { |
12059 | Instruction *UInst = dyn_cast<Instruction>(Val: U.getUser()); |
12060 | if (!UInst) { |
12061 | Follow = true; |
12062 | return true; |
12063 | } |
12064 | |
12065 | LLVM_DEBUG(dbgs() << "[AAGlobalValueInfo] Check use: " << *U.get() << " in " |
12066 | << *UInst << "\n" ); |
12067 | |
12068 | if (auto *Cmp = dyn_cast<ICmpInst>(Val: U.getUser())) { |
12069 | int Idx = &Cmp->getOperandUse(i: 0) == &U; |
12070 | if (isa<Constant>(Val: Cmp->getOperand(i_nocapture: Idx))) |
12071 | return true; |
12072 | return U == &getAnchorValue(); |
12073 | } |
12074 | |
12075 | // Explicitly catch return instructions. |
12076 | if (isa<ReturnInst>(Val: UInst)) { |
12077 | auto CallSitePred = [&](AbstractCallSite ACS) { |
12078 | Worklist.push_back(Elt: ACS.getInstruction()); |
12079 | return true; |
12080 | }; |
12081 | bool UsedAssumedInformation = false; |
12082 | // TODO: We should traverse the uses or add a "non-call-site" CB. |
12083 | if (!A.checkForAllCallSites(Pred: CallSitePred, Fn: *UInst->getFunction(), |
12084 | /*RequireAllCallSites=*/true, QueryingAA: this, |
12085 | UsedAssumedInformation)) |
12086 | return false; |
12087 | return true; |
12088 | } |
12089 | |
12090 | // For now we only use special logic for call sites. However, the tracker |
12091 | // itself knows about a lot of other non-capturing cases already. |
12092 | auto *CB = dyn_cast<CallBase>(Val: UInst); |
12093 | if (!CB) |
12094 | return false; |
12095 | // Direct calls are OK uses. |
12096 | if (CB->isCallee(U: &U)) |
12097 | return true; |
12098 | // Non-argument uses are scary. |
12099 | if (!CB->isArgOperand(U: &U)) |
12100 | return false; |
12101 | // TODO: Iterate callees. |
12102 | auto *Fn = dyn_cast<Function>(Val: CB->getCalledOperand()); |
12103 | if (!Fn || !A.isFunctionIPOAmendable(F: *Fn)) |
12104 | return false; |
12105 | |
12106 | unsigned ArgNo = CB->getArgOperandNo(U: &U); |
12107 | Worklist.push_back(Elt: Fn->getArg(i: ArgNo)); |
12108 | return true; |
12109 | } |
12110 | |
12111 | ChangeStatus updateImpl(Attributor &A) override { |
12112 | unsigned NumUsesBefore = Uses.size(); |
12113 | |
12114 | SmallPtrSet<const Value *, 8> Visited; |
12115 | SmallVector<const Value *> Worklist; |
12116 | Worklist.push_back(Elt: &getAnchorValue()); |
12117 | |
12118 | auto UsePred = [&](const Use &U, bool &Follow) -> bool { |
12119 | Uses.insert(Ptr: &U); |
12120 | switch (DetermineUseCaptureKind(U, IsDereferenceableOrNull: nullptr)) { |
12121 | case UseCaptureKind::NO_CAPTURE: |
12122 | return checkUse(A, U, Follow, Worklist); |
12123 | case UseCaptureKind::MAY_CAPTURE: |
12124 | return checkUse(A, U, Follow, Worklist); |
12125 | case UseCaptureKind::PASSTHROUGH: |
12126 | Follow = true; |
12127 | return true; |
12128 | } |
12129 | return true; |
12130 | }; |
12131 | auto EquivalentUseCB = [&](const Use &OldU, const Use &NewU) { |
12132 | Uses.insert(Ptr: &OldU); |
12133 | return true; |
12134 | }; |
12135 | |
12136 | while (!Worklist.empty()) { |
12137 | const Value *V = Worklist.pop_back_val(); |
12138 | if (!Visited.insert(Ptr: V).second) |
12139 | continue; |
12140 | if (!A.checkForAllUses(Pred: UsePred, QueryingAA: *this, V: *V, |
12141 | /* CheckBBLivenessOnly */ true, |
12142 | LivenessDepClass: DepClassTy::OPTIONAL, |
12143 | /* IgnoreDroppableUses */ true, EquivalentUseCB)) { |
12144 | return indicatePessimisticFixpoint(); |
12145 | } |
12146 | } |
12147 | |
12148 | return Uses.size() == NumUsesBefore ? ChangeStatus::UNCHANGED |
12149 | : ChangeStatus::CHANGED; |
12150 | } |
12151 | |
12152 | bool isPotentialUse(const Use &U) const override { |
12153 | return !isValidState() || Uses.contains(Ptr: &U); |
12154 | } |
12155 | |
12156 | /// See AbstractAttribute::manifest(...). |
12157 | ChangeStatus manifest(Attributor &A) override { |
12158 | return ChangeStatus::UNCHANGED; |
12159 | } |
12160 | |
12161 | /// See AbstractAttribute::getAsStr(). |
12162 | const std::string getAsStr(Attributor *A) const override { |
12163 | return "[" + std::to_string(val: Uses.size()) + " uses]" ; |
12164 | } |
12165 | |
12166 | void trackStatistics() const override { |
12167 | STATS_DECLTRACK_FLOATING_ATTR(GlobalValuesTracked); |
12168 | } |
12169 | |
12170 | private: |
12171 | /// Set of (transitive) uses of this GlobalValue. |
12172 | SmallPtrSet<const Use *, 8> Uses; |
12173 | }; |
12174 | } // namespace |
12175 | |
12176 | /// ------------------------ Indirect Call Info ------------------------------- |
12177 | namespace { |
12178 | struct AAIndirectCallInfoCallSite : public AAIndirectCallInfo { |
12179 | AAIndirectCallInfoCallSite(const IRPosition &IRP, Attributor &A) |
12180 | : AAIndirectCallInfo(IRP, A) {} |
12181 | |
12182 | /// See AbstractAttribute::initialize(...). |
12183 | void initialize(Attributor &A) override { |
12184 | auto *MD = getCtxI()->getMetadata(KindID: LLVMContext::MD_callees); |
12185 | if (!MD && !A.isClosedWorldModule()) |
12186 | return; |
12187 | |
12188 | if (MD) { |
12189 | for (const auto &Op : MD->operands()) |
12190 | if (Function *Callee = mdconst::dyn_extract_or_null<Function>(MD: Op)) |
12191 | PotentialCallees.insert(X: Callee); |
12192 | } else if (A.isClosedWorldModule()) { |
12193 | ArrayRef<Function *> IndirectlyCallableFunctions = |
12194 | A.getInfoCache().getIndirectlyCallableFunctions(A); |
12195 | PotentialCallees.insert(Start: IndirectlyCallableFunctions.begin(), |
12196 | End: IndirectlyCallableFunctions.end()); |
12197 | } |
12198 | |
12199 | if (PotentialCallees.empty()) |
12200 | indicateOptimisticFixpoint(); |
12201 | } |
12202 | |
12203 | ChangeStatus updateImpl(Attributor &A) override { |
12204 | CallBase *CB = cast<CallBase>(Val: getCtxI()); |
12205 | const Use &CalleeUse = CB->getCalledOperandUse(); |
12206 | Value *FP = CB->getCalledOperand(); |
12207 | |
12208 | SmallSetVector<Function *, 4> AssumedCalleesNow; |
12209 | bool AllCalleesKnownNow = AllCalleesKnown; |
12210 | |
12211 | auto CheckPotentialCalleeUse = [&](Function &PotentialCallee, |
12212 | bool &UsedAssumedInformation) { |
12213 | const auto *GIAA = A.getAAFor<AAGlobalValueInfo>( |
12214 | QueryingAA: *this, IRP: IRPosition::value(V: PotentialCallee), DepClass: DepClassTy::OPTIONAL); |
12215 | if (!GIAA || GIAA->isPotentialUse(U: CalleeUse)) |
12216 | return true; |
12217 | UsedAssumedInformation = !GIAA->isAtFixpoint(); |
12218 | return false; |
12219 | }; |
12220 | |
12221 | auto AddPotentialCallees = [&]() { |
12222 | for (auto *PotentialCallee : PotentialCallees) { |
12223 | bool UsedAssumedInformation = false; |
12224 | if (CheckPotentialCalleeUse(*PotentialCallee, UsedAssumedInformation)) |
12225 | AssumedCalleesNow.insert(X: PotentialCallee); |
12226 | } |
12227 | }; |
12228 | |
12229 | // Use simplification to find potential callees, if !callees was present, |
12230 | // fallback to that set if necessary. |
12231 | bool UsedAssumedInformation = false; |
12232 | SmallVector<AA::ValueAndContext> Values; |
12233 | if (!A.getAssumedSimplifiedValues(IRP: IRPosition::value(V: *FP), AA: this, Values, |
12234 | S: AA::ValueScope::AnyScope, |
12235 | UsedAssumedInformation)) { |
12236 | if (PotentialCallees.empty()) |
12237 | return indicatePessimisticFixpoint(); |
12238 | AddPotentialCallees(); |
12239 | } |
12240 | |
12241 | // Try to find a reason for \p Fn not to be a potential callee. If none was |
12242 | // found, add it to the assumed callees set. |
12243 | auto CheckPotentialCallee = [&](Function &Fn) { |
12244 | if (!PotentialCallees.empty() && !PotentialCallees.count(key: &Fn)) |
12245 | return false; |
12246 | |
12247 | auto &CachedResult = FilterResults[&Fn]; |
12248 | if (CachedResult.has_value()) |
12249 | return CachedResult.value(); |
12250 | |
12251 | bool UsedAssumedInformation = false; |
12252 | if (!CheckPotentialCalleeUse(Fn, UsedAssumedInformation)) { |
12253 | if (!UsedAssumedInformation) |
12254 | CachedResult = false; |
12255 | return false; |
12256 | } |
12257 | |
12258 | int NumFnArgs = Fn.arg_size(); |
12259 | int NumCBArgs = CB->arg_size(); |
12260 | |
12261 | // Check if any excess argument (which we fill up with poison) is known to |
12262 | // be UB on undef. |
12263 | for (int I = NumCBArgs; I < NumFnArgs; ++I) { |
12264 | bool IsKnown = false; |
12265 | if (AA::hasAssumedIRAttr<Attribute::NoUndef>( |
12266 | A, this, IRPosition::argument(*Fn.getArg(I)), |
12267 | DepClassTy::OPTIONAL, IsKnown)) { |
12268 | if (IsKnown) |
12269 | CachedResult = false; |
12270 | return false; |
12271 | } |
12272 | } |
12273 | |
12274 | CachedResult = true; |
12275 | return true; |
12276 | }; |
12277 | |
12278 | // Check simplification result, prune known UB callees, also restrict it to |
12279 | // the !callees set, if present. |
12280 | for (auto &VAC : Values) { |
12281 | if (isa<UndefValue>(Val: VAC.getValue())) |
12282 | continue; |
12283 | if (isa<ConstantPointerNull>(Val: VAC.getValue()) && |
12284 | VAC.getValue()->getType()->getPointerAddressSpace() == 0) |
12285 | continue; |
12286 | // TODO: Check for known UB, e.g., poison + noundef. |
12287 | if (auto *VACFn = dyn_cast<Function>(Val: VAC.getValue())) { |
12288 | if (CheckPotentialCallee(*VACFn)) |
12289 | AssumedCalleesNow.insert(X: VACFn); |
12290 | continue; |
12291 | } |
12292 | if (!PotentialCallees.empty()) { |
12293 | AddPotentialCallees(); |
12294 | break; |
12295 | } |
12296 | AllCalleesKnownNow = false; |
12297 | } |
12298 | |
12299 | if (AssumedCalleesNow == AssumedCallees && |
12300 | AllCalleesKnown == AllCalleesKnownNow) |
12301 | return ChangeStatus::UNCHANGED; |
12302 | |
12303 | std::swap(LHS&: AssumedCallees, RHS&: AssumedCalleesNow); |
12304 | AllCalleesKnown = AllCalleesKnownNow; |
12305 | return ChangeStatus::CHANGED; |
12306 | } |
12307 | |
12308 | /// See AbstractAttribute::manifest(...). |
12309 | ChangeStatus manifest(Attributor &A) override { |
12310 | // If we can't specialize at all, give up now. |
12311 | if (!AllCalleesKnown && AssumedCallees.empty()) |
12312 | return ChangeStatus::UNCHANGED; |
12313 | |
12314 | CallBase *CB = cast<CallBase>(Val: getCtxI()); |
12315 | bool UsedAssumedInformation = false; |
12316 | if (A.isAssumedDead(I: *CB, QueryingAA: this, /*LivenessAA=*/nullptr, |
12317 | UsedAssumedInformation)) |
12318 | return ChangeStatus::UNCHANGED; |
12319 | |
12320 | ChangeStatus Changed = ChangeStatus::UNCHANGED; |
12321 | Value *FP = CB->getCalledOperand(); |
12322 | if (FP->getType()->getPointerAddressSpace()) |
12323 | FP = new AddrSpaceCastInst(FP, PointerType::get(ElementType: FP->getType(), AddressSpace: 0), |
12324 | FP->getName() + ".as0" , CB->getIterator()); |
12325 | |
12326 | bool CBIsVoid = CB->getType()->isVoidTy(); |
12327 | BasicBlock::iterator IP = CB->getIterator(); |
12328 | FunctionType *CSFT = CB->getFunctionType(); |
12329 | SmallVector<Value *> CSArgs(CB->arg_begin(), CB->arg_end()); |
12330 | |
12331 | // If we know all callees and there are none, the call site is (effectively) |
12332 | // dead (or UB). |
12333 | if (AssumedCallees.empty()) { |
12334 | assert(AllCalleesKnown && |
12335 | "Expected all callees to be known if there are none." ); |
12336 | A.changeToUnreachableAfterManifest(I: CB); |
12337 | return ChangeStatus::CHANGED; |
12338 | } |
12339 | |
12340 | // Special handling for the single callee case. |
12341 | if (AllCalleesKnown && AssumedCallees.size() == 1) { |
12342 | auto *NewCallee = AssumedCallees.front(); |
12343 | if (isLegalToPromote(CB: *CB, Callee: NewCallee)) { |
12344 | promoteCall(CB&: *CB, Callee: NewCallee, RetBitCast: nullptr); |
12345 | return ChangeStatus::CHANGED; |
12346 | } |
12347 | Instruction *NewCall = |
12348 | CallInst::Create(Func: FunctionCallee(CSFT, NewCallee), Args: CSArgs, |
12349 | NameStr: CB->getName(), InsertBefore: CB->getIterator()); |
12350 | if (!CBIsVoid) |
12351 | A.changeAfterManifest(IRP: IRPosition::callsite_returned(CB: *CB), NV&: *NewCall); |
12352 | A.deleteAfterManifest(I&: *CB); |
12353 | return ChangeStatus::CHANGED; |
12354 | } |
12355 | |
12356 | // For each potential value we create a conditional |
12357 | // |
12358 | // ``` |
12359 | // if (ptr == value) value(args); |
12360 | // else ... |
12361 | // ``` |
12362 | // |
12363 | bool SpecializedForAnyCallees = false; |
12364 | bool SpecializedForAllCallees = AllCalleesKnown; |
12365 | ICmpInst *LastCmp = nullptr; |
12366 | SmallVector<Function *, 8> SkippedAssumedCallees; |
12367 | SmallVector<std::pair<CallInst *, Instruction *>> NewCalls; |
12368 | for (Function *NewCallee : AssumedCallees) { |
12369 | if (!A.shouldSpecializeCallSiteForCallee(AA: *this, CB&: *CB, Callee&: *NewCallee)) { |
12370 | SkippedAssumedCallees.push_back(Elt: NewCallee); |
12371 | SpecializedForAllCallees = false; |
12372 | continue; |
12373 | } |
12374 | SpecializedForAnyCallees = true; |
12375 | |
12376 | LastCmp = new ICmpInst(IP, llvm::CmpInst::ICMP_EQ, FP, NewCallee); |
12377 | Instruction *ThenTI = |
12378 | SplitBlockAndInsertIfThen(Cond: LastCmp, SplitBefore: IP, /* Unreachable */ false); |
12379 | BasicBlock *CBBB = CB->getParent(); |
12380 | A.registerManifestAddedBasicBlock(BB&: *ThenTI->getParent()); |
12381 | A.registerManifestAddedBasicBlock(BB&: *IP->getParent()); |
12382 | auto *SplitTI = cast<BranchInst>(Val: LastCmp->getNextNode()); |
12383 | BasicBlock *ElseBB; |
12384 | if (&*IP == CB) { |
12385 | ElseBB = BasicBlock::Create(Context&: ThenTI->getContext(), Name: "" , |
12386 | Parent: ThenTI->getFunction(), InsertBefore: CBBB); |
12387 | A.registerManifestAddedBasicBlock(BB&: *ElseBB); |
12388 | IP = BranchInst::Create(IfTrue: CBBB, InsertAtEnd: ElseBB)->getIterator(); |
12389 | SplitTI->replaceUsesOfWith(From: CBBB, To: ElseBB); |
12390 | } else { |
12391 | ElseBB = IP->getParent(); |
12392 | ThenTI->replaceUsesOfWith(From: ElseBB, To: CBBB); |
12393 | } |
12394 | CastInst *RetBC = nullptr; |
12395 | CallInst *NewCall = nullptr; |
12396 | if (isLegalToPromote(CB: *CB, Callee: NewCallee)) { |
12397 | auto *CBClone = cast<CallBase>(Val: CB->clone()); |
12398 | CBClone->insertBefore(InsertPos: ThenTI); |
12399 | NewCall = &cast<CallInst>(Val&: promoteCall(CB&: *CBClone, Callee: NewCallee, RetBitCast: &RetBC)); |
12400 | } else { |
12401 | NewCall = CallInst::Create(Func: FunctionCallee(CSFT, NewCallee), Args: CSArgs, |
12402 | NameStr: CB->getName(), InsertBefore: ThenTI->getIterator()); |
12403 | } |
12404 | NewCalls.push_back(Elt: {NewCall, RetBC}); |
12405 | } |
12406 | |
12407 | auto AttachCalleeMetadata = [&](CallBase &IndirectCB) { |
12408 | if (!AllCalleesKnown) |
12409 | return ChangeStatus::UNCHANGED; |
12410 | MDBuilder MDB(IndirectCB.getContext()); |
12411 | MDNode *Callees = MDB.createCallees(Callees: SkippedAssumedCallees); |
12412 | IndirectCB.setMetadata(KindID: LLVMContext::MD_callees, Node: Callees); |
12413 | return ChangeStatus::CHANGED; |
12414 | }; |
12415 | |
12416 | if (!SpecializedForAnyCallees) |
12417 | return AttachCalleeMetadata(*CB); |
12418 | |
12419 | // Check if we need the fallback indirect call still. |
12420 | if (SpecializedForAllCallees) { |
12421 | LastCmp->replaceAllUsesWith(V: ConstantInt::getTrue(Context&: LastCmp->getContext())); |
12422 | LastCmp->eraseFromParent(); |
12423 | new UnreachableInst(IP->getContext(), IP); |
12424 | IP->eraseFromParent(); |
12425 | } else { |
12426 | auto *CBClone = cast<CallInst>(Val: CB->clone()); |
12427 | CBClone->setName(CB->getName()); |
12428 | CBClone->insertBefore(BB&: *IP->getParent(), InsertPos: IP); |
12429 | NewCalls.push_back(Elt: {CBClone, nullptr}); |
12430 | AttachCalleeMetadata(*CBClone); |
12431 | } |
12432 | |
12433 | // Check if we need a PHI to merge the results. |
12434 | if (!CBIsVoid) { |
12435 | auto *PHI = PHINode::Create(Ty: CB->getType(), NumReservedValues: NewCalls.size(), |
12436 | NameStr: CB->getName() + ".phi" , |
12437 | InsertBefore: CB->getParent()->getFirstInsertionPt()); |
12438 | for (auto &It : NewCalls) { |
12439 | CallBase *NewCall = It.first; |
12440 | Instruction *CallRet = It.second ? It.second : It.first; |
12441 | if (CallRet->getType() == CB->getType()) |
12442 | PHI->addIncoming(V: CallRet, BB: CallRet->getParent()); |
12443 | else if (NewCall->getType()->isVoidTy()) |
12444 | PHI->addIncoming(V: PoisonValue::get(T: CB->getType()), |
12445 | BB: NewCall->getParent()); |
12446 | else |
12447 | llvm_unreachable("Call return should match or be void!" ); |
12448 | } |
12449 | A.changeAfterManifest(IRP: IRPosition::callsite_returned(CB: *CB), NV&: *PHI); |
12450 | } |
12451 | |
12452 | A.deleteAfterManifest(I&: *CB); |
12453 | Changed = ChangeStatus::CHANGED; |
12454 | |
12455 | return Changed; |
12456 | } |
12457 | |
12458 | /// See AbstractAttribute::getAsStr(). |
12459 | const std::string getAsStr(Attributor *A) const override { |
12460 | return std::string(AllCalleesKnown ? "eliminate" : "specialize" ) + |
12461 | " indirect call site with " + std::to_string(val: AssumedCallees.size()) + |
12462 | " functions" ; |
12463 | } |
12464 | |
12465 | void trackStatistics() const override { |
12466 | if (AllCalleesKnown) { |
12467 | STATS_DECLTRACK( |
12468 | Eliminated, CallSites, |
12469 | "Number of indirect call sites eliminated via specialization" ) |
12470 | } else { |
12471 | STATS_DECLTRACK(Specialized, CallSites, |
12472 | "Number of indirect call sites specialized" ) |
12473 | } |
12474 | } |
12475 | |
12476 | bool foreachCallee(function_ref<bool(Function *)> CB) const override { |
12477 | return isValidState() && AllCalleesKnown && all_of(Range: AssumedCallees, P: CB); |
12478 | } |
12479 | |
12480 | private: |
12481 | /// Map to remember filter results. |
12482 | DenseMap<Function *, std::optional<bool>> FilterResults; |
12483 | |
12484 | /// If the !callee metadata was present, this set will contain all potential |
12485 | /// callees (superset). |
12486 | SmallSetVector<Function *, 4> PotentialCallees; |
12487 | |
12488 | /// This set contains all currently assumed calllees, which might grow over |
12489 | /// time. |
12490 | SmallSetVector<Function *, 4> AssumedCallees; |
12491 | |
12492 | /// Flag to indicate if all possible callees are in the AssumedCallees set or |
12493 | /// if there could be others. |
12494 | bool AllCalleesKnown = true; |
12495 | }; |
12496 | } // namespace |
12497 | |
12498 | /// ------------------------ Address Space ------------------------------------ |
12499 | namespace { |
12500 | struct AAAddressSpaceImpl : public AAAddressSpace { |
12501 | AAAddressSpaceImpl(const IRPosition &IRP, Attributor &A) |
12502 | : AAAddressSpace(IRP, A) {} |
12503 | |
12504 | int32_t getAddressSpace() const override { |
12505 | assert(isValidState() && "the AA is invalid" ); |
12506 | return AssumedAddressSpace; |
12507 | } |
12508 | |
12509 | /// See AbstractAttribute::initialize(...). |
12510 | void initialize(Attributor &A) override { |
12511 | assert(getAssociatedType()->isPtrOrPtrVectorTy() && |
12512 | "Associated value is not a pointer" ); |
12513 | } |
12514 | |
12515 | ChangeStatus updateImpl(Attributor &A) override { |
12516 | int32_t OldAddressSpace = AssumedAddressSpace; |
12517 | auto *AUO = A.getOrCreateAAFor<AAUnderlyingObjects>(IRP: getIRPosition(), QueryingAA: this, |
12518 | DepClass: DepClassTy::REQUIRED); |
12519 | auto Pred = [&](Value &Obj) { |
12520 | if (isa<UndefValue>(Val: &Obj)) |
12521 | return true; |
12522 | return takeAddressSpace(AS: Obj.getType()->getPointerAddressSpace()); |
12523 | }; |
12524 | |
12525 | if (!AUO->forallUnderlyingObjects(Pred)) |
12526 | return indicatePessimisticFixpoint(); |
12527 | |
12528 | return OldAddressSpace == AssumedAddressSpace ? ChangeStatus::UNCHANGED |
12529 | : ChangeStatus::CHANGED; |
12530 | } |
12531 | |
12532 | /// See AbstractAttribute::manifest(...). |
12533 | ChangeStatus manifest(Attributor &A) override { |
12534 | Value *AssociatedValue = &getAssociatedValue(); |
12535 | Value *OriginalValue = peelAddrspacecast(V: AssociatedValue); |
12536 | if (getAddressSpace() == NoAddressSpace || |
12537 | static_cast<uint32_t>(getAddressSpace()) == |
12538 | getAssociatedType()->getPointerAddressSpace()) |
12539 | return ChangeStatus::UNCHANGED; |
12540 | |
12541 | Type *NewPtrTy = PointerType::get(C&: getAssociatedType()->getContext(), |
12542 | AddressSpace: static_cast<uint32_t>(getAddressSpace())); |
12543 | bool UseOriginalValue = |
12544 | OriginalValue->getType()->getPointerAddressSpace() == |
12545 | static_cast<uint32_t>(getAddressSpace()); |
12546 | |
12547 | bool Changed = false; |
12548 | |
12549 | auto MakeChange = [&](Instruction *I, Use &U) { |
12550 | Changed = true; |
12551 | if (UseOriginalValue) { |
12552 | A.changeUseAfterManifest(U, NV&: *OriginalValue); |
12553 | return; |
12554 | } |
12555 | Instruction *CastInst = new AddrSpaceCastInst(OriginalValue, NewPtrTy); |
12556 | CastInst->insertBefore(InsertPos: cast<Instruction>(Val: I)); |
12557 | A.changeUseAfterManifest(U, NV&: *CastInst); |
12558 | }; |
12559 | |
12560 | auto Pred = [&](const Use &U, bool &) { |
12561 | if (U.get() != AssociatedValue) |
12562 | return true; |
12563 | auto *Inst = dyn_cast<Instruction>(Val: U.getUser()); |
12564 | if (!Inst) |
12565 | return true; |
12566 | // This is a WA to make sure we only change uses from the corresponding |
12567 | // CGSCC if the AA is run on CGSCC instead of the entire module. |
12568 | if (!A.isRunOn(Fn: Inst->getFunction())) |
12569 | return true; |
12570 | if (isa<LoadInst>(Val: Inst)) |
12571 | MakeChange(Inst, const_cast<Use &>(U)); |
12572 | if (isa<StoreInst>(Val: Inst)) { |
12573 | // We only make changes if the use is the pointer operand. |
12574 | if (U.getOperandNo() == 1) |
12575 | MakeChange(Inst, const_cast<Use &>(U)); |
12576 | } |
12577 | return true; |
12578 | }; |
12579 | |
12580 | // It doesn't matter if we can't check all uses as we can simply |
12581 | // conservatively ignore those that can not be visited. |
12582 | (void)A.checkForAllUses(Pred, QueryingAA: *this, V: getAssociatedValue(), |
12583 | /* CheckBBLivenessOnly */ true); |
12584 | |
12585 | return Changed ? ChangeStatus::CHANGED : ChangeStatus::UNCHANGED; |
12586 | } |
12587 | |
12588 | /// See AbstractAttribute::getAsStr(). |
12589 | const std::string getAsStr(Attributor *A) const override { |
12590 | if (!isValidState()) |
12591 | return "addrspace(<invalid>)" ; |
12592 | return "addrspace(" + |
12593 | (AssumedAddressSpace == NoAddressSpace |
12594 | ? "none" |
12595 | : std::to_string(val: AssumedAddressSpace)) + |
12596 | ")" ; |
12597 | } |
12598 | |
12599 | private: |
12600 | int32_t AssumedAddressSpace = NoAddressSpace; |
12601 | |
12602 | bool takeAddressSpace(int32_t AS) { |
12603 | if (AssumedAddressSpace == NoAddressSpace) { |
12604 | AssumedAddressSpace = AS; |
12605 | return true; |
12606 | } |
12607 | return AssumedAddressSpace == AS; |
12608 | } |
12609 | |
12610 | static Value *peelAddrspacecast(Value *V) { |
12611 | if (auto *I = dyn_cast<AddrSpaceCastInst>(Val: V)) |
12612 | return peelAddrspacecast(V: I->getPointerOperand()); |
12613 | if (auto *C = dyn_cast<ConstantExpr>(Val: V)) |
12614 | if (C->getOpcode() == Instruction::AddrSpaceCast) |
12615 | return peelAddrspacecast(V: C->getOperand(i_nocapture: 0)); |
12616 | return V; |
12617 | } |
12618 | }; |
12619 | |
12620 | struct AAAddressSpaceFloating final : AAAddressSpaceImpl { |
12621 | AAAddressSpaceFloating(const IRPosition &IRP, Attributor &A) |
12622 | : AAAddressSpaceImpl(IRP, A) {} |
12623 | |
12624 | void trackStatistics() const override { |
12625 | STATS_DECLTRACK_FLOATING_ATTR(addrspace); |
12626 | } |
12627 | }; |
12628 | |
12629 | struct AAAddressSpaceReturned final : AAAddressSpaceImpl { |
12630 | AAAddressSpaceReturned(const IRPosition &IRP, Attributor &A) |
12631 | : AAAddressSpaceImpl(IRP, A) {} |
12632 | |
12633 | /// See AbstractAttribute::initialize(...). |
12634 | void initialize(Attributor &A) override { |
12635 | // TODO: we don't rewrite function argument for now because it will need to |
12636 | // rewrite the function signature and all call sites. |
12637 | (void)indicatePessimisticFixpoint(); |
12638 | } |
12639 | |
12640 | void trackStatistics() const override { |
12641 | STATS_DECLTRACK_FNRET_ATTR(addrspace); |
12642 | } |
12643 | }; |
12644 | |
12645 | struct AAAddressSpaceCallSiteReturned final : AAAddressSpaceImpl { |
12646 | AAAddressSpaceCallSiteReturned(const IRPosition &IRP, Attributor &A) |
12647 | : AAAddressSpaceImpl(IRP, A) {} |
12648 | |
12649 | void trackStatistics() const override { |
12650 | STATS_DECLTRACK_CSRET_ATTR(addrspace); |
12651 | } |
12652 | }; |
12653 | |
12654 | struct AAAddressSpaceArgument final : AAAddressSpaceImpl { |
12655 | AAAddressSpaceArgument(const IRPosition &IRP, Attributor &A) |
12656 | : AAAddressSpaceImpl(IRP, A) {} |
12657 | |
12658 | void trackStatistics() const override { STATS_DECLTRACK_ARG_ATTR(addrspace); } |
12659 | }; |
12660 | |
12661 | struct AAAddressSpaceCallSiteArgument final : AAAddressSpaceImpl { |
12662 | AAAddressSpaceCallSiteArgument(const IRPosition &IRP, Attributor &A) |
12663 | : AAAddressSpaceImpl(IRP, A) {} |
12664 | |
12665 | /// See AbstractAttribute::initialize(...). |
12666 | void initialize(Attributor &A) override { |
12667 | // TODO: we don't rewrite call site argument for now because it will need to |
12668 | // rewrite the function signature of the callee. |
12669 | (void)indicatePessimisticFixpoint(); |
12670 | } |
12671 | |
12672 | void trackStatistics() const override { |
12673 | STATS_DECLTRACK_CSARG_ATTR(addrspace); |
12674 | } |
12675 | }; |
12676 | } // namespace |
12677 | |
12678 | /// ----------- Allocation Info ---------- |
12679 | namespace { |
12680 | struct AAAllocationInfoImpl : public AAAllocationInfo { |
12681 | AAAllocationInfoImpl(const IRPosition &IRP, Attributor &A) |
12682 | : AAAllocationInfo(IRP, A) {} |
12683 | |
12684 | std::optional<TypeSize> getAllocatedSize() const override { |
12685 | assert(isValidState() && "the AA is invalid" ); |
12686 | return AssumedAllocatedSize; |
12687 | } |
12688 | |
12689 | std::optional<TypeSize> findInitialAllocationSize(Instruction *I, |
12690 | const DataLayout &DL) { |
12691 | |
12692 | // TODO: implement case for malloc like instructions |
12693 | switch (I->getOpcode()) { |
12694 | case Instruction::Alloca: { |
12695 | AllocaInst *AI = cast<AllocaInst>(Val: I); |
12696 | return AI->getAllocationSize(DL); |
12697 | } |
12698 | default: |
12699 | return std::nullopt; |
12700 | } |
12701 | } |
12702 | |
12703 | ChangeStatus updateImpl(Attributor &A) override { |
12704 | |
12705 | const IRPosition &IRP = getIRPosition(); |
12706 | Instruction *I = IRP.getCtxI(); |
12707 | |
12708 | // TODO: update check for malloc like calls |
12709 | if (!isa<AllocaInst>(Val: I)) |
12710 | return indicatePessimisticFixpoint(); |
12711 | |
12712 | bool IsKnownNoCapture; |
12713 | if (!AA::hasAssumedIRAttr<Attribute::NoCapture>( |
12714 | A, this, IRP, DepClassTy::OPTIONAL, IsKnownNoCapture)) |
12715 | return indicatePessimisticFixpoint(); |
12716 | |
12717 | const AAPointerInfo *PI = |
12718 | A.getOrCreateAAFor<AAPointerInfo>(IRP, QueryingAA: *this, DepClass: DepClassTy::REQUIRED); |
12719 | |
12720 | if (!PI) |
12721 | return indicatePessimisticFixpoint(); |
12722 | |
12723 | if (!PI->getState().isValidState()) |
12724 | return indicatePessimisticFixpoint(); |
12725 | |
12726 | const DataLayout &DL = A.getDataLayout(); |
12727 | const auto AllocationSize = findInitialAllocationSize(I, DL); |
12728 | |
12729 | // If allocation size is nullopt, we give up. |
12730 | if (!AllocationSize) |
12731 | return indicatePessimisticFixpoint(); |
12732 | |
12733 | // For zero sized allocations, we give up. |
12734 | // Since we can't reduce further |
12735 | if (*AllocationSize == 0) |
12736 | return indicatePessimisticFixpoint(); |
12737 | |
12738 | int64_t BinSize = PI->numOffsetBins(); |
12739 | |
12740 | // TODO: implement for multiple bins |
12741 | if (BinSize > 1) |
12742 | return indicatePessimisticFixpoint(); |
12743 | |
12744 | if (BinSize == 0) { |
12745 | auto NewAllocationSize = std::optional<TypeSize>(TypeSize(0, false)); |
12746 | if (!changeAllocationSize(Size: NewAllocationSize)) |
12747 | return ChangeStatus::UNCHANGED; |
12748 | return ChangeStatus::CHANGED; |
12749 | } |
12750 | |
12751 | // TODO: refactor this to be part of multiple bin case |
12752 | const auto &It = PI->begin(); |
12753 | |
12754 | // TODO: handle if Offset is not zero |
12755 | if (It->first.Offset != 0) |
12756 | return indicatePessimisticFixpoint(); |
12757 | |
12758 | uint64_t SizeOfBin = It->first.Offset + It->first.Size; |
12759 | |
12760 | if (SizeOfBin >= *AllocationSize) |
12761 | return indicatePessimisticFixpoint(); |
12762 | |
12763 | auto NewAllocationSize = |
12764 | std::optional<TypeSize>(TypeSize(SizeOfBin * 8, false)); |
12765 | |
12766 | if (!changeAllocationSize(Size: NewAllocationSize)) |
12767 | return ChangeStatus::UNCHANGED; |
12768 | |
12769 | return ChangeStatus::CHANGED; |
12770 | } |
12771 | |
12772 | /// See AbstractAttribute::manifest(...). |
12773 | ChangeStatus manifest(Attributor &A) override { |
12774 | |
12775 | assert(isValidState() && |
12776 | "Manifest should only be called if the state is valid." ); |
12777 | |
12778 | Instruction *I = getIRPosition().getCtxI(); |
12779 | |
12780 | auto FixedAllocatedSizeInBits = getAllocatedSize()->getFixedValue(); |
12781 | |
12782 | unsigned long NumBytesToAllocate = (FixedAllocatedSizeInBits + 7) / 8; |
12783 | |
12784 | switch (I->getOpcode()) { |
12785 | // TODO: add case for malloc like calls |
12786 | case Instruction::Alloca: { |
12787 | |
12788 | AllocaInst *AI = cast<AllocaInst>(Val: I); |
12789 | |
12790 | Type *CharType = Type::getInt8Ty(C&: I->getContext()); |
12791 | |
12792 | auto *NumBytesToValue = |
12793 | ConstantInt::get(Context&: I->getContext(), V: APInt(32, NumBytesToAllocate)); |
12794 | |
12795 | BasicBlock::iterator insertPt = AI->getIterator(); |
12796 | insertPt = std::next(x: insertPt); |
12797 | AllocaInst *NewAllocaInst = |
12798 | new AllocaInst(CharType, AI->getAddressSpace(), NumBytesToValue, |
12799 | AI->getAlign(), AI->getName(), insertPt); |
12800 | |
12801 | if (A.changeAfterManifest(IRP: IRPosition::inst(I: *AI), NV&: *NewAllocaInst)) |
12802 | return ChangeStatus::CHANGED; |
12803 | |
12804 | break; |
12805 | } |
12806 | default: |
12807 | break; |
12808 | } |
12809 | |
12810 | return ChangeStatus::UNCHANGED; |
12811 | } |
12812 | |
12813 | /// See AbstractAttribute::getAsStr(). |
12814 | const std::string getAsStr(Attributor *A) const override { |
12815 | if (!isValidState()) |
12816 | return "allocationinfo(<invalid>)" ; |
12817 | return "allocationinfo(" + |
12818 | (AssumedAllocatedSize == HasNoAllocationSize |
12819 | ? "none" |
12820 | : std::to_string(val: AssumedAllocatedSize->getFixedValue())) + |
12821 | ")" ; |
12822 | } |
12823 | |
12824 | private: |
12825 | std::optional<TypeSize> AssumedAllocatedSize = HasNoAllocationSize; |
12826 | |
12827 | // Maintain the computed allocation size of the object. |
12828 | // Returns (bool) weather the size of the allocation was modified or not. |
12829 | bool changeAllocationSize(std::optional<TypeSize> Size) { |
12830 | if (AssumedAllocatedSize == HasNoAllocationSize || |
12831 | AssumedAllocatedSize != Size) { |
12832 | AssumedAllocatedSize = Size; |
12833 | return true; |
12834 | } |
12835 | return false; |
12836 | } |
12837 | }; |
12838 | |
12839 | struct AAAllocationInfoFloating : AAAllocationInfoImpl { |
12840 | AAAllocationInfoFloating(const IRPosition &IRP, Attributor &A) |
12841 | : AAAllocationInfoImpl(IRP, A) {} |
12842 | |
12843 | void trackStatistics() const override { |
12844 | STATS_DECLTRACK_FLOATING_ATTR(allocationinfo); |
12845 | } |
12846 | }; |
12847 | |
12848 | struct AAAllocationInfoReturned : AAAllocationInfoImpl { |
12849 | AAAllocationInfoReturned(const IRPosition &IRP, Attributor &A) |
12850 | : AAAllocationInfoImpl(IRP, A) {} |
12851 | |
12852 | /// See AbstractAttribute::initialize(...). |
12853 | void initialize(Attributor &A) override { |
12854 | // TODO: we don't rewrite function argument for now because it will need to |
12855 | // rewrite the function signature and all call sites |
12856 | (void)indicatePessimisticFixpoint(); |
12857 | } |
12858 | |
12859 | void trackStatistics() const override { |
12860 | STATS_DECLTRACK_FNRET_ATTR(allocationinfo); |
12861 | } |
12862 | }; |
12863 | |
12864 | struct AAAllocationInfoCallSiteReturned : AAAllocationInfoImpl { |
12865 | AAAllocationInfoCallSiteReturned(const IRPosition &IRP, Attributor &A) |
12866 | : AAAllocationInfoImpl(IRP, A) {} |
12867 | |
12868 | void trackStatistics() const override { |
12869 | STATS_DECLTRACK_CSRET_ATTR(allocationinfo); |
12870 | } |
12871 | }; |
12872 | |
12873 | struct AAAllocationInfoArgument : AAAllocationInfoImpl { |
12874 | AAAllocationInfoArgument(const IRPosition &IRP, Attributor &A) |
12875 | : AAAllocationInfoImpl(IRP, A) {} |
12876 | |
12877 | void trackStatistics() const override { |
12878 | STATS_DECLTRACK_ARG_ATTR(allocationinfo); |
12879 | } |
12880 | }; |
12881 | |
12882 | struct AAAllocationInfoCallSiteArgument : AAAllocationInfoImpl { |
12883 | AAAllocationInfoCallSiteArgument(const IRPosition &IRP, Attributor &A) |
12884 | : AAAllocationInfoImpl(IRP, A) {} |
12885 | |
12886 | /// See AbstractAttribute::initialize(...). |
12887 | void initialize(Attributor &A) override { |
12888 | |
12889 | (void)indicatePessimisticFixpoint(); |
12890 | } |
12891 | |
12892 | void trackStatistics() const override { |
12893 | STATS_DECLTRACK_CSARG_ATTR(allocationinfo); |
12894 | } |
12895 | }; |
12896 | } // namespace |
12897 | |
12898 | const char AANoUnwind::ID = 0; |
12899 | const char AANoSync::ID = 0; |
12900 | const char AANoFree::ID = 0; |
12901 | const char AANonNull::ID = 0; |
12902 | const char AAMustProgress::ID = 0; |
12903 | const char AANoRecurse::ID = 0; |
12904 | const char AANonConvergent::ID = 0; |
12905 | const char AAWillReturn::ID = 0; |
12906 | const char AAUndefinedBehavior::ID = 0; |
12907 | const char AANoAlias::ID = 0; |
12908 | const char AAIntraFnReachability::ID = 0; |
12909 | const char AANoReturn::ID = 0; |
12910 | const char AAIsDead::ID = 0; |
12911 | const char AADereferenceable::ID = 0; |
12912 | const char AAAlign::ID = 0; |
12913 | const char AAInstanceInfo::ID = 0; |
12914 | const char AANoCapture::ID = 0; |
12915 | const char AAValueSimplify::ID = 0; |
12916 | const char AAHeapToStack::ID = 0; |
12917 | const char AAPrivatizablePtr::ID = 0; |
12918 | const char AAMemoryBehavior::ID = 0; |
12919 | const char AAMemoryLocation::ID = 0; |
12920 | const char AAValueConstantRange::ID = 0; |
12921 | const char AAPotentialConstantValues::ID = 0; |
12922 | const char AAPotentialValues::ID = 0; |
12923 | const char AANoUndef::ID = 0; |
12924 | const char AANoFPClass::ID = 0; |
12925 | const char AACallEdges::ID = 0; |
12926 | const char AAInterFnReachability::ID = 0; |
12927 | const char AAPointerInfo::ID = 0; |
12928 | const char AAAssumptionInfo::ID = 0; |
12929 | const char AAUnderlyingObjects::ID = 0; |
12930 | const char AAAddressSpace::ID = 0; |
12931 | const char AAAllocationInfo::ID = 0; |
12932 | const char AAIndirectCallInfo::ID = 0; |
12933 | const char AAGlobalValueInfo::ID = 0; |
12934 | const char AADenormalFPMath::ID = 0; |
12935 | |
12936 | // Macro magic to create the static generator function for attributes that |
12937 | // follow the naming scheme. |
12938 | |
12939 | #define SWITCH_PK_INV(CLASS, PK, POS_NAME) \ |
12940 | case IRPosition::PK: \ |
12941 | llvm_unreachable("Cannot create " #CLASS " for a " POS_NAME " position!"); |
12942 | |
12943 | #define SWITCH_PK_CREATE(CLASS, IRP, PK, SUFFIX) \ |
12944 | case IRPosition::PK: \ |
12945 | AA = new (A.Allocator) CLASS##SUFFIX(IRP, A); \ |
12946 | ++NumAAs; \ |
12947 | break; |
12948 | |
12949 | #define CREATE_FUNCTION_ABSTRACT_ATTRIBUTE_FOR_POSITION(CLASS) \ |
12950 | CLASS &CLASS::createForPosition(const IRPosition &IRP, Attributor &A) { \ |
12951 | CLASS *AA = nullptr; \ |
12952 | switch (IRP.getPositionKind()) { \ |
12953 | SWITCH_PK_INV(CLASS, IRP_INVALID, "invalid") \ |
12954 | SWITCH_PK_INV(CLASS, IRP_FLOAT, "floating") \ |
12955 | SWITCH_PK_INV(CLASS, IRP_ARGUMENT, "argument") \ |
12956 | SWITCH_PK_INV(CLASS, IRP_RETURNED, "returned") \ |
12957 | SWITCH_PK_INV(CLASS, IRP_CALL_SITE_RETURNED, "call site returned") \ |
12958 | SWITCH_PK_INV(CLASS, IRP_CALL_SITE_ARGUMENT, "call site argument") \ |
12959 | SWITCH_PK_CREATE(CLASS, IRP, IRP_FUNCTION, Function) \ |
12960 | SWITCH_PK_CREATE(CLASS, IRP, IRP_CALL_SITE, CallSite) \ |
12961 | } \ |
12962 | return *AA; \ |
12963 | } |
12964 | |
12965 | #define CREATE_VALUE_ABSTRACT_ATTRIBUTE_FOR_POSITION(CLASS) \ |
12966 | CLASS &CLASS::createForPosition(const IRPosition &IRP, Attributor &A) { \ |
12967 | CLASS *AA = nullptr; \ |
12968 | switch (IRP.getPositionKind()) { \ |
12969 | SWITCH_PK_INV(CLASS, IRP_INVALID, "invalid") \ |
12970 | SWITCH_PK_INV(CLASS, IRP_FUNCTION, "function") \ |
12971 | SWITCH_PK_INV(CLASS, IRP_CALL_SITE, "call site") \ |
12972 | SWITCH_PK_CREATE(CLASS, IRP, IRP_FLOAT, Floating) \ |
12973 | SWITCH_PK_CREATE(CLASS, IRP, IRP_ARGUMENT, Argument) \ |
12974 | SWITCH_PK_CREATE(CLASS, IRP, IRP_RETURNED, Returned) \ |
12975 | SWITCH_PK_CREATE(CLASS, IRP, IRP_CALL_SITE_RETURNED, CallSiteReturned) \ |
12976 | SWITCH_PK_CREATE(CLASS, IRP, IRP_CALL_SITE_ARGUMENT, CallSiteArgument) \ |
12977 | } \ |
12978 | return *AA; \ |
12979 | } |
12980 | |
12981 | #define CREATE_ABSTRACT_ATTRIBUTE_FOR_ONE_POSITION(POS, SUFFIX, CLASS) \ |
12982 | CLASS &CLASS::createForPosition(const IRPosition &IRP, Attributor &A) { \ |
12983 | CLASS *AA = nullptr; \ |
12984 | switch (IRP.getPositionKind()) { \ |
12985 | SWITCH_PK_CREATE(CLASS, IRP, POS, SUFFIX) \ |
12986 | default: \ |
12987 | llvm_unreachable("Cannot create " #CLASS " for position otherthan " #POS \ |
12988 | " position!"); \ |
12989 | } \ |
12990 | return *AA; \ |
12991 | } |
12992 | |
12993 | #define CREATE_ALL_ABSTRACT_ATTRIBUTE_FOR_POSITION(CLASS) \ |
12994 | CLASS &CLASS::createForPosition(const IRPosition &IRP, Attributor &A) { \ |
12995 | CLASS *AA = nullptr; \ |
12996 | switch (IRP.getPositionKind()) { \ |
12997 | SWITCH_PK_INV(CLASS, IRP_INVALID, "invalid") \ |
12998 | SWITCH_PK_CREATE(CLASS, IRP, IRP_FUNCTION, Function) \ |
12999 | SWITCH_PK_CREATE(CLASS, IRP, IRP_CALL_SITE, CallSite) \ |
13000 | SWITCH_PK_CREATE(CLASS, IRP, IRP_FLOAT, Floating) \ |
13001 | SWITCH_PK_CREATE(CLASS, IRP, IRP_ARGUMENT, Argument) \ |
13002 | SWITCH_PK_CREATE(CLASS, IRP, IRP_RETURNED, Returned) \ |
13003 | SWITCH_PK_CREATE(CLASS, IRP, IRP_CALL_SITE_RETURNED, CallSiteReturned) \ |
13004 | SWITCH_PK_CREATE(CLASS, IRP, IRP_CALL_SITE_ARGUMENT, CallSiteArgument) \ |
13005 | } \ |
13006 | return *AA; \ |
13007 | } |
13008 | |
13009 | #define CREATE_FUNCTION_ONLY_ABSTRACT_ATTRIBUTE_FOR_POSITION(CLASS) \ |
13010 | CLASS &CLASS::createForPosition(const IRPosition &IRP, Attributor &A) { \ |
13011 | CLASS *AA = nullptr; \ |
13012 | switch (IRP.getPositionKind()) { \ |
13013 | SWITCH_PK_INV(CLASS, IRP_INVALID, "invalid") \ |
13014 | SWITCH_PK_INV(CLASS, IRP_ARGUMENT, "argument") \ |
13015 | SWITCH_PK_INV(CLASS, IRP_FLOAT, "floating") \ |
13016 | SWITCH_PK_INV(CLASS, IRP_RETURNED, "returned") \ |
13017 | SWITCH_PK_INV(CLASS, IRP_CALL_SITE_RETURNED, "call site returned") \ |
13018 | SWITCH_PK_INV(CLASS, IRP_CALL_SITE_ARGUMENT, "call site argument") \ |
13019 | SWITCH_PK_INV(CLASS, IRP_CALL_SITE, "call site") \ |
13020 | SWITCH_PK_CREATE(CLASS, IRP, IRP_FUNCTION, Function) \ |
13021 | } \ |
13022 | return *AA; \ |
13023 | } |
13024 | |
13025 | #define CREATE_NON_RET_ABSTRACT_ATTRIBUTE_FOR_POSITION(CLASS) \ |
13026 | CLASS &CLASS::createForPosition(const IRPosition &IRP, Attributor &A) { \ |
13027 | CLASS *AA = nullptr; \ |
13028 | switch (IRP.getPositionKind()) { \ |
13029 | SWITCH_PK_INV(CLASS, IRP_INVALID, "invalid") \ |
13030 | SWITCH_PK_INV(CLASS, IRP_RETURNED, "returned") \ |
13031 | SWITCH_PK_CREATE(CLASS, IRP, IRP_FUNCTION, Function) \ |
13032 | SWITCH_PK_CREATE(CLASS, IRP, IRP_CALL_SITE, CallSite) \ |
13033 | SWITCH_PK_CREATE(CLASS, IRP, IRP_FLOAT, Floating) \ |
13034 | SWITCH_PK_CREATE(CLASS, IRP, IRP_ARGUMENT, Argument) \ |
13035 | SWITCH_PK_CREATE(CLASS, IRP, IRP_CALL_SITE_RETURNED, CallSiteReturned) \ |
13036 | SWITCH_PK_CREATE(CLASS, IRP, IRP_CALL_SITE_ARGUMENT, CallSiteArgument) \ |
13037 | } \ |
13038 | return *AA; \ |
13039 | } |
13040 | |
13041 | CREATE_FUNCTION_ABSTRACT_ATTRIBUTE_FOR_POSITION(AANoUnwind) |
13042 | CREATE_FUNCTION_ABSTRACT_ATTRIBUTE_FOR_POSITION(AANoSync) |
13043 | CREATE_FUNCTION_ABSTRACT_ATTRIBUTE_FOR_POSITION(AANoRecurse) |
13044 | CREATE_FUNCTION_ABSTRACT_ATTRIBUTE_FOR_POSITION(AAWillReturn) |
13045 | CREATE_FUNCTION_ABSTRACT_ATTRIBUTE_FOR_POSITION(AANoReturn) |
13046 | CREATE_FUNCTION_ABSTRACT_ATTRIBUTE_FOR_POSITION(AAMemoryLocation) |
13047 | CREATE_FUNCTION_ABSTRACT_ATTRIBUTE_FOR_POSITION(AACallEdges) |
13048 | CREATE_FUNCTION_ABSTRACT_ATTRIBUTE_FOR_POSITION(AAAssumptionInfo) |
13049 | CREATE_FUNCTION_ABSTRACT_ATTRIBUTE_FOR_POSITION(AAMustProgress) |
13050 | |
13051 | CREATE_VALUE_ABSTRACT_ATTRIBUTE_FOR_POSITION(AANonNull) |
13052 | CREATE_VALUE_ABSTRACT_ATTRIBUTE_FOR_POSITION(AANoAlias) |
13053 | CREATE_VALUE_ABSTRACT_ATTRIBUTE_FOR_POSITION(AAPrivatizablePtr) |
13054 | CREATE_VALUE_ABSTRACT_ATTRIBUTE_FOR_POSITION(AADereferenceable) |
13055 | CREATE_VALUE_ABSTRACT_ATTRIBUTE_FOR_POSITION(AAAlign) |
13056 | CREATE_VALUE_ABSTRACT_ATTRIBUTE_FOR_POSITION(AAInstanceInfo) |
13057 | CREATE_VALUE_ABSTRACT_ATTRIBUTE_FOR_POSITION(AANoCapture) |
13058 | CREATE_VALUE_ABSTRACT_ATTRIBUTE_FOR_POSITION(AAValueConstantRange) |
13059 | CREATE_VALUE_ABSTRACT_ATTRIBUTE_FOR_POSITION(AAPotentialConstantValues) |
13060 | CREATE_VALUE_ABSTRACT_ATTRIBUTE_FOR_POSITION(AAPotentialValues) |
13061 | CREATE_VALUE_ABSTRACT_ATTRIBUTE_FOR_POSITION(AANoUndef) |
13062 | CREATE_VALUE_ABSTRACT_ATTRIBUTE_FOR_POSITION(AANoFPClass) |
13063 | CREATE_VALUE_ABSTRACT_ATTRIBUTE_FOR_POSITION(AAPointerInfo) |
13064 | CREATE_VALUE_ABSTRACT_ATTRIBUTE_FOR_POSITION(AAAddressSpace) |
13065 | CREATE_VALUE_ABSTRACT_ATTRIBUTE_FOR_POSITION(AAAllocationInfo) |
13066 | |
13067 | CREATE_ALL_ABSTRACT_ATTRIBUTE_FOR_POSITION(AAValueSimplify) |
13068 | CREATE_ALL_ABSTRACT_ATTRIBUTE_FOR_POSITION(AAIsDead) |
13069 | CREATE_ALL_ABSTRACT_ATTRIBUTE_FOR_POSITION(AANoFree) |
13070 | CREATE_ALL_ABSTRACT_ATTRIBUTE_FOR_POSITION(AAUnderlyingObjects) |
13071 | |
13072 | CREATE_ABSTRACT_ATTRIBUTE_FOR_ONE_POSITION(IRP_CALL_SITE, CallSite, |
13073 | AAIndirectCallInfo) |
13074 | CREATE_ABSTRACT_ATTRIBUTE_FOR_ONE_POSITION(IRP_FLOAT, Floating, |
13075 | AAGlobalValueInfo) |
13076 | |
13077 | CREATE_FUNCTION_ONLY_ABSTRACT_ATTRIBUTE_FOR_POSITION(AAHeapToStack) |
13078 | CREATE_FUNCTION_ONLY_ABSTRACT_ATTRIBUTE_FOR_POSITION(AAUndefinedBehavior) |
13079 | CREATE_FUNCTION_ONLY_ABSTRACT_ATTRIBUTE_FOR_POSITION(AANonConvergent) |
13080 | CREATE_FUNCTION_ONLY_ABSTRACT_ATTRIBUTE_FOR_POSITION(AAIntraFnReachability) |
13081 | CREATE_FUNCTION_ONLY_ABSTRACT_ATTRIBUTE_FOR_POSITION(AAInterFnReachability) |
13082 | CREATE_FUNCTION_ONLY_ABSTRACT_ATTRIBUTE_FOR_POSITION(AADenormalFPMath) |
13083 | |
13084 | CREATE_NON_RET_ABSTRACT_ATTRIBUTE_FOR_POSITION(AAMemoryBehavior) |
13085 | |
13086 | #undef CREATE_FUNCTION_ONLY_ABSTRACT_ATTRIBUTE_FOR_POSITION |
13087 | #undef CREATE_FUNCTION_ABSTRACT_ATTRIBUTE_FOR_POSITION |
13088 | #undef CREATE_NON_RET_ABSTRACT_ATTRIBUTE_FOR_POSITION |
13089 | #undef CREATE_VALUE_ABSTRACT_ATTRIBUTE_FOR_POSITION |
13090 | #undef CREATE_ALL_ABSTRACT_ATTRIBUTE_FOR_POSITION |
13091 | #undef CREATE_ABSTRACT_ATTRIBUTE_FOR_ONE_POSITION |
13092 | #undef SWITCH_PK_CREATE |
13093 | #undef SWITCH_PK_INV |
13094 | |