1 | //===- llvm/Analysis/TargetTransformInfo.cpp ------------------------------===// |
2 | // |
3 | // Part of the LLVM Project, under the Apache License v2.0 with LLVM Exceptions. |
4 | // See https://llvm.org/LICENSE.txt for license information. |
5 | // SPDX-License-Identifier: Apache-2.0 WITH LLVM-exception |
6 | // |
7 | //===----------------------------------------------------------------------===// |
8 | |
9 | #include "llvm/Analysis/TargetTransformInfo.h" |
10 | #include "llvm/Analysis/CFG.h" |
11 | #include "llvm/Analysis/LoopIterator.h" |
12 | #include "llvm/Analysis/TargetLibraryInfo.h" |
13 | #include "llvm/Analysis/TargetTransformInfoImpl.h" |
14 | #include "llvm/IR/CFG.h" |
15 | #include "llvm/IR/Dominators.h" |
16 | #include "llvm/IR/Instruction.h" |
17 | #include "llvm/IR/Instructions.h" |
18 | #include "llvm/IR/IntrinsicInst.h" |
19 | #include "llvm/IR/Module.h" |
20 | #include "llvm/IR/Operator.h" |
21 | #include "llvm/IR/PatternMatch.h" |
22 | #include "llvm/InitializePasses.h" |
23 | #include "llvm/Support/CommandLine.h" |
24 | #include <optional> |
25 | #include <utility> |
26 | |
27 | using namespace llvm; |
28 | using namespace PatternMatch; |
29 | |
30 | #define DEBUG_TYPE "tti" |
31 | |
32 | static cl::opt<bool> EnableReduxCost("costmodel-reduxcost" , cl::init(Val: false), |
33 | cl::Hidden, |
34 | cl::desc("Recognize reduction patterns." )); |
35 | |
36 | static cl::opt<unsigned> CacheLineSize( |
37 | "cache-line-size" , cl::init(Val: 0), cl::Hidden, |
38 | cl::desc("Use this to override the target cache line size when " |
39 | "specified by the user." )); |
40 | |
41 | static cl::opt<unsigned> MinPageSize( |
42 | "min-page-size" , cl::init(Val: 0), cl::Hidden, |
43 | cl::desc("Use this to override the target's minimum page size." )); |
44 | |
45 | static cl::opt<unsigned> PredictableBranchThreshold( |
46 | "predictable-branch-threshold" , cl::init(Val: 99), cl::Hidden, |
47 | cl::desc( |
48 | "Use this to override the target's predictable branch threshold (%)." )); |
49 | |
50 | namespace { |
51 | /// No-op implementation of the TTI interface using the utility base |
52 | /// classes. |
53 | /// |
54 | /// This is used when no target specific information is available. |
55 | struct NoTTIImpl : TargetTransformInfoImplCRTPBase<NoTTIImpl> { |
56 | explicit NoTTIImpl(const DataLayout &DL) |
57 | : TargetTransformInfoImplCRTPBase<NoTTIImpl>(DL) {} |
58 | }; |
59 | } // namespace |
60 | |
61 | bool HardwareLoopInfo::canAnalyze(LoopInfo &LI) { |
62 | // If the loop has irreducible control flow, it can not be converted to |
63 | // Hardware loop. |
64 | LoopBlocksRPO RPOT(L); |
65 | RPOT.perform(LI: &LI); |
66 | if (containsIrreducibleCFG<const BasicBlock *>(RPOTraversal&: RPOT, LI)) |
67 | return false; |
68 | return true; |
69 | } |
70 | |
71 | IntrinsicCostAttributes::IntrinsicCostAttributes( |
72 | Intrinsic::ID Id, const CallBase &CI, InstructionCost ScalarizationCost, |
73 | bool TypeBasedOnly) |
74 | : II(dyn_cast<IntrinsicInst>(Val: &CI)), RetTy(CI.getType()), IID(Id), |
75 | ScalarizationCost(ScalarizationCost) { |
76 | |
77 | if (const auto *FPMO = dyn_cast<FPMathOperator>(Val: &CI)) |
78 | FMF = FPMO->getFastMathFlags(); |
79 | |
80 | if (!TypeBasedOnly) |
81 | Arguments.insert(I: Arguments.begin(), From: CI.arg_begin(), To: CI.arg_end()); |
82 | FunctionType *FTy = CI.getCalledFunction()->getFunctionType(); |
83 | ParamTys.insert(I: ParamTys.begin(), From: FTy->param_begin(), To: FTy->param_end()); |
84 | } |
85 | |
86 | IntrinsicCostAttributes::IntrinsicCostAttributes(Intrinsic::ID Id, Type *RTy, |
87 | ArrayRef<Type *> Tys, |
88 | FastMathFlags Flags, |
89 | const IntrinsicInst *I, |
90 | InstructionCost ScalarCost) |
91 | : II(I), RetTy(RTy), IID(Id), FMF(Flags), ScalarizationCost(ScalarCost) { |
92 | ParamTys.insert(I: ParamTys.begin(), From: Tys.begin(), To: Tys.end()); |
93 | } |
94 | |
95 | IntrinsicCostAttributes::IntrinsicCostAttributes(Intrinsic::ID Id, Type *Ty, |
96 | ArrayRef<const Value *> Args) |
97 | : RetTy(Ty), IID(Id) { |
98 | |
99 | Arguments.insert(I: Arguments.begin(), From: Args.begin(), To: Args.end()); |
100 | ParamTys.reserve(N: Arguments.size()); |
101 | for (unsigned Idx = 0, Size = Arguments.size(); Idx != Size; ++Idx) |
102 | ParamTys.push_back(Elt: Arguments[Idx]->getType()); |
103 | } |
104 | |
105 | IntrinsicCostAttributes::IntrinsicCostAttributes(Intrinsic::ID Id, Type *RTy, |
106 | ArrayRef<const Value *> Args, |
107 | ArrayRef<Type *> Tys, |
108 | FastMathFlags Flags, |
109 | const IntrinsicInst *I, |
110 | InstructionCost ScalarCost) |
111 | : II(I), RetTy(RTy), IID(Id), FMF(Flags), ScalarizationCost(ScalarCost) { |
112 | ParamTys.insert(I: ParamTys.begin(), From: Tys.begin(), To: Tys.end()); |
113 | Arguments.insert(I: Arguments.begin(), From: Args.begin(), To: Args.end()); |
114 | } |
115 | |
116 | HardwareLoopInfo::HardwareLoopInfo(Loop *L) : L(L) { |
117 | // Match default options: |
118 | // - hardware-loop-counter-bitwidth = 32 |
119 | // - hardware-loop-decrement = 1 |
120 | CountType = Type::getInt32Ty(C&: L->getHeader()->getContext()); |
121 | LoopDecrement = ConstantInt::get(Ty: CountType, V: 1); |
122 | } |
123 | |
124 | bool HardwareLoopInfo::isHardwareLoopCandidate(ScalarEvolution &SE, |
125 | LoopInfo &LI, DominatorTree &DT, |
126 | bool ForceNestedLoop, |
127 | bool ForceHardwareLoopPHI) { |
128 | SmallVector<BasicBlock *, 4> ExitingBlocks; |
129 | L->getExitingBlocks(ExitingBlocks); |
130 | |
131 | for (BasicBlock *BB : ExitingBlocks) { |
132 | // If we pass the updated counter back through a phi, we need to know |
133 | // which latch the updated value will be coming from. |
134 | if (!L->isLoopLatch(BB)) { |
135 | if (ForceHardwareLoopPHI || CounterInReg) |
136 | continue; |
137 | } |
138 | |
139 | const SCEV *EC = SE.getExitCount(L, ExitingBlock: BB); |
140 | if (isa<SCEVCouldNotCompute>(Val: EC)) |
141 | continue; |
142 | if (const SCEVConstant *ConstEC = dyn_cast<SCEVConstant>(Val: EC)) { |
143 | if (ConstEC->getValue()->isZero()) |
144 | continue; |
145 | } else if (!SE.isLoopInvariant(S: EC, L)) |
146 | continue; |
147 | |
148 | if (SE.getTypeSizeInBits(Ty: EC->getType()) > CountType->getBitWidth()) |
149 | continue; |
150 | |
151 | // If this exiting block is contained in a nested loop, it is not eligible |
152 | // for insertion of the branch-and-decrement since the inner loop would |
153 | // end up messing up the value in the CTR. |
154 | if (!IsNestingLegal && LI.getLoopFor(BB) != L && !ForceNestedLoop) |
155 | continue; |
156 | |
157 | // We now have a loop-invariant count of loop iterations (which is not the |
158 | // constant zero) for which we know that this loop will not exit via this |
159 | // existing block. |
160 | |
161 | // We need to make sure that this block will run on every loop iteration. |
162 | // For this to be true, we must dominate all blocks with backedges. Such |
163 | // blocks are in-loop predecessors to the header block. |
164 | bool NotAlways = false; |
165 | for (BasicBlock *Pred : predecessors(BB: L->getHeader())) { |
166 | if (!L->contains(BB: Pred)) |
167 | continue; |
168 | |
169 | if (!DT.dominates(A: BB, B: Pred)) { |
170 | NotAlways = true; |
171 | break; |
172 | } |
173 | } |
174 | |
175 | if (NotAlways) |
176 | continue; |
177 | |
178 | // Make sure this blocks ends with a conditional branch. |
179 | Instruction *TI = BB->getTerminator(); |
180 | if (!TI) |
181 | continue; |
182 | |
183 | if (BranchInst *BI = dyn_cast<BranchInst>(Val: TI)) { |
184 | if (!BI->isConditional()) |
185 | continue; |
186 | |
187 | ExitBranch = BI; |
188 | } else |
189 | continue; |
190 | |
191 | // Note that this block may not be the loop latch block, even if the loop |
192 | // has a latch block. |
193 | ExitBlock = BB; |
194 | ExitCount = EC; |
195 | break; |
196 | } |
197 | |
198 | if (!ExitBlock) |
199 | return false; |
200 | return true; |
201 | } |
202 | |
203 | TargetTransformInfo::TargetTransformInfo(const DataLayout &DL) |
204 | : TTIImpl(new Model<NoTTIImpl>(NoTTIImpl(DL))) {} |
205 | |
206 | TargetTransformInfo::~TargetTransformInfo() = default; |
207 | |
208 | TargetTransformInfo::TargetTransformInfo(TargetTransformInfo &&Arg) |
209 | : TTIImpl(std::move(Arg.TTIImpl)) {} |
210 | |
211 | TargetTransformInfo &TargetTransformInfo::operator=(TargetTransformInfo &&RHS) { |
212 | TTIImpl = std::move(RHS.TTIImpl); |
213 | return *this; |
214 | } |
215 | |
216 | unsigned TargetTransformInfo::getInliningThresholdMultiplier() const { |
217 | return TTIImpl->getInliningThresholdMultiplier(); |
218 | } |
219 | |
220 | unsigned |
221 | TargetTransformInfo::getInliningCostBenefitAnalysisSavingsMultiplier() const { |
222 | return TTIImpl->getInliningCostBenefitAnalysisSavingsMultiplier(); |
223 | } |
224 | |
225 | unsigned |
226 | TargetTransformInfo::getInliningCostBenefitAnalysisProfitableMultiplier() |
227 | const { |
228 | return TTIImpl->getInliningCostBenefitAnalysisProfitableMultiplier(); |
229 | } |
230 | |
231 | unsigned |
232 | TargetTransformInfo::adjustInliningThreshold(const CallBase *CB) const { |
233 | return TTIImpl->adjustInliningThreshold(CB); |
234 | } |
235 | |
236 | unsigned TargetTransformInfo::getCallerAllocaCost(const CallBase *CB, |
237 | const AllocaInst *AI) const { |
238 | return TTIImpl->getCallerAllocaCost(CB, AI); |
239 | } |
240 | |
241 | int TargetTransformInfo::getInlinerVectorBonusPercent() const { |
242 | return TTIImpl->getInlinerVectorBonusPercent(); |
243 | } |
244 | |
245 | InstructionCost TargetTransformInfo::getGEPCost( |
246 | Type *PointeeType, const Value *Ptr, ArrayRef<const Value *> Operands, |
247 | Type *AccessType, TTI::TargetCostKind CostKind) const { |
248 | return TTIImpl->getGEPCost(PointeeType, Ptr, Operands, AccessType, CostKind); |
249 | } |
250 | |
251 | InstructionCost TargetTransformInfo::getPointersChainCost( |
252 | ArrayRef<const Value *> Ptrs, const Value *Base, |
253 | const TTI::PointersChainInfo &Info, Type *AccessTy, |
254 | TTI::TargetCostKind CostKind) const { |
255 | assert((Base || !Info.isSameBase()) && |
256 | "If pointers have same base address it has to be provided." ); |
257 | return TTIImpl->getPointersChainCost(Ptrs, Base, Info, AccessTy, CostKind); |
258 | } |
259 | |
260 | unsigned TargetTransformInfo::getEstimatedNumberOfCaseClusters( |
261 | const SwitchInst &SI, unsigned &JTSize, ProfileSummaryInfo *PSI, |
262 | BlockFrequencyInfo *BFI) const { |
263 | return TTIImpl->getEstimatedNumberOfCaseClusters(SI, JTSize, PSI, BFI); |
264 | } |
265 | |
266 | InstructionCost |
267 | TargetTransformInfo::getInstructionCost(const User *U, |
268 | ArrayRef<const Value *> Operands, |
269 | enum TargetCostKind CostKind) const { |
270 | InstructionCost Cost = TTIImpl->getInstructionCost(U, Operands, CostKind); |
271 | assert((CostKind == TTI::TCK_RecipThroughput || Cost >= 0) && |
272 | "TTI should not produce negative costs!" ); |
273 | return Cost; |
274 | } |
275 | |
276 | BranchProbability TargetTransformInfo::getPredictableBranchThreshold() const { |
277 | return PredictableBranchThreshold.getNumOccurrences() > 0 |
278 | ? BranchProbability(PredictableBranchThreshold, 100) |
279 | : TTIImpl->getPredictableBranchThreshold(); |
280 | } |
281 | |
282 | bool TargetTransformInfo::hasBranchDivergence(const Function *F) const { |
283 | return TTIImpl->hasBranchDivergence(F); |
284 | } |
285 | |
286 | bool TargetTransformInfo::isSourceOfDivergence(const Value *V) const { |
287 | return TTIImpl->isSourceOfDivergence(V); |
288 | } |
289 | |
290 | bool llvm::TargetTransformInfo::isAlwaysUniform(const Value *V) const { |
291 | return TTIImpl->isAlwaysUniform(V); |
292 | } |
293 | |
294 | bool llvm::TargetTransformInfo::isValidAddrSpaceCast(unsigned FromAS, |
295 | unsigned ToAS) const { |
296 | return TTIImpl->isValidAddrSpaceCast(FromAS, ToAS); |
297 | } |
298 | |
299 | bool llvm::TargetTransformInfo::addrspacesMayAlias(unsigned FromAS, |
300 | unsigned ToAS) const { |
301 | return TTIImpl->addrspacesMayAlias(AS0: FromAS, AS1: ToAS); |
302 | } |
303 | |
304 | unsigned TargetTransformInfo::getFlatAddressSpace() const { |
305 | return TTIImpl->getFlatAddressSpace(); |
306 | } |
307 | |
308 | bool TargetTransformInfo::collectFlatAddressOperands( |
309 | SmallVectorImpl<int> &OpIndexes, Intrinsic::ID IID) const { |
310 | return TTIImpl->collectFlatAddressOperands(OpIndexes, IID); |
311 | } |
312 | |
313 | bool TargetTransformInfo::isNoopAddrSpaceCast(unsigned FromAS, |
314 | unsigned ToAS) const { |
315 | return TTIImpl->isNoopAddrSpaceCast(FromAS, ToAS); |
316 | } |
317 | |
318 | bool TargetTransformInfo::canHaveNonUndefGlobalInitializerInAddressSpace( |
319 | unsigned AS) const { |
320 | return TTIImpl->canHaveNonUndefGlobalInitializerInAddressSpace(AS); |
321 | } |
322 | |
323 | unsigned TargetTransformInfo::getAssumedAddrSpace(const Value *V) const { |
324 | return TTIImpl->getAssumedAddrSpace(V); |
325 | } |
326 | |
327 | bool TargetTransformInfo::isSingleThreaded() const { |
328 | return TTIImpl->isSingleThreaded(); |
329 | } |
330 | |
331 | std::pair<const Value *, unsigned> |
332 | TargetTransformInfo::getPredicatedAddrSpace(const Value *V) const { |
333 | return TTIImpl->getPredicatedAddrSpace(V); |
334 | } |
335 | |
336 | Value *TargetTransformInfo::rewriteIntrinsicWithAddressSpace( |
337 | IntrinsicInst *II, Value *OldV, Value *NewV) const { |
338 | return TTIImpl->rewriteIntrinsicWithAddressSpace(II, OldV, NewV); |
339 | } |
340 | |
341 | bool TargetTransformInfo::isLoweredToCall(const Function *F) const { |
342 | return TTIImpl->isLoweredToCall(F); |
343 | } |
344 | |
345 | bool TargetTransformInfo::isHardwareLoopProfitable( |
346 | Loop *L, ScalarEvolution &SE, AssumptionCache &AC, |
347 | TargetLibraryInfo *LibInfo, HardwareLoopInfo &HWLoopInfo) const { |
348 | return TTIImpl->isHardwareLoopProfitable(L, SE, AC, LibInfo, HWLoopInfo); |
349 | } |
350 | |
351 | bool TargetTransformInfo::preferPredicateOverEpilogue( |
352 | TailFoldingInfo *TFI) const { |
353 | return TTIImpl->preferPredicateOverEpilogue(TFI); |
354 | } |
355 | |
356 | TailFoldingStyle TargetTransformInfo::getPreferredTailFoldingStyle( |
357 | bool IVUpdateMayOverflow) const { |
358 | return TTIImpl->getPreferredTailFoldingStyle(IVUpdateMayOverflow); |
359 | } |
360 | |
361 | std::optional<Instruction *> |
362 | TargetTransformInfo::instCombineIntrinsic(InstCombiner &IC, |
363 | IntrinsicInst &II) const { |
364 | return TTIImpl->instCombineIntrinsic(IC, II); |
365 | } |
366 | |
367 | std::optional<Value *> TargetTransformInfo::simplifyDemandedUseBitsIntrinsic( |
368 | InstCombiner &IC, IntrinsicInst &II, APInt DemandedMask, KnownBits &Known, |
369 | bool &KnownBitsComputed) const { |
370 | return TTIImpl->simplifyDemandedUseBitsIntrinsic(IC, II, DemandedMask, Known, |
371 | KnownBitsComputed); |
372 | } |
373 | |
374 | std::optional<Value *> TargetTransformInfo::simplifyDemandedVectorEltsIntrinsic( |
375 | InstCombiner &IC, IntrinsicInst &II, APInt DemandedElts, APInt &UndefElts, |
376 | APInt &UndefElts2, APInt &UndefElts3, |
377 | std::function<void(Instruction *, unsigned, APInt, APInt &)> |
378 | SimplifyAndSetOp) const { |
379 | return TTIImpl->simplifyDemandedVectorEltsIntrinsic( |
380 | IC, II, DemandedElts, UndefElts, UndefElts2, UndefElts3, |
381 | SimplifyAndSetOp); |
382 | } |
383 | |
384 | void TargetTransformInfo::( |
385 | Loop *L, ScalarEvolution &SE, UnrollingPreferences &UP, |
386 | OptimizationRemarkEmitter *ORE) const { |
387 | return TTIImpl->getUnrollingPreferences(L, SE, UP, ORE); |
388 | } |
389 | |
390 | void TargetTransformInfo::getPeelingPreferences(Loop *L, ScalarEvolution &SE, |
391 | PeelingPreferences &PP) const { |
392 | return TTIImpl->getPeelingPreferences(L, SE, PP); |
393 | } |
394 | |
395 | bool TargetTransformInfo::isLegalAddImmediate(int64_t Imm) const { |
396 | return TTIImpl->isLegalAddImmediate(Imm); |
397 | } |
398 | |
399 | bool TargetTransformInfo::isLegalAddScalableImmediate(int64_t Imm) const { |
400 | return TTIImpl->isLegalAddScalableImmediate(Imm); |
401 | } |
402 | |
403 | bool TargetTransformInfo::isLegalICmpImmediate(int64_t Imm) const { |
404 | return TTIImpl->isLegalICmpImmediate(Imm); |
405 | } |
406 | |
407 | bool TargetTransformInfo::isLegalAddressingMode(Type *Ty, GlobalValue *BaseGV, |
408 | int64_t BaseOffset, |
409 | bool HasBaseReg, int64_t Scale, |
410 | unsigned AddrSpace, |
411 | Instruction *I, |
412 | int64_t ScalableOffset) const { |
413 | return TTIImpl->isLegalAddressingMode(Ty, BaseGV, BaseOffset, HasBaseReg, |
414 | Scale, AddrSpace, I, ScalableOffset); |
415 | } |
416 | |
417 | bool TargetTransformInfo::isLSRCostLess(const LSRCost &C1, |
418 | const LSRCost &C2) const { |
419 | return TTIImpl->isLSRCostLess(C1, C2); |
420 | } |
421 | |
422 | bool TargetTransformInfo::isNumRegsMajorCostOfLSR() const { |
423 | return TTIImpl->isNumRegsMajorCostOfLSR(); |
424 | } |
425 | |
426 | bool TargetTransformInfo::shouldFoldTerminatingConditionAfterLSR() const { |
427 | return TTIImpl->shouldFoldTerminatingConditionAfterLSR(); |
428 | } |
429 | |
430 | bool TargetTransformInfo::isProfitableLSRChainElement(Instruction *I) const { |
431 | return TTIImpl->isProfitableLSRChainElement(I); |
432 | } |
433 | |
434 | bool TargetTransformInfo::canMacroFuseCmp() const { |
435 | return TTIImpl->canMacroFuseCmp(); |
436 | } |
437 | |
438 | bool TargetTransformInfo::canSaveCmp(Loop *L, BranchInst **BI, |
439 | ScalarEvolution *SE, LoopInfo *LI, |
440 | DominatorTree *DT, AssumptionCache *AC, |
441 | TargetLibraryInfo *LibInfo) const { |
442 | return TTIImpl->canSaveCmp(L, BI, SE, LI, DT, AC, LibInfo); |
443 | } |
444 | |
445 | TTI::AddressingModeKind |
446 | TargetTransformInfo::getPreferredAddressingMode(const Loop *L, |
447 | ScalarEvolution *SE) const { |
448 | return TTIImpl->getPreferredAddressingMode(L, SE); |
449 | } |
450 | |
451 | bool TargetTransformInfo::isLegalMaskedStore(Type *DataType, |
452 | Align Alignment) const { |
453 | return TTIImpl->isLegalMaskedStore(DataType, Alignment); |
454 | } |
455 | |
456 | bool TargetTransformInfo::isLegalMaskedLoad(Type *DataType, |
457 | Align Alignment) const { |
458 | return TTIImpl->isLegalMaskedLoad(DataType, Alignment); |
459 | } |
460 | |
461 | bool TargetTransformInfo::isLegalNTStore(Type *DataType, |
462 | Align Alignment) const { |
463 | return TTIImpl->isLegalNTStore(DataType, Alignment); |
464 | } |
465 | |
466 | bool TargetTransformInfo::isLegalNTLoad(Type *DataType, Align Alignment) const { |
467 | return TTIImpl->isLegalNTLoad(DataType, Alignment); |
468 | } |
469 | |
470 | bool TargetTransformInfo::isLegalBroadcastLoad(Type *ElementTy, |
471 | ElementCount NumElements) const { |
472 | return TTIImpl->isLegalBroadcastLoad(ElementTy, NumElements); |
473 | } |
474 | |
475 | bool TargetTransformInfo::isLegalMaskedGather(Type *DataType, |
476 | Align Alignment) const { |
477 | return TTIImpl->isLegalMaskedGather(DataType, Alignment); |
478 | } |
479 | |
480 | bool TargetTransformInfo::isLegalAltInstr( |
481 | VectorType *VecTy, unsigned Opcode0, unsigned Opcode1, |
482 | const SmallBitVector &OpcodeMask) const { |
483 | return TTIImpl->isLegalAltInstr(VecTy, Opcode0, Opcode1, OpcodeMask); |
484 | } |
485 | |
486 | bool TargetTransformInfo::isLegalMaskedScatter(Type *DataType, |
487 | Align Alignment) const { |
488 | return TTIImpl->isLegalMaskedScatter(DataType, Alignment); |
489 | } |
490 | |
491 | bool TargetTransformInfo::forceScalarizeMaskedGather(VectorType *DataType, |
492 | Align Alignment) const { |
493 | return TTIImpl->forceScalarizeMaskedGather(DataType, Alignment); |
494 | } |
495 | |
496 | bool TargetTransformInfo::forceScalarizeMaskedScatter(VectorType *DataType, |
497 | Align Alignment) const { |
498 | return TTIImpl->forceScalarizeMaskedScatter(DataType, Alignment); |
499 | } |
500 | |
501 | bool TargetTransformInfo::isLegalMaskedCompressStore(Type *DataType, |
502 | Align Alignment) const { |
503 | return TTIImpl->isLegalMaskedCompressStore(DataType, Alignment); |
504 | } |
505 | |
506 | bool TargetTransformInfo::isLegalMaskedExpandLoad(Type *DataType, |
507 | Align Alignment) const { |
508 | return TTIImpl->isLegalMaskedExpandLoad(DataType, Alignment); |
509 | } |
510 | |
511 | bool TargetTransformInfo::isLegalStridedLoadStore(Type *DataType, |
512 | Align Alignment) const { |
513 | return TTIImpl->isLegalStridedLoadStore(DataType, Alignment); |
514 | } |
515 | |
516 | bool TargetTransformInfo::enableOrderedReductions() const { |
517 | return TTIImpl->enableOrderedReductions(); |
518 | } |
519 | |
520 | bool TargetTransformInfo::hasDivRemOp(Type *DataType, bool IsSigned) const { |
521 | return TTIImpl->hasDivRemOp(DataType, IsSigned); |
522 | } |
523 | |
524 | bool TargetTransformInfo::hasVolatileVariant(Instruction *I, |
525 | unsigned AddrSpace) const { |
526 | return TTIImpl->hasVolatileVariant(I, AddrSpace); |
527 | } |
528 | |
529 | bool TargetTransformInfo::prefersVectorizedAddressing() const { |
530 | return TTIImpl->prefersVectorizedAddressing(); |
531 | } |
532 | |
533 | InstructionCost TargetTransformInfo::getScalingFactorCost( |
534 | Type *Ty, GlobalValue *BaseGV, int64_t BaseOffset, bool HasBaseReg, |
535 | int64_t Scale, unsigned AddrSpace) const { |
536 | InstructionCost Cost = TTIImpl->getScalingFactorCost( |
537 | Ty, BaseGV, BaseOffset, HasBaseReg, Scale, AddrSpace); |
538 | assert(Cost >= 0 && "TTI should not produce negative costs!" ); |
539 | return Cost; |
540 | } |
541 | |
542 | bool TargetTransformInfo::LSRWithInstrQueries() const { |
543 | return TTIImpl->LSRWithInstrQueries(); |
544 | } |
545 | |
546 | bool TargetTransformInfo::isTruncateFree(Type *Ty1, Type *Ty2) const { |
547 | return TTIImpl->isTruncateFree(Ty1, Ty2); |
548 | } |
549 | |
550 | bool TargetTransformInfo::isProfitableToHoist(Instruction *I) const { |
551 | return TTIImpl->isProfitableToHoist(I); |
552 | } |
553 | |
554 | bool TargetTransformInfo::useAA() const { return TTIImpl->useAA(); } |
555 | |
556 | bool TargetTransformInfo::isTypeLegal(Type *Ty) const { |
557 | return TTIImpl->isTypeLegal(Ty); |
558 | } |
559 | |
560 | unsigned TargetTransformInfo::getRegUsageForType(Type *Ty) const { |
561 | return TTIImpl->getRegUsageForType(Ty); |
562 | } |
563 | |
564 | bool TargetTransformInfo::shouldBuildLookupTables() const { |
565 | return TTIImpl->shouldBuildLookupTables(); |
566 | } |
567 | |
568 | bool TargetTransformInfo::shouldBuildLookupTablesForConstant( |
569 | Constant *C) const { |
570 | return TTIImpl->shouldBuildLookupTablesForConstant(C); |
571 | } |
572 | |
573 | bool TargetTransformInfo::shouldBuildRelLookupTables() const { |
574 | return TTIImpl->shouldBuildRelLookupTables(); |
575 | } |
576 | |
577 | bool TargetTransformInfo::useColdCCForColdCall(Function &F) const { |
578 | return TTIImpl->useColdCCForColdCall(F); |
579 | } |
580 | |
581 | InstructionCost TargetTransformInfo::getScalarizationOverhead( |
582 | VectorType *Ty, const APInt &DemandedElts, bool Insert, bool , |
583 | TTI::TargetCostKind CostKind) const { |
584 | return TTIImpl->getScalarizationOverhead(Ty, DemandedElts, Insert, Extract, |
585 | CostKind); |
586 | } |
587 | |
588 | InstructionCost TargetTransformInfo::getOperandsScalarizationOverhead( |
589 | ArrayRef<const Value *> Args, ArrayRef<Type *> Tys, |
590 | TTI::TargetCostKind CostKind) const { |
591 | return TTIImpl->getOperandsScalarizationOverhead(Args, Tys, CostKind); |
592 | } |
593 | |
594 | bool TargetTransformInfo::supportsEfficientVectorElementLoadStore() const { |
595 | return TTIImpl->supportsEfficientVectorElementLoadStore(); |
596 | } |
597 | |
598 | bool TargetTransformInfo::supportsTailCalls() const { |
599 | return TTIImpl->supportsTailCalls(); |
600 | } |
601 | |
602 | bool TargetTransformInfo::supportsTailCallFor(const CallBase *CB) const { |
603 | return TTIImpl->supportsTailCallFor(CB); |
604 | } |
605 | |
606 | bool TargetTransformInfo::enableAggressiveInterleaving( |
607 | bool LoopHasReductions) const { |
608 | return TTIImpl->enableAggressiveInterleaving(LoopHasReductions); |
609 | } |
610 | |
611 | TargetTransformInfo::MemCmpExpansionOptions |
612 | TargetTransformInfo::enableMemCmpExpansion(bool OptSize, bool IsZeroCmp) const { |
613 | return TTIImpl->enableMemCmpExpansion(OptSize, IsZeroCmp); |
614 | } |
615 | |
616 | bool TargetTransformInfo::enableSelectOptimize() const { |
617 | return TTIImpl->enableSelectOptimize(); |
618 | } |
619 | |
620 | bool TargetTransformInfo::shouldTreatInstructionLikeSelect( |
621 | const Instruction *I) const { |
622 | return TTIImpl->shouldTreatInstructionLikeSelect(I); |
623 | } |
624 | |
625 | bool TargetTransformInfo::enableInterleavedAccessVectorization() const { |
626 | return TTIImpl->enableInterleavedAccessVectorization(); |
627 | } |
628 | |
629 | bool TargetTransformInfo::enableMaskedInterleavedAccessVectorization() const { |
630 | return TTIImpl->enableMaskedInterleavedAccessVectorization(); |
631 | } |
632 | |
633 | bool TargetTransformInfo::isFPVectorizationPotentiallyUnsafe() const { |
634 | return TTIImpl->isFPVectorizationPotentiallyUnsafe(); |
635 | } |
636 | |
637 | bool |
638 | TargetTransformInfo::allowsMisalignedMemoryAccesses(LLVMContext &Context, |
639 | unsigned BitWidth, |
640 | unsigned AddressSpace, |
641 | Align Alignment, |
642 | unsigned *Fast) const { |
643 | return TTIImpl->allowsMisalignedMemoryAccesses(Context, BitWidth, |
644 | AddressSpace, Alignment, Fast); |
645 | } |
646 | |
647 | TargetTransformInfo::PopcntSupportKind |
648 | TargetTransformInfo::getPopcntSupport(unsigned IntTyWidthInBit) const { |
649 | return TTIImpl->getPopcntSupport(IntTyWidthInBit); |
650 | } |
651 | |
652 | bool TargetTransformInfo::haveFastSqrt(Type *Ty) const { |
653 | return TTIImpl->haveFastSqrt(Ty); |
654 | } |
655 | |
656 | bool TargetTransformInfo::isExpensiveToSpeculativelyExecute( |
657 | const Instruction *I) const { |
658 | return TTIImpl->isExpensiveToSpeculativelyExecute(I); |
659 | } |
660 | |
661 | bool TargetTransformInfo::isFCmpOrdCheaperThanFCmpZero(Type *Ty) const { |
662 | return TTIImpl->isFCmpOrdCheaperThanFCmpZero(Ty); |
663 | } |
664 | |
665 | InstructionCost TargetTransformInfo::getFPOpCost(Type *Ty) const { |
666 | InstructionCost Cost = TTIImpl->getFPOpCost(Ty); |
667 | assert(Cost >= 0 && "TTI should not produce negative costs!" ); |
668 | return Cost; |
669 | } |
670 | |
671 | InstructionCost TargetTransformInfo::getIntImmCodeSizeCost(unsigned Opcode, |
672 | unsigned Idx, |
673 | const APInt &Imm, |
674 | Type *Ty) const { |
675 | InstructionCost Cost = TTIImpl->getIntImmCodeSizeCost(Opc: Opcode, Idx, Imm, Ty); |
676 | assert(Cost >= 0 && "TTI should not produce negative costs!" ); |
677 | return Cost; |
678 | } |
679 | |
680 | InstructionCost |
681 | TargetTransformInfo::getIntImmCost(const APInt &Imm, Type *Ty, |
682 | TTI::TargetCostKind CostKind) const { |
683 | InstructionCost Cost = TTIImpl->getIntImmCost(Imm, Ty, CostKind); |
684 | assert(Cost >= 0 && "TTI should not produce negative costs!" ); |
685 | return Cost; |
686 | } |
687 | |
688 | InstructionCost TargetTransformInfo::getIntImmCostInst( |
689 | unsigned Opcode, unsigned Idx, const APInt &Imm, Type *Ty, |
690 | TTI::TargetCostKind CostKind, Instruction *Inst) const { |
691 | InstructionCost Cost = |
692 | TTIImpl->getIntImmCostInst(Opc: Opcode, Idx, Imm, Ty, CostKind, Inst); |
693 | assert(Cost >= 0 && "TTI should not produce negative costs!" ); |
694 | return Cost; |
695 | } |
696 | |
697 | InstructionCost |
698 | TargetTransformInfo::getIntImmCostIntrin(Intrinsic::ID IID, unsigned Idx, |
699 | const APInt &Imm, Type *Ty, |
700 | TTI::TargetCostKind CostKind) const { |
701 | InstructionCost Cost = |
702 | TTIImpl->getIntImmCostIntrin(IID, Idx, Imm, Ty, CostKind); |
703 | assert(Cost >= 0 && "TTI should not produce negative costs!" ); |
704 | return Cost; |
705 | } |
706 | |
707 | bool TargetTransformInfo::preferToKeepConstantsAttached( |
708 | const Instruction &Inst, const Function &Fn) const { |
709 | return TTIImpl->preferToKeepConstantsAttached(Inst, Fn); |
710 | } |
711 | |
712 | unsigned TargetTransformInfo::getNumberOfRegisters(unsigned ClassID) const { |
713 | return TTIImpl->getNumberOfRegisters(ClassID); |
714 | } |
715 | |
716 | unsigned TargetTransformInfo::getRegisterClassForType(bool Vector, |
717 | Type *Ty) const { |
718 | return TTIImpl->getRegisterClassForType(Vector, Ty); |
719 | } |
720 | |
721 | const char *TargetTransformInfo::getRegisterClassName(unsigned ClassID) const { |
722 | return TTIImpl->getRegisterClassName(ClassID); |
723 | } |
724 | |
725 | TypeSize TargetTransformInfo::getRegisterBitWidth( |
726 | TargetTransformInfo::RegisterKind K) const { |
727 | return TTIImpl->getRegisterBitWidth(K); |
728 | } |
729 | |
730 | unsigned TargetTransformInfo::getMinVectorRegisterBitWidth() const { |
731 | return TTIImpl->getMinVectorRegisterBitWidth(); |
732 | } |
733 | |
734 | std::optional<unsigned> TargetTransformInfo::getMaxVScale() const { |
735 | return TTIImpl->getMaxVScale(); |
736 | } |
737 | |
738 | std::optional<unsigned> TargetTransformInfo::getVScaleForTuning() const { |
739 | return TTIImpl->getVScaleForTuning(); |
740 | } |
741 | |
742 | bool TargetTransformInfo::isVScaleKnownToBeAPowerOfTwo() const { |
743 | return TTIImpl->isVScaleKnownToBeAPowerOfTwo(); |
744 | } |
745 | |
746 | bool TargetTransformInfo::shouldMaximizeVectorBandwidth( |
747 | TargetTransformInfo::RegisterKind K) const { |
748 | return TTIImpl->shouldMaximizeVectorBandwidth(K); |
749 | } |
750 | |
751 | ElementCount TargetTransformInfo::getMinimumVF(unsigned ElemWidth, |
752 | bool IsScalable) const { |
753 | return TTIImpl->getMinimumVF(ElemWidth, IsScalable); |
754 | } |
755 | |
756 | unsigned TargetTransformInfo::getMaximumVF(unsigned ElemWidth, |
757 | unsigned Opcode) const { |
758 | return TTIImpl->getMaximumVF(ElemWidth, Opcode); |
759 | } |
760 | |
761 | unsigned TargetTransformInfo::getStoreMinimumVF(unsigned VF, Type *ScalarMemTy, |
762 | Type *ScalarValTy) const { |
763 | return TTIImpl->getStoreMinimumVF(VF, ScalarMemTy, ScalarValTy); |
764 | } |
765 | |
766 | bool TargetTransformInfo::shouldConsiderAddressTypePromotion( |
767 | const Instruction &I, bool &) const { |
768 | return TTIImpl->shouldConsiderAddressTypePromotion( |
769 | I, AllowPromotionWithoutCommonHeader); |
770 | } |
771 | |
772 | unsigned TargetTransformInfo::getCacheLineSize() const { |
773 | return CacheLineSize.getNumOccurrences() > 0 ? CacheLineSize |
774 | : TTIImpl->getCacheLineSize(); |
775 | } |
776 | |
777 | std::optional<unsigned> |
778 | TargetTransformInfo::getCacheSize(CacheLevel Level) const { |
779 | return TTIImpl->getCacheSize(Level); |
780 | } |
781 | |
782 | std::optional<unsigned> |
783 | TargetTransformInfo::getCacheAssociativity(CacheLevel Level) const { |
784 | return TTIImpl->getCacheAssociativity(Level); |
785 | } |
786 | |
787 | std::optional<unsigned> TargetTransformInfo::getMinPageSize() const { |
788 | return MinPageSize.getNumOccurrences() > 0 ? MinPageSize |
789 | : TTIImpl->getMinPageSize(); |
790 | } |
791 | |
792 | unsigned TargetTransformInfo::getPrefetchDistance() const { |
793 | return TTIImpl->getPrefetchDistance(); |
794 | } |
795 | |
796 | unsigned TargetTransformInfo::getMinPrefetchStride( |
797 | unsigned NumMemAccesses, unsigned NumStridedMemAccesses, |
798 | unsigned NumPrefetches, bool HasCall) const { |
799 | return TTIImpl->getMinPrefetchStride(NumMemAccesses, NumStridedMemAccesses, |
800 | NumPrefetches, HasCall); |
801 | } |
802 | |
803 | unsigned TargetTransformInfo::getMaxPrefetchIterationsAhead() const { |
804 | return TTIImpl->getMaxPrefetchIterationsAhead(); |
805 | } |
806 | |
807 | bool TargetTransformInfo::enableWritePrefetching() const { |
808 | return TTIImpl->enableWritePrefetching(); |
809 | } |
810 | |
811 | bool TargetTransformInfo::shouldPrefetchAddressSpace(unsigned AS) const { |
812 | return TTIImpl->shouldPrefetchAddressSpace(AS); |
813 | } |
814 | |
815 | unsigned TargetTransformInfo::getMaxInterleaveFactor(ElementCount VF) const { |
816 | return TTIImpl->getMaxInterleaveFactor(VF); |
817 | } |
818 | |
819 | TargetTransformInfo::OperandValueInfo |
820 | TargetTransformInfo::getOperandInfo(const Value *V) { |
821 | OperandValueKind OpInfo = OK_AnyValue; |
822 | OperandValueProperties OpProps = OP_None; |
823 | |
824 | if (isa<ConstantInt>(Val: V) || isa<ConstantFP>(Val: V)) { |
825 | if (const auto *CI = dyn_cast<ConstantInt>(Val: V)) { |
826 | if (CI->getValue().isPowerOf2()) |
827 | OpProps = OP_PowerOf2; |
828 | else if (CI->getValue().isNegatedPowerOf2()) |
829 | OpProps = OP_NegatedPowerOf2; |
830 | } |
831 | return {.Kind: OK_UniformConstantValue, .Properties: OpProps}; |
832 | } |
833 | |
834 | // A broadcast shuffle creates a uniform value. |
835 | // TODO: Add support for non-zero index broadcasts. |
836 | // TODO: Add support for different source vector width. |
837 | if (const auto *ShuffleInst = dyn_cast<ShuffleVectorInst>(Val: V)) |
838 | if (ShuffleInst->isZeroEltSplat()) |
839 | OpInfo = OK_UniformValue; |
840 | |
841 | const Value *Splat = getSplatValue(V); |
842 | |
843 | // Check for a splat of a constant or for a non uniform vector of constants |
844 | // and check if the constant(s) are all powers of two. |
845 | if (isa<ConstantVector>(Val: V) || isa<ConstantDataVector>(Val: V)) { |
846 | OpInfo = OK_NonUniformConstantValue; |
847 | if (Splat) { |
848 | OpInfo = OK_UniformConstantValue; |
849 | if (auto *CI = dyn_cast<ConstantInt>(Val: Splat)) { |
850 | if (CI->getValue().isPowerOf2()) |
851 | OpProps = OP_PowerOf2; |
852 | else if (CI->getValue().isNegatedPowerOf2()) |
853 | OpProps = OP_NegatedPowerOf2; |
854 | } |
855 | } else if (const auto *CDS = dyn_cast<ConstantDataSequential>(Val: V)) { |
856 | bool AllPow2 = true, AllNegPow2 = true; |
857 | for (unsigned I = 0, E = CDS->getNumElements(); I != E; ++I) { |
858 | if (auto *CI = dyn_cast<ConstantInt>(Val: CDS->getElementAsConstant(i: I))) { |
859 | AllPow2 &= CI->getValue().isPowerOf2(); |
860 | AllNegPow2 &= CI->getValue().isNegatedPowerOf2(); |
861 | if (AllPow2 || AllNegPow2) |
862 | continue; |
863 | } |
864 | AllPow2 = AllNegPow2 = false; |
865 | break; |
866 | } |
867 | OpProps = AllPow2 ? OP_PowerOf2 : OpProps; |
868 | OpProps = AllNegPow2 ? OP_NegatedPowerOf2 : OpProps; |
869 | } |
870 | } |
871 | |
872 | // Check for a splat of a uniform value. This is not loop aware, so return |
873 | // true only for the obviously uniform cases (argument, globalvalue) |
874 | if (Splat && (isa<Argument>(Val: Splat) || isa<GlobalValue>(Val: Splat))) |
875 | OpInfo = OK_UniformValue; |
876 | |
877 | return {.Kind: OpInfo, .Properties: OpProps}; |
878 | } |
879 | |
880 | InstructionCost TargetTransformInfo::getArithmeticInstrCost( |
881 | unsigned Opcode, Type *Ty, TTI::TargetCostKind CostKind, |
882 | OperandValueInfo Op1Info, OperandValueInfo Op2Info, |
883 | ArrayRef<const Value *> Args, const Instruction *CxtI, |
884 | const TargetLibraryInfo *TLibInfo) const { |
885 | |
886 | // Use call cost for frem intructions that have platform specific vector math |
887 | // functions, as those will be replaced with calls later by SelectionDAG or |
888 | // ReplaceWithVecLib pass. |
889 | if (TLibInfo && Opcode == Instruction::FRem) { |
890 | VectorType *VecTy = dyn_cast<VectorType>(Val: Ty); |
891 | LibFunc Func; |
892 | if (VecTy && |
893 | TLibInfo->getLibFunc(Opcode: Instruction::FRem, Ty: Ty->getScalarType(), F&: Func) && |
894 | TLibInfo->isFunctionVectorizable(F: TLibInfo->getName(F: Func), |
895 | VF: VecTy->getElementCount())) |
896 | return getCallInstrCost(F: nullptr, RetTy: VecTy, Tys: {VecTy, VecTy}, CostKind); |
897 | } |
898 | |
899 | InstructionCost Cost = |
900 | TTIImpl->getArithmeticInstrCost(Opcode, Ty, CostKind, |
901 | Opd1Info: Op1Info, Opd2Info: Op2Info, |
902 | Args, CxtI); |
903 | assert(Cost >= 0 && "TTI should not produce negative costs!" ); |
904 | return Cost; |
905 | } |
906 | |
907 | InstructionCost TargetTransformInfo::getAltInstrCost( |
908 | VectorType *VecTy, unsigned Opcode0, unsigned Opcode1, |
909 | const SmallBitVector &OpcodeMask, TTI::TargetCostKind CostKind) const { |
910 | InstructionCost Cost = |
911 | TTIImpl->getAltInstrCost(VecTy, Opcode0, Opcode1, OpcodeMask, CostKind); |
912 | assert(Cost >= 0 && "TTI should not produce negative costs!" ); |
913 | return Cost; |
914 | } |
915 | |
916 | InstructionCost TargetTransformInfo::getShuffleCost( |
917 | ShuffleKind Kind, VectorType *Ty, ArrayRef<int> Mask, |
918 | TTI::TargetCostKind CostKind, int Index, VectorType *SubTp, |
919 | ArrayRef<const Value *> Args, const Instruction *CxtI) const { |
920 | InstructionCost Cost = TTIImpl->getShuffleCost(Kind, Tp: Ty, Mask, CostKind, |
921 | Index, SubTp, Args, CxtI); |
922 | assert(Cost >= 0 && "TTI should not produce negative costs!" ); |
923 | return Cost; |
924 | } |
925 | |
926 | TTI::CastContextHint |
927 | TargetTransformInfo::getCastContextHint(const Instruction *I) { |
928 | if (!I) |
929 | return CastContextHint::None; |
930 | |
931 | auto getLoadStoreKind = [](const Value *V, unsigned LdStOp, unsigned MaskedOp, |
932 | unsigned GatScatOp) { |
933 | const Instruction *I = dyn_cast<Instruction>(Val: V); |
934 | if (!I) |
935 | return CastContextHint::None; |
936 | |
937 | if (I->getOpcode() == LdStOp) |
938 | return CastContextHint::Normal; |
939 | |
940 | if (const IntrinsicInst *II = dyn_cast<IntrinsicInst>(Val: I)) { |
941 | if (II->getIntrinsicID() == MaskedOp) |
942 | return TTI::CastContextHint::Masked; |
943 | if (II->getIntrinsicID() == GatScatOp) |
944 | return TTI::CastContextHint::GatherScatter; |
945 | } |
946 | |
947 | return TTI::CastContextHint::None; |
948 | }; |
949 | |
950 | switch (I->getOpcode()) { |
951 | case Instruction::ZExt: |
952 | case Instruction::SExt: |
953 | case Instruction::FPExt: |
954 | return getLoadStoreKind(I->getOperand(i: 0), Instruction::Load, |
955 | Intrinsic::masked_load, Intrinsic::masked_gather); |
956 | case Instruction::Trunc: |
957 | case Instruction::FPTrunc: |
958 | if (I->hasOneUse()) |
959 | return getLoadStoreKind(*I->user_begin(), Instruction::Store, |
960 | Intrinsic::masked_store, |
961 | Intrinsic::masked_scatter); |
962 | break; |
963 | default: |
964 | return CastContextHint::None; |
965 | } |
966 | |
967 | return TTI::CastContextHint::None; |
968 | } |
969 | |
970 | InstructionCost TargetTransformInfo::getCastInstrCost( |
971 | unsigned Opcode, Type *Dst, Type *Src, CastContextHint CCH, |
972 | TTI::TargetCostKind CostKind, const Instruction *I) const { |
973 | assert((I == nullptr || I->getOpcode() == Opcode) && |
974 | "Opcode should reflect passed instruction." ); |
975 | InstructionCost Cost = |
976 | TTIImpl->getCastInstrCost(Opcode, Dst, Src, CCH, CostKind, I); |
977 | assert(Cost >= 0 && "TTI should not produce negative costs!" ); |
978 | return Cost; |
979 | } |
980 | |
981 | InstructionCost TargetTransformInfo::( |
982 | unsigned Opcode, Type *Dst, VectorType *VecTy, unsigned Index) const { |
983 | InstructionCost Cost = |
984 | TTIImpl->getExtractWithExtendCost(Opcode, Dst, VecTy, Index); |
985 | assert(Cost >= 0 && "TTI should not produce negative costs!" ); |
986 | return Cost; |
987 | } |
988 | |
989 | InstructionCost TargetTransformInfo::getCFInstrCost( |
990 | unsigned Opcode, TTI::TargetCostKind CostKind, const Instruction *I) const { |
991 | assert((I == nullptr || I->getOpcode() == Opcode) && |
992 | "Opcode should reflect passed instruction." ); |
993 | InstructionCost Cost = TTIImpl->getCFInstrCost(Opcode, CostKind, I); |
994 | assert(Cost >= 0 && "TTI should not produce negative costs!" ); |
995 | return Cost; |
996 | } |
997 | |
998 | InstructionCost TargetTransformInfo::getCmpSelInstrCost( |
999 | unsigned Opcode, Type *ValTy, Type *CondTy, CmpInst::Predicate VecPred, |
1000 | TTI::TargetCostKind CostKind, const Instruction *I) const { |
1001 | assert((I == nullptr || I->getOpcode() == Opcode) && |
1002 | "Opcode should reflect passed instruction." ); |
1003 | InstructionCost Cost = |
1004 | TTIImpl->getCmpSelInstrCost(Opcode, ValTy, CondTy, VecPred, CostKind, I); |
1005 | assert(Cost >= 0 && "TTI should not produce negative costs!" ); |
1006 | return Cost; |
1007 | } |
1008 | |
1009 | InstructionCost TargetTransformInfo::getVectorInstrCost( |
1010 | unsigned Opcode, Type *Val, TTI::TargetCostKind CostKind, unsigned Index, |
1011 | Value *Op0, Value *Op1) const { |
1012 | // FIXME: Assert that Opcode is either InsertElement or ExtractElement. |
1013 | // This is mentioned in the interface description and respected by all |
1014 | // callers, but never asserted upon. |
1015 | InstructionCost Cost = |
1016 | TTIImpl->getVectorInstrCost(Opcode, Val, CostKind, Index, Op0, Op1); |
1017 | assert(Cost >= 0 && "TTI should not produce negative costs!" ); |
1018 | return Cost; |
1019 | } |
1020 | |
1021 | InstructionCost |
1022 | TargetTransformInfo::getVectorInstrCost(const Instruction &I, Type *Val, |
1023 | TTI::TargetCostKind CostKind, |
1024 | unsigned Index) const { |
1025 | // FIXME: Assert that Opcode is either InsertElement or ExtractElement. |
1026 | // This is mentioned in the interface description and respected by all |
1027 | // callers, but never asserted upon. |
1028 | InstructionCost Cost = TTIImpl->getVectorInstrCost(I, Val, CostKind, Index); |
1029 | assert(Cost >= 0 && "TTI should not produce negative costs!" ); |
1030 | return Cost; |
1031 | } |
1032 | |
1033 | InstructionCost TargetTransformInfo::getReplicationShuffleCost( |
1034 | Type *EltTy, int ReplicationFactor, int VF, const APInt &DemandedDstElts, |
1035 | TTI::TargetCostKind CostKind) { |
1036 | InstructionCost Cost = TTIImpl->getReplicationShuffleCost( |
1037 | EltTy, ReplicationFactor, VF, DemandedDstElts, CostKind); |
1038 | assert(Cost >= 0 && "TTI should not produce negative costs!" ); |
1039 | return Cost; |
1040 | } |
1041 | |
1042 | InstructionCost TargetTransformInfo::getMemoryOpCost( |
1043 | unsigned Opcode, Type *Src, Align Alignment, unsigned AddressSpace, |
1044 | TTI::TargetCostKind CostKind, TTI::OperandValueInfo OpInfo, |
1045 | const Instruction *I) const { |
1046 | assert((I == nullptr || I->getOpcode() == Opcode) && |
1047 | "Opcode should reflect passed instruction." ); |
1048 | InstructionCost Cost = TTIImpl->getMemoryOpCost( |
1049 | Opcode, Src, Alignment, AddressSpace, CostKind, OpInfo, I); |
1050 | assert(Cost >= 0 && "TTI should not produce negative costs!" ); |
1051 | return Cost; |
1052 | } |
1053 | |
1054 | InstructionCost TargetTransformInfo::getMaskedMemoryOpCost( |
1055 | unsigned Opcode, Type *Src, Align Alignment, unsigned AddressSpace, |
1056 | TTI::TargetCostKind CostKind) const { |
1057 | InstructionCost Cost = TTIImpl->getMaskedMemoryOpCost(Opcode, Src, Alignment, |
1058 | AddressSpace, CostKind); |
1059 | assert(Cost >= 0 && "TTI should not produce negative costs!" ); |
1060 | return Cost; |
1061 | } |
1062 | |
1063 | InstructionCost TargetTransformInfo::getGatherScatterOpCost( |
1064 | unsigned Opcode, Type *DataTy, const Value *Ptr, bool VariableMask, |
1065 | Align Alignment, TTI::TargetCostKind CostKind, const Instruction *I) const { |
1066 | InstructionCost Cost = TTIImpl->getGatherScatterOpCost( |
1067 | Opcode, DataTy, Ptr, VariableMask, Alignment, CostKind, I); |
1068 | assert((!Cost.isValid() || Cost >= 0) && |
1069 | "TTI should not produce negative costs!" ); |
1070 | return Cost; |
1071 | } |
1072 | |
1073 | InstructionCost TargetTransformInfo::getStridedMemoryOpCost( |
1074 | unsigned Opcode, Type *DataTy, const Value *Ptr, bool VariableMask, |
1075 | Align Alignment, TTI::TargetCostKind CostKind, const Instruction *I) const { |
1076 | InstructionCost Cost = TTIImpl->getStridedMemoryOpCost( |
1077 | Opcode, DataTy, Ptr, VariableMask, Alignment, CostKind, I); |
1078 | assert(Cost >= 0 && "TTI should not produce negative costs!" ); |
1079 | return Cost; |
1080 | } |
1081 | |
1082 | InstructionCost TargetTransformInfo::getInterleavedMemoryOpCost( |
1083 | unsigned Opcode, Type *VecTy, unsigned Factor, ArrayRef<unsigned> Indices, |
1084 | Align Alignment, unsigned AddressSpace, TTI::TargetCostKind CostKind, |
1085 | bool UseMaskForCond, bool UseMaskForGaps) const { |
1086 | InstructionCost Cost = TTIImpl->getInterleavedMemoryOpCost( |
1087 | Opcode, VecTy, Factor, Indices, Alignment, AddressSpace, CostKind, |
1088 | UseMaskForCond, UseMaskForGaps); |
1089 | assert(Cost >= 0 && "TTI should not produce negative costs!" ); |
1090 | return Cost; |
1091 | } |
1092 | |
1093 | InstructionCost |
1094 | TargetTransformInfo::getIntrinsicInstrCost(const IntrinsicCostAttributes &ICA, |
1095 | TTI::TargetCostKind CostKind) const { |
1096 | InstructionCost Cost = TTIImpl->getIntrinsicInstrCost(ICA, CostKind); |
1097 | assert(Cost >= 0 && "TTI should not produce negative costs!" ); |
1098 | return Cost; |
1099 | } |
1100 | |
1101 | InstructionCost |
1102 | TargetTransformInfo::getCallInstrCost(Function *F, Type *RetTy, |
1103 | ArrayRef<Type *> Tys, |
1104 | TTI::TargetCostKind CostKind) const { |
1105 | InstructionCost Cost = TTIImpl->getCallInstrCost(F, RetTy, Tys, CostKind); |
1106 | assert(Cost >= 0 && "TTI should not produce negative costs!" ); |
1107 | return Cost; |
1108 | } |
1109 | |
1110 | unsigned TargetTransformInfo::getNumberOfParts(Type *Tp) const { |
1111 | return TTIImpl->getNumberOfParts(Tp); |
1112 | } |
1113 | |
1114 | InstructionCost |
1115 | TargetTransformInfo::getAddressComputationCost(Type *Tp, ScalarEvolution *SE, |
1116 | const SCEV *Ptr) const { |
1117 | InstructionCost Cost = TTIImpl->getAddressComputationCost(Ty: Tp, SE, Ptr); |
1118 | assert(Cost >= 0 && "TTI should not produce negative costs!" ); |
1119 | return Cost; |
1120 | } |
1121 | |
1122 | InstructionCost TargetTransformInfo::getMemcpyCost(const Instruction *I) const { |
1123 | InstructionCost Cost = TTIImpl->getMemcpyCost(I); |
1124 | assert(Cost >= 0 && "TTI should not produce negative costs!" ); |
1125 | return Cost; |
1126 | } |
1127 | |
1128 | uint64_t TargetTransformInfo::getMaxMemIntrinsicInlineSizeThreshold() const { |
1129 | return TTIImpl->getMaxMemIntrinsicInlineSizeThreshold(); |
1130 | } |
1131 | |
1132 | InstructionCost TargetTransformInfo::getArithmeticReductionCost( |
1133 | unsigned Opcode, VectorType *Ty, std::optional<FastMathFlags> FMF, |
1134 | TTI::TargetCostKind CostKind) const { |
1135 | InstructionCost Cost = |
1136 | TTIImpl->getArithmeticReductionCost(Opcode, Ty, FMF, CostKind); |
1137 | assert(Cost >= 0 && "TTI should not produce negative costs!" ); |
1138 | return Cost; |
1139 | } |
1140 | |
1141 | InstructionCost TargetTransformInfo::getMinMaxReductionCost( |
1142 | Intrinsic::ID IID, VectorType *Ty, FastMathFlags FMF, |
1143 | TTI::TargetCostKind CostKind) const { |
1144 | InstructionCost Cost = |
1145 | TTIImpl->getMinMaxReductionCost(IID, Ty, FMF, CostKind); |
1146 | assert(Cost >= 0 && "TTI should not produce negative costs!" ); |
1147 | return Cost; |
1148 | } |
1149 | |
1150 | InstructionCost TargetTransformInfo::getExtendedReductionCost( |
1151 | unsigned Opcode, bool IsUnsigned, Type *ResTy, VectorType *Ty, |
1152 | FastMathFlags FMF, TTI::TargetCostKind CostKind) const { |
1153 | return TTIImpl->getExtendedReductionCost(Opcode, IsUnsigned, ResTy, Ty, FMF, |
1154 | CostKind); |
1155 | } |
1156 | |
1157 | InstructionCost TargetTransformInfo::getMulAccReductionCost( |
1158 | bool IsUnsigned, Type *ResTy, VectorType *Ty, |
1159 | TTI::TargetCostKind CostKind) const { |
1160 | return TTIImpl->getMulAccReductionCost(IsUnsigned, ResTy, Ty, CostKind); |
1161 | } |
1162 | |
1163 | InstructionCost |
1164 | TargetTransformInfo::getCostOfKeepingLiveOverCall(ArrayRef<Type *> Tys) const { |
1165 | return TTIImpl->getCostOfKeepingLiveOverCall(Tys); |
1166 | } |
1167 | |
1168 | bool TargetTransformInfo::getTgtMemIntrinsic(IntrinsicInst *Inst, |
1169 | MemIntrinsicInfo &Info) const { |
1170 | return TTIImpl->getTgtMemIntrinsic(Inst, Info); |
1171 | } |
1172 | |
1173 | unsigned TargetTransformInfo::getAtomicMemIntrinsicMaxElementSize() const { |
1174 | return TTIImpl->getAtomicMemIntrinsicMaxElementSize(); |
1175 | } |
1176 | |
1177 | Value *TargetTransformInfo::getOrCreateResultFromMemIntrinsic( |
1178 | IntrinsicInst *Inst, Type *ExpectedType) const { |
1179 | return TTIImpl->getOrCreateResultFromMemIntrinsic(Inst, ExpectedType); |
1180 | } |
1181 | |
1182 | Type *TargetTransformInfo::getMemcpyLoopLoweringType( |
1183 | LLVMContext &Context, Value *Length, unsigned SrcAddrSpace, |
1184 | unsigned DestAddrSpace, unsigned SrcAlign, unsigned DestAlign, |
1185 | std::optional<uint32_t> AtomicElementSize) const { |
1186 | return TTIImpl->getMemcpyLoopLoweringType(Context, Length, SrcAddrSpace, |
1187 | DestAddrSpace, SrcAlign, DestAlign, |
1188 | AtomicElementSize); |
1189 | } |
1190 | |
1191 | void TargetTransformInfo::getMemcpyLoopResidualLoweringType( |
1192 | SmallVectorImpl<Type *> &OpsOut, LLVMContext &Context, |
1193 | unsigned RemainingBytes, unsigned SrcAddrSpace, unsigned DestAddrSpace, |
1194 | unsigned SrcAlign, unsigned DestAlign, |
1195 | std::optional<uint32_t> AtomicCpySize) const { |
1196 | TTIImpl->getMemcpyLoopResidualLoweringType( |
1197 | OpsOut, Context, RemainingBytes, SrcAddrSpace, DestAddrSpace, SrcAlign, |
1198 | DestAlign, AtomicCpySize); |
1199 | } |
1200 | |
1201 | bool TargetTransformInfo::areInlineCompatible(const Function *Caller, |
1202 | const Function *Callee) const { |
1203 | return TTIImpl->areInlineCompatible(Caller, Callee); |
1204 | } |
1205 | |
1206 | unsigned |
1207 | TargetTransformInfo::getInlineCallPenalty(const Function *F, |
1208 | const CallBase &Call, |
1209 | unsigned DefaultCallPenalty) const { |
1210 | return TTIImpl->getInlineCallPenalty(F, Call, DefaultCallPenalty); |
1211 | } |
1212 | |
1213 | bool TargetTransformInfo::areTypesABICompatible( |
1214 | const Function *Caller, const Function *Callee, |
1215 | const ArrayRef<Type *> &Types) const { |
1216 | return TTIImpl->areTypesABICompatible(Caller, Callee, Types); |
1217 | } |
1218 | |
1219 | bool TargetTransformInfo::isIndexedLoadLegal(MemIndexedMode Mode, |
1220 | Type *Ty) const { |
1221 | return TTIImpl->isIndexedLoadLegal(Mode, Ty); |
1222 | } |
1223 | |
1224 | bool TargetTransformInfo::isIndexedStoreLegal(MemIndexedMode Mode, |
1225 | Type *Ty) const { |
1226 | return TTIImpl->isIndexedStoreLegal(Mode, Ty); |
1227 | } |
1228 | |
1229 | unsigned TargetTransformInfo::getLoadStoreVecRegBitWidth(unsigned AS) const { |
1230 | return TTIImpl->getLoadStoreVecRegBitWidth(AddrSpace: AS); |
1231 | } |
1232 | |
1233 | bool TargetTransformInfo::isLegalToVectorizeLoad(LoadInst *LI) const { |
1234 | return TTIImpl->isLegalToVectorizeLoad(LI); |
1235 | } |
1236 | |
1237 | bool TargetTransformInfo::isLegalToVectorizeStore(StoreInst *SI) const { |
1238 | return TTIImpl->isLegalToVectorizeStore(SI); |
1239 | } |
1240 | |
1241 | bool TargetTransformInfo::isLegalToVectorizeLoadChain( |
1242 | unsigned ChainSizeInBytes, Align Alignment, unsigned AddrSpace) const { |
1243 | return TTIImpl->isLegalToVectorizeLoadChain(ChainSizeInBytes, Alignment, |
1244 | AddrSpace); |
1245 | } |
1246 | |
1247 | bool TargetTransformInfo::isLegalToVectorizeStoreChain( |
1248 | unsigned ChainSizeInBytes, Align Alignment, unsigned AddrSpace) const { |
1249 | return TTIImpl->isLegalToVectorizeStoreChain(ChainSizeInBytes, Alignment, |
1250 | AddrSpace); |
1251 | } |
1252 | |
1253 | bool TargetTransformInfo::isLegalToVectorizeReduction( |
1254 | const RecurrenceDescriptor &RdxDesc, ElementCount VF) const { |
1255 | return TTIImpl->isLegalToVectorizeReduction(RdxDesc, VF); |
1256 | } |
1257 | |
1258 | bool TargetTransformInfo::isElementTypeLegalForScalableVector(Type *Ty) const { |
1259 | return TTIImpl->isElementTypeLegalForScalableVector(Ty); |
1260 | } |
1261 | |
1262 | unsigned TargetTransformInfo::getLoadVectorFactor(unsigned VF, |
1263 | unsigned LoadSize, |
1264 | unsigned ChainSizeInBytes, |
1265 | VectorType *VecTy) const { |
1266 | return TTIImpl->getLoadVectorFactor(VF, LoadSize, ChainSizeInBytes, VecTy); |
1267 | } |
1268 | |
1269 | unsigned TargetTransformInfo::getStoreVectorFactor(unsigned VF, |
1270 | unsigned StoreSize, |
1271 | unsigned ChainSizeInBytes, |
1272 | VectorType *VecTy) const { |
1273 | return TTIImpl->getStoreVectorFactor(VF, StoreSize, ChainSizeInBytes, VecTy); |
1274 | } |
1275 | |
1276 | bool TargetTransformInfo::preferInLoopReduction(unsigned Opcode, Type *Ty, |
1277 | ReductionFlags Flags) const { |
1278 | return TTIImpl->preferInLoopReduction(Opcode, Ty, Flags); |
1279 | } |
1280 | |
1281 | bool TargetTransformInfo::preferPredicatedReductionSelect( |
1282 | unsigned Opcode, Type *Ty, ReductionFlags Flags) const { |
1283 | return TTIImpl->preferPredicatedReductionSelect(Opcode, Ty, Flags); |
1284 | } |
1285 | |
1286 | bool TargetTransformInfo::preferEpilogueVectorization() const { |
1287 | return TTIImpl->preferEpilogueVectorization(); |
1288 | } |
1289 | |
1290 | TargetTransformInfo::VPLegalization |
1291 | TargetTransformInfo::getVPLegalizationStrategy(const VPIntrinsic &VPI) const { |
1292 | return TTIImpl->getVPLegalizationStrategy(PI: VPI); |
1293 | } |
1294 | |
1295 | bool TargetTransformInfo::hasArmWideBranch(bool Thumb) const { |
1296 | return TTIImpl->hasArmWideBranch(Thumb); |
1297 | } |
1298 | |
1299 | unsigned TargetTransformInfo::getMaxNumArgs() const { |
1300 | return TTIImpl->getMaxNumArgs(); |
1301 | } |
1302 | |
1303 | bool TargetTransformInfo::shouldExpandReduction(const IntrinsicInst *II) const { |
1304 | return TTIImpl->shouldExpandReduction(II); |
1305 | } |
1306 | |
1307 | unsigned TargetTransformInfo::getGISelRematGlobalCost() const { |
1308 | return TTIImpl->getGISelRematGlobalCost(); |
1309 | } |
1310 | |
1311 | unsigned TargetTransformInfo::getMinTripCountTailFoldingThreshold() const { |
1312 | return TTIImpl->getMinTripCountTailFoldingThreshold(); |
1313 | } |
1314 | |
1315 | bool TargetTransformInfo::supportsScalableVectors() const { |
1316 | return TTIImpl->supportsScalableVectors(); |
1317 | } |
1318 | |
1319 | bool TargetTransformInfo::enableScalableVectorization() const { |
1320 | return TTIImpl->enableScalableVectorization(); |
1321 | } |
1322 | |
1323 | bool TargetTransformInfo::hasActiveVectorLength(unsigned Opcode, Type *DataType, |
1324 | Align Alignment) const { |
1325 | return TTIImpl->hasActiveVectorLength(Opcode, DataType, Alignment); |
1326 | } |
1327 | |
1328 | TargetTransformInfo::Concept::~Concept() = default; |
1329 | |
1330 | TargetIRAnalysis::TargetIRAnalysis() : TTICallback(&getDefaultTTI) {} |
1331 | |
1332 | TargetIRAnalysis::TargetIRAnalysis( |
1333 | std::function<Result(const Function &)> TTICallback) |
1334 | : TTICallback(std::move(TTICallback)) {} |
1335 | |
1336 | TargetIRAnalysis::Result TargetIRAnalysis::run(const Function &F, |
1337 | FunctionAnalysisManager &) { |
1338 | return TTICallback(F); |
1339 | } |
1340 | |
1341 | AnalysisKey TargetIRAnalysis::Key; |
1342 | |
1343 | TargetIRAnalysis::Result TargetIRAnalysis::getDefaultTTI(const Function &F) { |
1344 | return Result(F.getParent()->getDataLayout()); |
1345 | } |
1346 | |
1347 | // Register the basic pass. |
1348 | INITIALIZE_PASS(TargetTransformInfoWrapperPass, "tti" , |
1349 | "Target Transform Information" , false, true) |
1350 | char TargetTransformInfoWrapperPass::ID = 0; |
1351 | |
1352 | void TargetTransformInfoWrapperPass::anchor() {} |
1353 | |
1354 | TargetTransformInfoWrapperPass::TargetTransformInfoWrapperPass() |
1355 | : ImmutablePass(ID) { |
1356 | initializeTargetTransformInfoWrapperPassPass( |
1357 | Registry&: *PassRegistry::getPassRegistry()); |
1358 | } |
1359 | |
1360 | TargetTransformInfoWrapperPass::TargetTransformInfoWrapperPass( |
1361 | TargetIRAnalysis TIRA) |
1362 | : ImmutablePass(ID), TIRA(std::move(TIRA)) { |
1363 | initializeTargetTransformInfoWrapperPassPass( |
1364 | Registry&: *PassRegistry::getPassRegistry()); |
1365 | } |
1366 | |
1367 | TargetTransformInfo &TargetTransformInfoWrapperPass::getTTI(const Function &F) { |
1368 | FunctionAnalysisManager DummyFAM; |
1369 | TTI = TIRA.run(F, DummyFAM); |
1370 | return *TTI; |
1371 | } |
1372 | |
1373 | ImmutablePass * |
1374 | llvm::createTargetTransformInfoWrapperPass(TargetIRAnalysis TIRA) { |
1375 | return new TargetTransformInfoWrapperPass(std::move(TIRA)); |
1376 | } |
1377 | |