1 | //===----------- VectorUtils.cpp - Vectorizer utility functions -----------===// |
2 | // |
3 | // Part of the LLVM Project, under the Apache License v2.0 with LLVM Exceptions. |
4 | // See https://llvm.org/LICENSE.txt for license information. |
5 | // SPDX-License-Identifier: Apache-2.0 WITH LLVM-exception |
6 | // |
7 | //===----------------------------------------------------------------------===// |
8 | // |
9 | // This file defines vectorizer utilities. |
10 | // |
11 | //===----------------------------------------------------------------------===// |
12 | |
13 | #include "llvm/Analysis/VectorUtils.h" |
14 | #include "llvm/ADT/EquivalenceClasses.h" |
15 | #include "llvm/ADT/SmallVector.h" |
16 | #include "llvm/Analysis/DemandedBits.h" |
17 | #include "llvm/Analysis/LoopInfo.h" |
18 | #include "llvm/Analysis/LoopIterator.h" |
19 | #include "llvm/Analysis/ScalarEvolution.h" |
20 | #include "llvm/Analysis/ScalarEvolutionExpressions.h" |
21 | #include "llvm/Analysis/TargetTransformInfo.h" |
22 | #include "llvm/Analysis/ValueTracking.h" |
23 | #include "llvm/IR/Constants.h" |
24 | #include "llvm/IR/DerivedTypes.h" |
25 | #include "llvm/IR/IRBuilder.h" |
26 | #include "llvm/IR/MemoryModelRelaxationAnnotations.h" |
27 | #include "llvm/IR/PatternMatch.h" |
28 | #include "llvm/IR/Value.h" |
29 | #include "llvm/Support/CommandLine.h" |
30 | |
31 | #define DEBUG_TYPE "vectorutils" |
32 | |
33 | using namespace llvm; |
34 | using namespace llvm::PatternMatch; |
35 | |
36 | /// Maximum factor for an interleaved memory access. |
37 | static cl::opt<unsigned> MaxInterleaveGroupFactor( |
38 | "max-interleave-group-factor" , cl::Hidden, |
39 | cl::desc("Maximum factor for an interleaved access group (default = 8)" ), |
40 | cl::init(Val: 8)); |
41 | |
42 | /// Return true if all of the intrinsic's arguments and return type are scalars |
43 | /// for the scalar form of the intrinsic, and vectors for the vector form of the |
44 | /// intrinsic (except operands that are marked as always being scalar by |
45 | /// isVectorIntrinsicWithScalarOpAtArg). |
46 | bool llvm::isTriviallyVectorizable(Intrinsic::ID ID) { |
47 | switch (ID) { |
48 | case Intrinsic::abs: // Begin integer bit-manipulation. |
49 | case Intrinsic::bswap: |
50 | case Intrinsic::bitreverse: |
51 | case Intrinsic::ctpop: |
52 | case Intrinsic::ctlz: |
53 | case Intrinsic::cttz: |
54 | case Intrinsic::fshl: |
55 | case Intrinsic::fshr: |
56 | case Intrinsic::smax: |
57 | case Intrinsic::smin: |
58 | case Intrinsic::umax: |
59 | case Intrinsic::umin: |
60 | case Intrinsic::sadd_sat: |
61 | case Intrinsic::ssub_sat: |
62 | case Intrinsic::uadd_sat: |
63 | case Intrinsic::usub_sat: |
64 | case Intrinsic::smul_fix: |
65 | case Intrinsic::smul_fix_sat: |
66 | case Intrinsic::umul_fix: |
67 | case Intrinsic::umul_fix_sat: |
68 | case Intrinsic::sqrt: // Begin floating-point. |
69 | case Intrinsic::sin: |
70 | case Intrinsic::cos: |
71 | case Intrinsic::exp: |
72 | case Intrinsic::exp2: |
73 | case Intrinsic::log: |
74 | case Intrinsic::log10: |
75 | case Intrinsic::log2: |
76 | case Intrinsic::fabs: |
77 | case Intrinsic::minnum: |
78 | case Intrinsic::maxnum: |
79 | case Intrinsic::minimum: |
80 | case Intrinsic::maximum: |
81 | case Intrinsic::copysign: |
82 | case Intrinsic::floor: |
83 | case Intrinsic::ceil: |
84 | case Intrinsic::trunc: |
85 | case Intrinsic::rint: |
86 | case Intrinsic::nearbyint: |
87 | case Intrinsic::round: |
88 | case Intrinsic::roundeven: |
89 | case Intrinsic::pow: |
90 | case Intrinsic::fma: |
91 | case Intrinsic::fmuladd: |
92 | case Intrinsic::is_fpclass: |
93 | case Intrinsic::powi: |
94 | case Intrinsic::canonicalize: |
95 | case Intrinsic::fptosi_sat: |
96 | case Intrinsic::fptoui_sat: |
97 | case Intrinsic::lrint: |
98 | case Intrinsic::llrint: |
99 | return true; |
100 | default: |
101 | return false; |
102 | } |
103 | } |
104 | |
105 | /// Identifies if the vector form of the intrinsic has a scalar operand. |
106 | bool llvm::isVectorIntrinsicWithScalarOpAtArg(Intrinsic::ID ID, |
107 | unsigned ScalarOpdIdx) { |
108 | switch (ID) { |
109 | case Intrinsic::abs: |
110 | case Intrinsic::ctlz: |
111 | case Intrinsic::cttz: |
112 | case Intrinsic::is_fpclass: |
113 | case Intrinsic::powi: |
114 | return (ScalarOpdIdx == 1); |
115 | case Intrinsic::smul_fix: |
116 | case Intrinsic::smul_fix_sat: |
117 | case Intrinsic::umul_fix: |
118 | case Intrinsic::umul_fix_sat: |
119 | return (ScalarOpdIdx == 2); |
120 | default: |
121 | return false; |
122 | } |
123 | } |
124 | |
125 | bool llvm::isVectorIntrinsicWithOverloadTypeAtArg(Intrinsic::ID ID, |
126 | int OpdIdx) { |
127 | assert(ID != Intrinsic::not_intrinsic && "Not an intrinsic!" ); |
128 | |
129 | switch (ID) { |
130 | case Intrinsic::fptosi_sat: |
131 | case Intrinsic::fptoui_sat: |
132 | case Intrinsic::lrint: |
133 | case Intrinsic::llrint: |
134 | return OpdIdx == -1 || OpdIdx == 0; |
135 | case Intrinsic::is_fpclass: |
136 | return OpdIdx == 0; |
137 | case Intrinsic::powi: |
138 | return OpdIdx == -1 || OpdIdx == 1; |
139 | default: |
140 | return OpdIdx == -1; |
141 | } |
142 | } |
143 | |
144 | /// Returns intrinsic ID for call. |
145 | /// For the input call instruction it finds mapping intrinsic and returns |
146 | /// its ID, in case it does not found it return not_intrinsic. |
147 | Intrinsic::ID llvm::getVectorIntrinsicIDForCall(const CallInst *CI, |
148 | const TargetLibraryInfo *TLI) { |
149 | Intrinsic::ID ID = getIntrinsicForCallSite(CB: *CI, TLI); |
150 | if (ID == Intrinsic::not_intrinsic) |
151 | return Intrinsic::not_intrinsic; |
152 | |
153 | if (isTriviallyVectorizable(ID) || ID == Intrinsic::lifetime_start || |
154 | ID == Intrinsic::lifetime_end || ID == Intrinsic::assume || |
155 | ID == Intrinsic::experimental_noalias_scope_decl || |
156 | ID == Intrinsic::sideeffect || ID == Intrinsic::pseudoprobe) |
157 | return ID; |
158 | return Intrinsic::not_intrinsic; |
159 | } |
160 | |
161 | /// Given a vector and an element number, see if the scalar value is |
162 | /// already around as a register, for example if it were inserted then extracted |
163 | /// from the vector. |
164 | Value *llvm::findScalarElement(Value *V, unsigned EltNo) { |
165 | assert(V->getType()->isVectorTy() && "Not looking at a vector?" ); |
166 | VectorType *VTy = cast<VectorType>(Val: V->getType()); |
167 | // For fixed-length vector, return undef for out of range access. |
168 | if (auto *FVTy = dyn_cast<FixedVectorType>(Val: VTy)) { |
169 | unsigned Width = FVTy->getNumElements(); |
170 | if (EltNo >= Width) |
171 | return UndefValue::get(T: FVTy->getElementType()); |
172 | } |
173 | |
174 | if (Constant *C = dyn_cast<Constant>(Val: V)) |
175 | return C->getAggregateElement(Elt: EltNo); |
176 | |
177 | if (InsertElementInst *III = dyn_cast<InsertElementInst>(Val: V)) { |
178 | // If this is an insert to a variable element, we don't know what it is. |
179 | if (!isa<ConstantInt>(Val: III->getOperand(i_nocapture: 2))) |
180 | return nullptr; |
181 | unsigned IIElt = cast<ConstantInt>(Val: III->getOperand(i_nocapture: 2))->getZExtValue(); |
182 | |
183 | // If this is an insert to the element we are looking for, return the |
184 | // inserted value. |
185 | if (EltNo == IIElt) |
186 | return III->getOperand(i_nocapture: 1); |
187 | |
188 | // Guard against infinite loop on malformed, unreachable IR. |
189 | if (III == III->getOperand(i_nocapture: 0)) |
190 | return nullptr; |
191 | |
192 | // Otherwise, the insertelement doesn't modify the value, recurse on its |
193 | // vector input. |
194 | return findScalarElement(V: III->getOperand(i_nocapture: 0), EltNo); |
195 | } |
196 | |
197 | ShuffleVectorInst *SVI = dyn_cast<ShuffleVectorInst>(Val: V); |
198 | // Restrict the following transformation to fixed-length vector. |
199 | if (SVI && isa<FixedVectorType>(Val: SVI->getType())) { |
200 | unsigned LHSWidth = |
201 | cast<FixedVectorType>(Val: SVI->getOperand(i_nocapture: 0)->getType())->getNumElements(); |
202 | int InEl = SVI->getMaskValue(Elt: EltNo); |
203 | if (InEl < 0) |
204 | return UndefValue::get(T: VTy->getElementType()); |
205 | if (InEl < (int)LHSWidth) |
206 | return findScalarElement(V: SVI->getOperand(i_nocapture: 0), EltNo: InEl); |
207 | return findScalarElement(V: SVI->getOperand(i_nocapture: 1), EltNo: InEl - LHSWidth); |
208 | } |
209 | |
210 | // Extract a value from a vector add operation with a constant zero. |
211 | // TODO: Use getBinOpIdentity() to generalize this. |
212 | Value *Val; Constant *C; |
213 | if (match(V, P: m_Add(L: m_Value(V&: Val), R: m_Constant(C)))) |
214 | if (Constant *Elt = C->getAggregateElement(Elt: EltNo)) |
215 | if (Elt->isNullValue()) |
216 | return findScalarElement(V: Val, EltNo); |
217 | |
218 | // If the vector is a splat then we can trivially find the scalar element. |
219 | if (isa<ScalableVectorType>(Val: VTy)) |
220 | if (Value *Splat = getSplatValue(V)) |
221 | if (EltNo < VTy->getElementCount().getKnownMinValue()) |
222 | return Splat; |
223 | |
224 | // Otherwise, we don't know. |
225 | return nullptr; |
226 | } |
227 | |
228 | int llvm::getSplatIndex(ArrayRef<int> Mask) { |
229 | int SplatIndex = -1; |
230 | for (int M : Mask) { |
231 | // Ignore invalid (undefined) mask elements. |
232 | if (M < 0) |
233 | continue; |
234 | |
235 | // There can be only 1 non-negative mask element value if this is a splat. |
236 | if (SplatIndex != -1 && SplatIndex != M) |
237 | return -1; |
238 | |
239 | // Initialize the splat index to the 1st non-negative mask element. |
240 | SplatIndex = M; |
241 | } |
242 | assert((SplatIndex == -1 || SplatIndex >= 0) && "Negative index?" ); |
243 | return SplatIndex; |
244 | } |
245 | |
246 | /// Get splat value if the input is a splat vector or return nullptr. |
247 | /// This function is not fully general. It checks only 2 cases: |
248 | /// the input value is (1) a splat constant vector or (2) a sequence |
249 | /// of instructions that broadcasts a scalar at element 0. |
250 | Value *llvm::getSplatValue(const Value *V) { |
251 | if (isa<VectorType>(Val: V->getType())) |
252 | if (auto *C = dyn_cast<Constant>(Val: V)) |
253 | return C->getSplatValue(); |
254 | |
255 | // shuf (inselt ?, Splat, 0), ?, <0, undef, 0, ...> |
256 | Value *Splat; |
257 | if (match(V, |
258 | P: m_Shuffle(v1: m_InsertElt(Val: m_Value(), Elt: m_Value(V&: Splat), Idx: m_ZeroInt()), |
259 | v2: m_Value(), mask: m_ZeroMask()))) |
260 | return Splat; |
261 | |
262 | return nullptr; |
263 | } |
264 | |
265 | bool llvm::isSplatValue(const Value *V, int Index, unsigned Depth) { |
266 | assert(Depth <= MaxAnalysisRecursionDepth && "Limit Search Depth" ); |
267 | |
268 | if (isa<VectorType>(Val: V->getType())) { |
269 | if (isa<UndefValue>(Val: V)) |
270 | return true; |
271 | // FIXME: We can allow undefs, but if Index was specified, we may want to |
272 | // check that the constant is defined at that index. |
273 | if (auto *C = dyn_cast<Constant>(Val: V)) |
274 | return C->getSplatValue() != nullptr; |
275 | } |
276 | |
277 | if (auto *Shuf = dyn_cast<ShuffleVectorInst>(Val: V)) { |
278 | // FIXME: We can safely allow undefs here. If Index was specified, we will |
279 | // check that the mask elt is defined at the required index. |
280 | if (!all_equal(Range: Shuf->getShuffleMask())) |
281 | return false; |
282 | |
283 | // Match any index. |
284 | if (Index == -1) |
285 | return true; |
286 | |
287 | // Match a specific element. The mask should be defined at and match the |
288 | // specified index. |
289 | return Shuf->getMaskValue(Elt: Index) == Index; |
290 | } |
291 | |
292 | // The remaining tests are all recursive, so bail out if we hit the limit. |
293 | if (Depth++ == MaxAnalysisRecursionDepth) |
294 | return false; |
295 | |
296 | // If both operands of a binop are splats, the result is a splat. |
297 | Value *X, *Y, *Z; |
298 | if (match(V, P: m_BinOp(L: m_Value(V&: X), R: m_Value(V&: Y)))) |
299 | return isSplatValue(V: X, Index, Depth) && isSplatValue(V: Y, Index, Depth); |
300 | |
301 | // If all operands of a select are splats, the result is a splat. |
302 | if (match(V, P: m_Select(C: m_Value(V&: X), L: m_Value(V&: Y), R: m_Value(V&: Z)))) |
303 | return isSplatValue(V: X, Index, Depth) && isSplatValue(V: Y, Index, Depth) && |
304 | isSplatValue(V: Z, Index, Depth); |
305 | |
306 | // TODO: Add support for unary ops (fneg), casts, intrinsics (overflow ops). |
307 | |
308 | return false; |
309 | } |
310 | |
311 | bool llvm::getShuffleDemandedElts(int SrcWidth, ArrayRef<int> Mask, |
312 | const APInt &DemandedElts, APInt &DemandedLHS, |
313 | APInt &DemandedRHS, bool AllowUndefElts) { |
314 | DemandedLHS = DemandedRHS = APInt::getZero(numBits: SrcWidth); |
315 | |
316 | // Early out if we don't demand any elements. |
317 | if (DemandedElts.isZero()) |
318 | return true; |
319 | |
320 | // Simple case of a shuffle with zeroinitializer. |
321 | if (all_of(Range&: Mask, P: [](int Elt) { return Elt == 0; })) { |
322 | DemandedLHS.setBit(0); |
323 | return true; |
324 | } |
325 | |
326 | for (unsigned I = 0, E = Mask.size(); I != E; ++I) { |
327 | int M = Mask[I]; |
328 | assert((-1 <= M) && (M < (SrcWidth * 2)) && |
329 | "Invalid shuffle mask constant" ); |
330 | |
331 | if (!DemandedElts[I] || (AllowUndefElts && (M < 0))) |
332 | continue; |
333 | |
334 | // For undef elements, we don't know anything about the common state of |
335 | // the shuffle result. |
336 | if (M < 0) |
337 | return false; |
338 | |
339 | if (M < SrcWidth) |
340 | DemandedLHS.setBit(M); |
341 | else |
342 | DemandedRHS.setBit(M - SrcWidth); |
343 | } |
344 | |
345 | return true; |
346 | } |
347 | |
348 | void llvm::narrowShuffleMaskElts(int Scale, ArrayRef<int> Mask, |
349 | SmallVectorImpl<int> &ScaledMask) { |
350 | assert(Scale > 0 && "Unexpected scaling factor" ); |
351 | |
352 | // Fast-path: if no scaling, then it is just a copy. |
353 | if (Scale == 1) { |
354 | ScaledMask.assign(in_start: Mask.begin(), in_end: Mask.end()); |
355 | return; |
356 | } |
357 | |
358 | ScaledMask.clear(); |
359 | for (int MaskElt : Mask) { |
360 | if (MaskElt >= 0) { |
361 | assert(((uint64_t)Scale * MaskElt + (Scale - 1)) <= INT32_MAX && |
362 | "Overflowed 32-bits" ); |
363 | } |
364 | for (int SliceElt = 0; SliceElt != Scale; ++SliceElt) |
365 | ScaledMask.push_back(Elt: MaskElt < 0 ? MaskElt : Scale * MaskElt + SliceElt); |
366 | } |
367 | } |
368 | |
369 | bool llvm::widenShuffleMaskElts(int Scale, ArrayRef<int> Mask, |
370 | SmallVectorImpl<int> &ScaledMask) { |
371 | assert(Scale > 0 && "Unexpected scaling factor" ); |
372 | |
373 | // Fast-path: if no scaling, then it is just a copy. |
374 | if (Scale == 1) { |
375 | ScaledMask.assign(in_start: Mask.begin(), in_end: Mask.end()); |
376 | return true; |
377 | } |
378 | |
379 | // We must map the original elements down evenly to a type with less elements. |
380 | int NumElts = Mask.size(); |
381 | if (NumElts % Scale != 0) |
382 | return false; |
383 | |
384 | ScaledMask.clear(); |
385 | ScaledMask.reserve(N: NumElts / Scale); |
386 | |
387 | // Step through the input mask by splitting into Scale-sized slices. |
388 | do { |
389 | ArrayRef<int> MaskSlice = Mask.take_front(N: Scale); |
390 | assert((int)MaskSlice.size() == Scale && "Expected Scale-sized slice." ); |
391 | |
392 | // The first element of the slice determines how we evaluate this slice. |
393 | int SliceFront = MaskSlice.front(); |
394 | if (SliceFront < 0) { |
395 | // Negative values (undef or other "sentinel" values) must be equal across |
396 | // the entire slice. |
397 | if (!all_equal(Range&: MaskSlice)) |
398 | return false; |
399 | ScaledMask.push_back(Elt: SliceFront); |
400 | } else { |
401 | // A positive mask element must be cleanly divisible. |
402 | if (SliceFront % Scale != 0) |
403 | return false; |
404 | // Elements of the slice must be consecutive. |
405 | for (int i = 1; i < Scale; ++i) |
406 | if (MaskSlice[i] != SliceFront + i) |
407 | return false; |
408 | ScaledMask.push_back(Elt: SliceFront / Scale); |
409 | } |
410 | Mask = Mask.drop_front(N: Scale); |
411 | } while (!Mask.empty()); |
412 | |
413 | assert((int)ScaledMask.size() * Scale == NumElts && "Unexpected scaled mask" ); |
414 | |
415 | // All elements of the original mask can be scaled down to map to the elements |
416 | // of a mask with wider elements. |
417 | return true; |
418 | } |
419 | |
420 | void llvm::getShuffleMaskWithWidestElts(ArrayRef<int> Mask, |
421 | SmallVectorImpl<int> &ScaledMask) { |
422 | std::array<SmallVector<int, 16>, 2> TmpMasks; |
423 | SmallVectorImpl<int> *Output = &TmpMasks[0], *Tmp = &TmpMasks[1]; |
424 | ArrayRef<int> InputMask = Mask; |
425 | for (unsigned Scale = 2; Scale <= InputMask.size(); ++Scale) { |
426 | while (widenShuffleMaskElts(Scale, Mask: InputMask, ScaledMask&: *Output)) { |
427 | InputMask = *Output; |
428 | std::swap(a&: Output, b&: Tmp); |
429 | } |
430 | } |
431 | ScaledMask.assign(in_start: InputMask.begin(), in_end: InputMask.end()); |
432 | } |
433 | |
434 | void llvm::processShuffleMasks( |
435 | ArrayRef<int> Mask, unsigned NumOfSrcRegs, unsigned NumOfDestRegs, |
436 | unsigned NumOfUsedRegs, function_ref<void()> NoInputAction, |
437 | function_ref<void(ArrayRef<int>, unsigned, unsigned)> SingleInputAction, |
438 | function_ref<void(ArrayRef<int>, unsigned, unsigned)> ManyInputsAction) { |
439 | SmallVector<SmallVector<SmallVector<int>>> Res(NumOfDestRegs); |
440 | // Try to perform better estimation of the permutation. |
441 | // 1. Split the source/destination vectors into real registers. |
442 | // 2. Do the mask analysis to identify which real registers are |
443 | // permuted. |
444 | int Sz = Mask.size(); |
445 | unsigned SzDest = Sz / NumOfDestRegs; |
446 | unsigned SzSrc = Sz / NumOfSrcRegs; |
447 | for (unsigned I = 0; I < NumOfDestRegs; ++I) { |
448 | auto &RegMasks = Res[I]; |
449 | RegMasks.assign(NumElts: NumOfSrcRegs, Elt: {}); |
450 | // Check that the values in dest registers are in the one src |
451 | // register. |
452 | for (unsigned K = 0; K < SzDest; ++K) { |
453 | int Idx = I * SzDest + K; |
454 | if (Idx == Sz) |
455 | break; |
456 | if (Mask[Idx] >= Sz || Mask[Idx] == PoisonMaskElem) |
457 | continue; |
458 | int SrcRegIdx = Mask[Idx] / SzSrc; |
459 | // Add a cost of PermuteTwoSrc for each new source register permute, |
460 | // if we have more than one source registers. |
461 | if (RegMasks[SrcRegIdx].empty()) |
462 | RegMasks[SrcRegIdx].assign(NumElts: SzDest, Elt: PoisonMaskElem); |
463 | RegMasks[SrcRegIdx][K] = Mask[Idx] % SzSrc; |
464 | } |
465 | } |
466 | // Process split mask. |
467 | for (unsigned I = 0; I < NumOfUsedRegs; ++I) { |
468 | auto &Dest = Res[I]; |
469 | int NumSrcRegs = |
470 | count_if(Range&: Dest, P: [](ArrayRef<int> Mask) { return !Mask.empty(); }); |
471 | switch (NumSrcRegs) { |
472 | case 0: |
473 | // No input vectors were used! |
474 | NoInputAction(); |
475 | break; |
476 | case 1: { |
477 | // Find the only mask with at least single undef mask elem. |
478 | auto *It = |
479 | find_if(Range&: Dest, P: [](ArrayRef<int> Mask) { return !Mask.empty(); }); |
480 | unsigned SrcReg = std::distance(first: Dest.begin(), last: It); |
481 | SingleInputAction(*It, SrcReg, I); |
482 | break; |
483 | } |
484 | default: { |
485 | // The first mask is a permutation of a single register. Since we have >2 |
486 | // input registers to shuffle, we merge the masks for 2 first registers |
487 | // and generate a shuffle of 2 registers rather than the reordering of the |
488 | // first register and then shuffle with the second register. Next, |
489 | // generate the shuffles of the resulting register + the remaining |
490 | // registers from the list. |
491 | auto &&CombineMasks = [](MutableArrayRef<int> FirstMask, |
492 | ArrayRef<int> SecondMask) { |
493 | for (int Idx = 0, VF = FirstMask.size(); Idx < VF; ++Idx) { |
494 | if (SecondMask[Idx] != PoisonMaskElem) { |
495 | assert(FirstMask[Idx] == PoisonMaskElem && |
496 | "Expected undefined mask element." ); |
497 | FirstMask[Idx] = SecondMask[Idx] + VF; |
498 | } |
499 | } |
500 | }; |
501 | auto &&NormalizeMask = [](MutableArrayRef<int> Mask) { |
502 | for (int Idx = 0, VF = Mask.size(); Idx < VF; ++Idx) { |
503 | if (Mask[Idx] != PoisonMaskElem) |
504 | Mask[Idx] = Idx; |
505 | } |
506 | }; |
507 | int SecondIdx; |
508 | do { |
509 | int FirstIdx = -1; |
510 | SecondIdx = -1; |
511 | MutableArrayRef<int> FirstMask, SecondMask; |
512 | for (unsigned I = 0; I < NumOfDestRegs; ++I) { |
513 | SmallVectorImpl<int> &RegMask = Dest[I]; |
514 | if (RegMask.empty()) |
515 | continue; |
516 | |
517 | if (FirstIdx == SecondIdx) { |
518 | FirstIdx = I; |
519 | FirstMask = RegMask; |
520 | continue; |
521 | } |
522 | SecondIdx = I; |
523 | SecondMask = RegMask; |
524 | CombineMasks(FirstMask, SecondMask); |
525 | ManyInputsAction(FirstMask, FirstIdx, SecondIdx); |
526 | NormalizeMask(FirstMask); |
527 | RegMask.clear(); |
528 | SecondMask = FirstMask; |
529 | SecondIdx = FirstIdx; |
530 | } |
531 | if (FirstIdx != SecondIdx && SecondIdx >= 0) { |
532 | CombineMasks(SecondMask, FirstMask); |
533 | ManyInputsAction(SecondMask, SecondIdx, FirstIdx); |
534 | Dest[FirstIdx].clear(); |
535 | NormalizeMask(SecondMask); |
536 | } |
537 | } while (SecondIdx >= 0); |
538 | break; |
539 | } |
540 | } |
541 | } |
542 | } |
543 | |
544 | MapVector<Instruction *, uint64_t> |
545 | llvm::computeMinimumValueSizes(ArrayRef<BasicBlock *> Blocks, DemandedBits &DB, |
546 | const TargetTransformInfo *TTI) { |
547 | |
548 | // DemandedBits will give us every value's live-out bits. But we want |
549 | // to ensure no extra casts would need to be inserted, so every DAG |
550 | // of connected values must have the same minimum bitwidth. |
551 | EquivalenceClasses<Value *> ECs; |
552 | SmallVector<Value *, 16> Worklist; |
553 | SmallPtrSet<Value *, 4> Roots; |
554 | SmallPtrSet<Value *, 16> Visited; |
555 | DenseMap<Value *, uint64_t> DBits; |
556 | SmallPtrSet<Instruction *, 4> InstructionSet; |
557 | MapVector<Instruction *, uint64_t> MinBWs; |
558 | |
559 | // Determine the roots. We work bottom-up, from truncs or icmps. |
560 | bool SeenExtFromIllegalType = false; |
561 | for (auto *BB : Blocks) |
562 | for (auto &I : *BB) { |
563 | InstructionSet.insert(Ptr: &I); |
564 | |
565 | if (TTI && (isa<ZExtInst>(Val: &I) || isa<SExtInst>(Val: &I)) && |
566 | !TTI->isTypeLegal(Ty: I.getOperand(i: 0)->getType())) |
567 | SeenExtFromIllegalType = true; |
568 | |
569 | // Only deal with non-vector integers up to 64-bits wide. |
570 | if ((isa<TruncInst>(Val: &I) || isa<ICmpInst>(Val: &I)) && |
571 | !I.getType()->isVectorTy() && |
572 | I.getOperand(i: 0)->getType()->getScalarSizeInBits() <= 64) { |
573 | // Don't make work for ourselves. If we know the loaded type is legal, |
574 | // don't add it to the worklist. |
575 | if (TTI && isa<TruncInst>(Val: &I) && TTI->isTypeLegal(Ty: I.getType())) |
576 | continue; |
577 | |
578 | Worklist.push_back(Elt: &I); |
579 | Roots.insert(Ptr: &I); |
580 | } |
581 | } |
582 | // Early exit. |
583 | if (Worklist.empty() || (TTI && !SeenExtFromIllegalType)) |
584 | return MinBWs; |
585 | |
586 | // Now proceed breadth-first, unioning values together. |
587 | while (!Worklist.empty()) { |
588 | Value *Val = Worklist.pop_back_val(); |
589 | Value *Leader = ECs.getOrInsertLeaderValue(V: Val); |
590 | |
591 | if (!Visited.insert(Ptr: Val).second) |
592 | continue; |
593 | |
594 | // Non-instructions terminate a chain successfully. |
595 | if (!isa<Instruction>(Val)) |
596 | continue; |
597 | Instruction *I = cast<Instruction>(Val); |
598 | |
599 | // If we encounter a type that is larger than 64 bits, we can't represent |
600 | // it so bail out. |
601 | if (DB.getDemandedBits(I).getBitWidth() > 64) |
602 | return MapVector<Instruction *, uint64_t>(); |
603 | |
604 | uint64_t V = DB.getDemandedBits(I).getZExtValue(); |
605 | DBits[Leader] |= V; |
606 | DBits[I] = V; |
607 | |
608 | // Casts, loads and instructions outside of our range terminate a chain |
609 | // successfully. |
610 | if (isa<SExtInst>(Val: I) || isa<ZExtInst>(Val: I) || isa<LoadInst>(Val: I) || |
611 | !InstructionSet.count(Ptr: I)) |
612 | continue; |
613 | |
614 | // Unsafe casts terminate a chain unsuccessfully. We can't do anything |
615 | // useful with bitcasts, ptrtoints or inttoptrs and it'd be unsafe to |
616 | // transform anything that relies on them. |
617 | if (isa<BitCastInst>(Val: I) || isa<PtrToIntInst>(Val: I) || isa<IntToPtrInst>(Val: I) || |
618 | !I->getType()->isIntegerTy()) { |
619 | DBits[Leader] |= ~0ULL; |
620 | continue; |
621 | } |
622 | |
623 | // We don't modify the types of PHIs. Reductions will already have been |
624 | // truncated if possible, and inductions' sizes will have been chosen by |
625 | // indvars. |
626 | if (isa<PHINode>(Val: I)) |
627 | continue; |
628 | |
629 | if (DBits[Leader] == ~0ULL) |
630 | // All bits demanded, no point continuing. |
631 | continue; |
632 | |
633 | for (Value *O : cast<User>(Val: I)->operands()) { |
634 | ECs.unionSets(V1: Leader, V2: O); |
635 | Worklist.push_back(Elt: O); |
636 | } |
637 | } |
638 | |
639 | // Now we've discovered all values, walk them to see if there are |
640 | // any users we didn't see. If there are, we can't optimize that |
641 | // chain. |
642 | for (auto &I : DBits) |
643 | for (auto *U : I.first->users()) |
644 | if (U->getType()->isIntegerTy() && DBits.count(Val: U) == 0) |
645 | DBits[ECs.getOrInsertLeaderValue(V: I.first)] |= ~0ULL; |
646 | |
647 | for (auto I = ECs.begin(), E = ECs.end(); I != E; ++I) { |
648 | uint64_t LeaderDemandedBits = 0; |
649 | for (Value *M : llvm::make_range(x: ECs.member_begin(I), y: ECs.member_end())) |
650 | LeaderDemandedBits |= DBits[M]; |
651 | |
652 | uint64_t MinBW = llvm::bit_width(Value: LeaderDemandedBits); |
653 | // Round up to a power of 2 |
654 | MinBW = llvm::bit_ceil(Value: MinBW); |
655 | |
656 | // We don't modify the types of PHIs. Reductions will already have been |
657 | // truncated if possible, and inductions' sizes will have been chosen by |
658 | // indvars. |
659 | // If we are required to shrink a PHI, abandon this entire equivalence class. |
660 | bool Abort = false; |
661 | for (Value *M : llvm::make_range(x: ECs.member_begin(I), y: ECs.member_end())) |
662 | if (isa<PHINode>(Val: M) && MinBW < M->getType()->getScalarSizeInBits()) { |
663 | Abort = true; |
664 | break; |
665 | } |
666 | if (Abort) |
667 | continue; |
668 | |
669 | for (Value *M : llvm::make_range(x: ECs.member_begin(I), y: ECs.member_end())) { |
670 | auto *MI = dyn_cast<Instruction>(Val: M); |
671 | if (!MI) |
672 | continue; |
673 | Type *Ty = M->getType(); |
674 | if (Roots.count(Ptr: M)) |
675 | Ty = MI->getOperand(i: 0)->getType(); |
676 | |
677 | if (MinBW >= Ty->getScalarSizeInBits()) |
678 | continue; |
679 | |
680 | // If any of M's operands demand more bits than MinBW then M cannot be |
681 | // performed safely in MinBW. |
682 | if (any_of(Range: MI->operands(), P: [&DB, MinBW](Use &U) { |
683 | auto *CI = dyn_cast<ConstantInt>(Val&: U); |
684 | // For constants shift amounts, check if the shift would result in |
685 | // poison. |
686 | if (CI && |
687 | isa<ShlOperator, LShrOperator, AShrOperator>(Val: U.getUser()) && |
688 | U.getOperandNo() == 1) |
689 | return CI->uge(Num: MinBW); |
690 | uint64_t BW = bit_width(Value: DB.getDemandedBits(U: &U).getZExtValue()); |
691 | return bit_ceil(Value: BW) > MinBW; |
692 | })) |
693 | continue; |
694 | |
695 | MinBWs[MI] = MinBW; |
696 | } |
697 | } |
698 | |
699 | return MinBWs; |
700 | } |
701 | |
702 | /// Add all access groups in @p AccGroups to @p List. |
703 | template <typename ListT> |
704 | static void addToAccessGroupList(ListT &List, MDNode *AccGroups) { |
705 | // Interpret an access group as a list containing itself. |
706 | if (AccGroups->getNumOperands() == 0) { |
707 | assert(isValidAsAccessGroup(AccGroups) && "Node must be an access group" ); |
708 | List.insert(AccGroups); |
709 | return; |
710 | } |
711 | |
712 | for (const auto &AccGroupListOp : AccGroups->operands()) { |
713 | auto *Item = cast<MDNode>(Val: AccGroupListOp.get()); |
714 | assert(isValidAsAccessGroup(Item) && "List item must be an access group" ); |
715 | List.insert(Item); |
716 | } |
717 | } |
718 | |
719 | MDNode *llvm::uniteAccessGroups(MDNode *AccGroups1, MDNode *AccGroups2) { |
720 | if (!AccGroups1) |
721 | return AccGroups2; |
722 | if (!AccGroups2) |
723 | return AccGroups1; |
724 | if (AccGroups1 == AccGroups2) |
725 | return AccGroups1; |
726 | |
727 | SmallSetVector<Metadata *, 4> Union; |
728 | addToAccessGroupList(List&: Union, AccGroups: AccGroups1); |
729 | addToAccessGroupList(List&: Union, AccGroups: AccGroups2); |
730 | |
731 | if (Union.size() == 0) |
732 | return nullptr; |
733 | if (Union.size() == 1) |
734 | return cast<MDNode>(Val: Union.front()); |
735 | |
736 | LLVMContext &Ctx = AccGroups1->getContext(); |
737 | return MDNode::get(Context&: Ctx, MDs: Union.getArrayRef()); |
738 | } |
739 | |
740 | MDNode *llvm::intersectAccessGroups(const Instruction *Inst1, |
741 | const Instruction *Inst2) { |
742 | bool MayAccessMem1 = Inst1->mayReadOrWriteMemory(); |
743 | bool MayAccessMem2 = Inst2->mayReadOrWriteMemory(); |
744 | |
745 | if (!MayAccessMem1 && !MayAccessMem2) |
746 | return nullptr; |
747 | if (!MayAccessMem1) |
748 | return Inst2->getMetadata(KindID: LLVMContext::MD_access_group); |
749 | if (!MayAccessMem2) |
750 | return Inst1->getMetadata(KindID: LLVMContext::MD_access_group); |
751 | |
752 | MDNode *MD1 = Inst1->getMetadata(KindID: LLVMContext::MD_access_group); |
753 | MDNode *MD2 = Inst2->getMetadata(KindID: LLVMContext::MD_access_group); |
754 | if (!MD1 || !MD2) |
755 | return nullptr; |
756 | if (MD1 == MD2) |
757 | return MD1; |
758 | |
759 | // Use set for scalable 'contains' check. |
760 | SmallPtrSet<Metadata *, 4> AccGroupSet2; |
761 | addToAccessGroupList(List&: AccGroupSet2, AccGroups: MD2); |
762 | |
763 | SmallVector<Metadata *, 4> Intersection; |
764 | if (MD1->getNumOperands() == 0) { |
765 | assert(isValidAsAccessGroup(MD1) && "Node must be an access group" ); |
766 | if (AccGroupSet2.count(Ptr: MD1)) |
767 | Intersection.push_back(Elt: MD1); |
768 | } else { |
769 | for (const MDOperand &Node : MD1->operands()) { |
770 | auto *Item = cast<MDNode>(Val: Node.get()); |
771 | assert(isValidAsAccessGroup(Item) && "List item must be an access group" ); |
772 | if (AccGroupSet2.count(Ptr: Item)) |
773 | Intersection.push_back(Elt: Item); |
774 | } |
775 | } |
776 | |
777 | if (Intersection.size() == 0) |
778 | return nullptr; |
779 | if (Intersection.size() == 1) |
780 | return cast<MDNode>(Val: Intersection.front()); |
781 | |
782 | LLVMContext &Ctx = Inst1->getContext(); |
783 | return MDNode::get(Context&: Ctx, MDs: Intersection); |
784 | } |
785 | |
786 | /// \returns \p I after propagating metadata from \p VL. |
787 | Instruction *llvm::propagateMetadata(Instruction *Inst, ArrayRef<Value *> VL) { |
788 | if (VL.empty()) |
789 | return Inst; |
790 | Instruction *I0 = cast<Instruction>(Val: VL[0]); |
791 | SmallVector<std::pair<unsigned, MDNode *>, 4> Metadata; |
792 | I0->getAllMetadataOtherThanDebugLoc(MDs&: Metadata); |
793 | |
794 | for (auto Kind : {LLVMContext::MD_tbaa, LLVMContext::MD_alias_scope, |
795 | LLVMContext::MD_noalias, LLVMContext::MD_fpmath, |
796 | LLVMContext::MD_nontemporal, LLVMContext::MD_invariant_load, |
797 | LLVMContext::MD_access_group, LLVMContext::MD_mmra}) { |
798 | MDNode *MD = I0->getMetadata(KindID: Kind); |
799 | for (int J = 1, E = VL.size(); MD && J != E; ++J) { |
800 | const Instruction *IJ = cast<Instruction>(Val: VL[J]); |
801 | MDNode *IMD = IJ->getMetadata(KindID: Kind); |
802 | |
803 | switch (Kind) { |
804 | case LLVMContext::MD_mmra: { |
805 | MD = MMRAMetadata::combine(Ctx&: Inst->getContext(), A: MD, B: IMD); |
806 | break; |
807 | } |
808 | case LLVMContext::MD_tbaa: |
809 | MD = MDNode::getMostGenericTBAA(A: MD, B: IMD); |
810 | break; |
811 | case LLVMContext::MD_alias_scope: |
812 | MD = MDNode::getMostGenericAliasScope(A: MD, B: IMD); |
813 | break; |
814 | case LLVMContext::MD_fpmath: |
815 | MD = MDNode::getMostGenericFPMath(A: MD, B: IMD); |
816 | break; |
817 | case LLVMContext::MD_noalias: |
818 | case LLVMContext::MD_nontemporal: |
819 | case LLVMContext::MD_invariant_load: |
820 | MD = MDNode::intersect(A: MD, B: IMD); |
821 | break; |
822 | case LLVMContext::MD_access_group: |
823 | MD = intersectAccessGroups(Inst1: Inst, Inst2: IJ); |
824 | break; |
825 | default: |
826 | llvm_unreachable("unhandled metadata" ); |
827 | } |
828 | } |
829 | |
830 | Inst->setMetadata(KindID: Kind, Node: MD); |
831 | } |
832 | |
833 | return Inst; |
834 | } |
835 | |
836 | Constant * |
837 | llvm::createBitMaskForGaps(IRBuilderBase &Builder, unsigned VF, |
838 | const InterleaveGroup<Instruction> &Group) { |
839 | // All 1's means mask is not needed. |
840 | if (Group.getNumMembers() == Group.getFactor()) |
841 | return nullptr; |
842 | |
843 | // TODO: support reversed access. |
844 | assert(!Group.isReverse() && "Reversed group not supported." ); |
845 | |
846 | SmallVector<Constant *, 16> Mask; |
847 | for (unsigned i = 0; i < VF; i++) |
848 | for (unsigned j = 0; j < Group.getFactor(); ++j) { |
849 | unsigned HasMember = Group.getMember(Index: j) ? 1 : 0; |
850 | Mask.push_back(Elt: Builder.getInt1(V: HasMember)); |
851 | } |
852 | |
853 | return ConstantVector::get(V: Mask); |
854 | } |
855 | |
856 | llvm::SmallVector<int, 16> |
857 | llvm::createReplicatedMask(unsigned ReplicationFactor, unsigned VF) { |
858 | SmallVector<int, 16> MaskVec; |
859 | for (unsigned i = 0; i < VF; i++) |
860 | for (unsigned j = 0; j < ReplicationFactor; j++) |
861 | MaskVec.push_back(Elt: i); |
862 | |
863 | return MaskVec; |
864 | } |
865 | |
866 | llvm::SmallVector<int, 16> llvm::createInterleaveMask(unsigned VF, |
867 | unsigned NumVecs) { |
868 | SmallVector<int, 16> Mask; |
869 | for (unsigned i = 0; i < VF; i++) |
870 | for (unsigned j = 0; j < NumVecs; j++) |
871 | Mask.push_back(Elt: j * VF + i); |
872 | |
873 | return Mask; |
874 | } |
875 | |
876 | llvm::SmallVector<int, 16> |
877 | llvm::createStrideMask(unsigned Start, unsigned Stride, unsigned VF) { |
878 | SmallVector<int, 16> Mask; |
879 | for (unsigned i = 0; i < VF; i++) |
880 | Mask.push_back(Elt: Start + i * Stride); |
881 | |
882 | return Mask; |
883 | } |
884 | |
885 | llvm::SmallVector<int, 16> llvm::createSequentialMask(unsigned Start, |
886 | unsigned NumInts, |
887 | unsigned NumUndefs) { |
888 | SmallVector<int, 16> Mask; |
889 | for (unsigned i = 0; i < NumInts; i++) |
890 | Mask.push_back(Elt: Start + i); |
891 | |
892 | for (unsigned i = 0; i < NumUndefs; i++) |
893 | Mask.push_back(Elt: -1); |
894 | |
895 | return Mask; |
896 | } |
897 | |
898 | llvm::SmallVector<int, 16> llvm::createUnaryMask(ArrayRef<int> Mask, |
899 | unsigned NumElts) { |
900 | // Avoid casts in the loop and make sure we have a reasonable number. |
901 | int NumEltsSigned = NumElts; |
902 | assert(NumEltsSigned > 0 && "Expected smaller or non-zero element count" ); |
903 | |
904 | // If the mask chooses an element from operand 1, reduce it to choose from the |
905 | // corresponding element of operand 0. Undef mask elements are unchanged. |
906 | SmallVector<int, 16> UnaryMask; |
907 | for (int MaskElt : Mask) { |
908 | assert((MaskElt < NumEltsSigned * 2) && "Expected valid shuffle mask" ); |
909 | int UnaryElt = MaskElt >= NumEltsSigned ? MaskElt - NumEltsSigned : MaskElt; |
910 | UnaryMask.push_back(Elt: UnaryElt); |
911 | } |
912 | return UnaryMask; |
913 | } |
914 | |
915 | /// A helper function for concatenating vectors. This function concatenates two |
916 | /// vectors having the same element type. If the second vector has fewer |
917 | /// elements than the first, it is padded with undefs. |
918 | static Value *concatenateTwoVectors(IRBuilderBase &Builder, Value *V1, |
919 | Value *V2) { |
920 | VectorType *VecTy1 = dyn_cast<VectorType>(Val: V1->getType()); |
921 | VectorType *VecTy2 = dyn_cast<VectorType>(Val: V2->getType()); |
922 | assert(VecTy1 && VecTy2 && |
923 | VecTy1->getScalarType() == VecTy2->getScalarType() && |
924 | "Expect two vectors with the same element type" ); |
925 | |
926 | unsigned NumElts1 = cast<FixedVectorType>(Val: VecTy1)->getNumElements(); |
927 | unsigned NumElts2 = cast<FixedVectorType>(Val: VecTy2)->getNumElements(); |
928 | assert(NumElts1 >= NumElts2 && "Unexpect the first vector has less elements" ); |
929 | |
930 | if (NumElts1 > NumElts2) { |
931 | // Extend with UNDEFs. |
932 | V2 = Builder.CreateShuffleVector( |
933 | V: V2, Mask: createSequentialMask(Start: 0, NumInts: NumElts2, NumUndefs: NumElts1 - NumElts2)); |
934 | } |
935 | |
936 | return Builder.CreateShuffleVector( |
937 | V1, V2, Mask: createSequentialMask(Start: 0, NumInts: NumElts1 + NumElts2, NumUndefs: 0)); |
938 | } |
939 | |
940 | Value *llvm::concatenateVectors(IRBuilderBase &Builder, |
941 | ArrayRef<Value *> Vecs) { |
942 | unsigned NumVecs = Vecs.size(); |
943 | assert(NumVecs > 1 && "Should be at least two vectors" ); |
944 | |
945 | SmallVector<Value *, 8> ResList; |
946 | ResList.append(in_start: Vecs.begin(), in_end: Vecs.end()); |
947 | do { |
948 | SmallVector<Value *, 8> TmpList; |
949 | for (unsigned i = 0; i < NumVecs - 1; i += 2) { |
950 | Value *V0 = ResList[i], *V1 = ResList[i + 1]; |
951 | assert((V0->getType() == V1->getType() || i == NumVecs - 2) && |
952 | "Only the last vector may have a different type" ); |
953 | |
954 | TmpList.push_back(Elt: concatenateTwoVectors(Builder, V1: V0, V2: V1)); |
955 | } |
956 | |
957 | // Push the last vector if the total number of vectors is odd. |
958 | if (NumVecs % 2 != 0) |
959 | TmpList.push_back(Elt: ResList[NumVecs - 1]); |
960 | |
961 | ResList = TmpList; |
962 | NumVecs = ResList.size(); |
963 | } while (NumVecs > 1); |
964 | |
965 | return ResList[0]; |
966 | } |
967 | |
968 | bool llvm::maskIsAllZeroOrUndef(Value *Mask) { |
969 | assert(isa<VectorType>(Mask->getType()) && |
970 | isa<IntegerType>(Mask->getType()->getScalarType()) && |
971 | cast<IntegerType>(Mask->getType()->getScalarType())->getBitWidth() == |
972 | 1 && |
973 | "Mask must be a vector of i1" ); |
974 | |
975 | auto *ConstMask = dyn_cast<Constant>(Val: Mask); |
976 | if (!ConstMask) |
977 | return false; |
978 | if (ConstMask->isNullValue() || isa<UndefValue>(Val: ConstMask)) |
979 | return true; |
980 | if (isa<ScalableVectorType>(Val: ConstMask->getType())) |
981 | return false; |
982 | for (unsigned |
983 | I = 0, |
984 | E = cast<FixedVectorType>(Val: ConstMask->getType())->getNumElements(); |
985 | I != E; ++I) { |
986 | if (auto *MaskElt = ConstMask->getAggregateElement(Elt: I)) |
987 | if (MaskElt->isNullValue() || isa<UndefValue>(Val: MaskElt)) |
988 | continue; |
989 | return false; |
990 | } |
991 | return true; |
992 | } |
993 | |
994 | bool llvm::maskIsAllOneOrUndef(Value *Mask) { |
995 | assert(isa<VectorType>(Mask->getType()) && |
996 | isa<IntegerType>(Mask->getType()->getScalarType()) && |
997 | cast<IntegerType>(Mask->getType()->getScalarType())->getBitWidth() == |
998 | 1 && |
999 | "Mask must be a vector of i1" ); |
1000 | |
1001 | auto *ConstMask = dyn_cast<Constant>(Val: Mask); |
1002 | if (!ConstMask) |
1003 | return false; |
1004 | if (ConstMask->isAllOnesValue() || isa<UndefValue>(Val: ConstMask)) |
1005 | return true; |
1006 | if (isa<ScalableVectorType>(Val: ConstMask->getType())) |
1007 | return false; |
1008 | for (unsigned |
1009 | I = 0, |
1010 | E = cast<FixedVectorType>(Val: ConstMask->getType())->getNumElements(); |
1011 | I != E; ++I) { |
1012 | if (auto *MaskElt = ConstMask->getAggregateElement(Elt: I)) |
1013 | if (MaskElt->isAllOnesValue() || isa<UndefValue>(Val: MaskElt)) |
1014 | continue; |
1015 | return false; |
1016 | } |
1017 | return true; |
1018 | } |
1019 | |
1020 | bool llvm::maskContainsAllOneOrUndef(Value *Mask) { |
1021 | assert(isa<VectorType>(Mask->getType()) && |
1022 | isa<IntegerType>(Mask->getType()->getScalarType()) && |
1023 | cast<IntegerType>(Mask->getType()->getScalarType())->getBitWidth() == |
1024 | 1 && |
1025 | "Mask must be a vector of i1" ); |
1026 | |
1027 | auto *ConstMask = dyn_cast<Constant>(Val: Mask); |
1028 | if (!ConstMask) |
1029 | return false; |
1030 | if (ConstMask->isAllOnesValue() || isa<UndefValue>(Val: ConstMask)) |
1031 | return true; |
1032 | if (isa<ScalableVectorType>(Val: ConstMask->getType())) |
1033 | return false; |
1034 | for (unsigned |
1035 | I = 0, |
1036 | E = cast<FixedVectorType>(Val: ConstMask->getType())->getNumElements(); |
1037 | I != E; ++I) { |
1038 | if (auto *MaskElt = ConstMask->getAggregateElement(Elt: I)) |
1039 | if (MaskElt->isAllOnesValue() || isa<UndefValue>(Val: MaskElt)) |
1040 | return true; |
1041 | } |
1042 | return false; |
1043 | } |
1044 | |
1045 | /// TODO: This is a lot like known bits, but for |
1046 | /// vectors. Is there something we can common this with? |
1047 | APInt llvm::possiblyDemandedEltsInMask(Value *Mask) { |
1048 | assert(isa<FixedVectorType>(Mask->getType()) && |
1049 | isa<IntegerType>(Mask->getType()->getScalarType()) && |
1050 | cast<IntegerType>(Mask->getType()->getScalarType())->getBitWidth() == |
1051 | 1 && |
1052 | "Mask must be a fixed width vector of i1" ); |
1053 | |
1054 | const unsigned VWidth = |
1055 | cast<FixedVectorType>(Val: Mask->getType())->getNumElements(); |
1056 | APInt DemandedElts = APInt::getAllOnes(numBits: VWidth); |
1057 | if (auto *CV = dyn_cast<ConstantVector>(Val: Mask)) |
1058 | for (unsigned i = 0; i < VWidth; i++) |
1059 | if (CV->getAggregateElement(Elt: i)->isNullValue()) |
1060 | DemandedElts.clearBit(BitPosition: i); |
1061 | return DemandedElts; |
1062 | } |
1063 | |
1064 | bool InterleavedAccessInfo::isStrided(int Stride) { |
1065 | unsigned Factor = std::abs(x: Stride); |
1066 | return Factor >= 2 && Factor <= MaxInterleaveGroupFactor; |
1067 | } |
1068 | |
1069 | void InterleavedAccessInfo::collectConstStrideAccesses( |
1070 | MapVector<Instruction *, StrideDescriptor> &AccessStrideInfo, |
1071 | const DenseMap<Value*, const SCEV*> &Strides) { |
1072 | auto &DL = TheLoop->getHeader()->getModule()->getDataLayout(); |
1073 | |
1074 | // Since it's desired that the load/store instructions be maintained in |
1075 | // "program order" for the interleaved access analysis, we have to visit the |
1076 | // blocks in the loop in reverse postorder (i.e., in a topological order). |
1077 | // Such an ordering will ensure that any load/store that may be executed |
1078 | // before a second load/store will precede the second load/store in |
1079 | // AccessStrideInfo. |
1080 | LoopBlocksDFS DFS(TheLoop); |
1081 | DFS.perform(LI); |
1082 | for (BasicBlock *BB : make_range(x: DFS.beginRPO(), y: DFS.endRPO())) |
1083 | for (auto &I : *BB) { |
1084 | Value *Ptr = getLoadStorePointerOperand(V: &I); |
1085 | if (!Ptr) |
1086 | continue; |
1087 | Type *ElementTy = getLoadStoreType(I: &I); |
1088 | |
1089 | // Currently, codegen doesn't support cases where the type size doesn't |
1090 | // match the alloc size. Skip them for now. |
1091 | uint64_t Size = DL.getTypeAllocSize(Ty: ElementTy); |
1092 | if (Size * 8 != DL.getTypeSizeInBits(Ty: ElementTy)) |
1093 | continue; |
1094 | |
1095 | // We don't check wrapping here because we don't know yet if Ptr will be |
1096 | // part of a full group or a group with gaps. Checking wrapping for all |
1097 | // pointers (even those that end up in groups with no gaps) will be overly |
1098 | // conservative. For full groups, wrapping should be ok since if we would |
1099 | // wrap around the address space we would do a memory access at nullptr |
1100 | // even without the transformation. The wrapping checks are therefore |
1101 | // deferred until after we've formed the interleaved groups. |
1102 | int64_t Stride = |
1103 | getPtrStride(PSE, AccessTy: ElementTy, Ptr, Lp: TheLoop, StridesMap: Strides, |
1104 | /*Assume=*/true, /*ShouldCheckWrap=*/false).value_or(u: 0); |
1105 | |
1106 | const SCEV *Scev = replaceSymbolicStrideSCEV(PSE, PtrToStride: Strides, Ptr); |
1107 | AccessStrideInfo[&I] = StrideDescriptor(Stride, Scev, Size, |
1108 | getLoadStoreAlignment(I: &I)); |
1109 | } |
1110 | } |
1111 | |
1112 | // Analyze interleaved accesses and collect them into interleaved load and |
1113 | // store groups. |
1114 | // |
1115 | // When generating code for an interleaved load group, we effectively hoist all |
1116 | // loads in the group to the location of the first load in program order. When |
1117 | // generating code for an interleaved store group, we sink all stores to the |
1118 | // location of the last store. This code motion can change the order of load |
1119 | // and store instructions and may break dependences. |
1120 | // |
1121 | // The code generation strategy mentioned above ensures that we won't violate |
1122 | // any write-after-read (WAR) dependences. |
1123 | // |
1124 | // E.g., for the WAR dependence: a = A[i]; // (1) |
1125 | // A[i] = b; // (2) |
1126 | // |
1127 | // The store group of (2) is always inserted at or below (2), and the load |
1128 | // group of (1) is always inserted at or above (1). Thus, the instructions will |
1129 | // never be reordered. All other dependences are checked to ensure the |
1130 | // correctness of the instruction reordering. |
1131 | // |
1132 | // The algorithm visits all memory accesses in the loop in bottom-up program |
1133 | // order. Program order is established by traversing the blocks in the loop in |
1134 | // reverse postorder when collecting the accesses. |
1135 | // |
1136 | // We visit the memory accesses in bottom-up order because it can simplify the |
1137 | // construction of store groups in the presence of write-after-write (WAW) |
1138 | // dependences. |
1139 | // |
1140 | // E.g., for the WAW dependence: A[i] = a; // (1) |
1141 | // A[i] = b; // (2) |
1142 | // A[i + 1] = c; // (3) |
1143 | // |
1144 | // We will first create a store group with (3) and (2). (1) can't be added to |
1145 | // this group because it and (2) are dependent. However, (1) can be grouped |
1146 | // with other accesses that may precede it in program order. Note that a |
1147 | // bottom-up order does not imply that WAW dependences should not be checked. |
1148 | void InterleavedAccessInfo::analyzeInterleaving( |
1149 | bool EnablePredicatedInterleavedMemAccesses) { |
1150 | LLVM_DEBUG(dbgs() << "LV: Analyzing interleaved accesses...\n" ); |
1151 | const auto &Strides = LAI->getSymbolicStrides(); |
1152 | |
1153 | // Holds all accesses with a constant stride. |
1154 | MapVector<Instruction *, StrideDescriptor> AccessStrideInfo; |
1155 | collectConstStrideAccesses(AccessStrideInfo, Strides); |
1156 | |
1157 | if (AccessStrideInfo.empty()) |
1158 | return; |
1159 | |
1160 | // Collect the dependences in the loop. |
1161 | collectDependences(); |
1162 | |
1163 | // Holds all interleaved store groups temporarily. |
1164 | SmallSetVector<InterleaveGroup<Instruction> *, 4> StoreGroups; |
1165 | // Holds all interleaved load groups temporarily. |
1166 | SmallSetVector<InterleaveGroup<Instruction> *, 4> LoadGroups; |
1167 | // Groups added to this set cannot have new members added. |
1168 | SmallPtrSet<InterleaveGroup<Instruction> *, 4> CompletedLoadGroups; |
1169 | |
1170 | // Search in bottom-up program order for pairs of accesses (A and B) that can |
1171 | // form interleaved load or store groups. In the algorithm below, access A |
1172 | // precedes access B in program order. We initialize a group for B in the |
1173 | // outer loop of the algorithm, and then in the inner loop, we attempt to |
1174 | // insert each A into B's group if: |
1175 | // |
1176 | // 1. A and B have the same stride, |
1177 | // 2. A and B have the same memory object size, and |
1178 | // 3. A belongs in B's group according to its distance from B. |
1179 | // |
1180 | // Special care is taken to ensure group formation will not break any |
1181 | // dependences. |
1182 | for (auto BI = AccessStrideInfo.rbegin(), E = AccessStrideInfo.rend(); |
1183 | BI != E; ++BI) { |
1184 | Instruction *B = BI->first; |
1185 | StrideDescriptor DesB = BI->second; |
1186 | |
1187 | // Initialize a group for B if it has an allowable stride. Even if we don't |
1188 | // create a group for B, we continue with the bottom-up algorithm to ensure |
1189 | // we don't break any of B's dependences. |
1190 | InterleaveGroup<Instruction> *GroupB = nullptr; |
1191 | if (isStrided(Stride: DesB.Stride) && |
1192 | (!isPredicated(BB: B->getParent()) || EnablePredicatedInterleavedMemAccesses)) { |
1193 | GroupB = getInterleaveGroup(Instr: B); |
1194 | if (!GroupB) { |
1195 | LLVM_DEBUG(dbgs() << "LV: Creating an interleave group with:" << *B |
1196 | << '\n'); |
1197 | GroupB = createInterleaveGroup(Instr: B, Stride: DesB.Stride, Alignment: DesB.Alignment); |
1198 | if (B->mayWriteToMemory()) |
1199 | StoreGroups.insert(X: GroupB); |
1200 | else |
1201 | LoadGroups.insert(X: GroupB); |
1202 | } |
1203 | } |
1204 | |
1205 | for (auto AI = std::next(x: BI); AI != E; ++AI) { |
1206 | Instruction *A = AI->first; |
1207 | StrideDescriptor DesA = AI->second; |
1208 | |
1209 | // Our code motion strategy implies that we can't have dependences |
1210 | // between accesses in an interleaved group and other accesses located |
1211 | // between the first and last member of the group. Note that this also |
1212 | // means that a group can't have more than one member at a given offset. |
1213 | // The accesses in a group can have dependences with other accesses, but |
1214 | // we must ensure we don't extend the boundaries of the group such that |
1215 | // we encompass those dependent accesses. |
1216 | // |
1217 | // For example, assume we have the sequence of accesses shown below in a |
1218 | // stride-2 loop: |
1219 | // |
1220 | // (1, 2) is a group | A[i] = a; // (1) |
1221 | // | A[i-1] = b; // (2) | |
1222 | // A[i-3] = c; // (3) |
1223 | // A[i] = d; // (4) | (2, 4) is not a group |
1224 | // |
1225 | // Because accesses (2) and (3) are dependent, we can group (2) with (1) |
1226 | // but not with (4). If we did, the dependent access (3) would be within |
1227 | // the boundaries of the (2, 4) group. |
1228 | auto DependentMember = [&](InterleaveGroup<Instruction> *Group, |
1229 | StrideEntry *A) -> Instruction * { |
1230 | for (uint32_t Index = 0; Index < Group->getFactor(); ++Index) { |
1231 | Instruction *MemberOfGroupB = Group->getMember(Index); |
1232 | if (MemberOfGroupB && !canReorderMemAccessesForInterleavedGroups( |
1233 | A, B: &*AccessStrideInfo.find(Key: MemberOfGroupB))) |
1234 | return MemberOfGroupB; |
1235 | } |
1236 | return nullptr; |
1237 | }; |
1238 | |
1239 | auto GroupA = getInterleaveGroup(Instr: A); |
1240 | // If A is a load, dependencies are tolerable, there's nothing to do here. |
1241 | // If both A and B belong to the same (store) group, they are independent, |
1242 | // even if dependencies have not been recorded. |
1243 | // If both GroupA and GroupB are null, there's nothing to do here. |
1244 | if (A->mayWriteToMemory() && GroupA != GroupB) { |
1245 | Instruction *DependentInst = nullptr; |
1246 | // If GroupB is a load group, we have to compare AI against all |
1247 | // members of GroupB because if any load within GroupB has a dependency |
1248 | // on AI, we need to mark GroupB as complete and also release the |
1249 | // store GroupA (if A belongs to one). The former prevents incorrect |
1250 | // hoisting of load B above store A while the latter prevents incorrect |
1251 | // sinking of store A below load B. |
1252 | if (GroupB && LoadGroups.contains(key: GroupB)) |
1253 | DependentInst = DependentMember(GroupB, &*AI); |
1254 | else if (!canReorderMemAccessesForInterleavedGroups(A: &*AI, B: &*BI)) |
1255 | DependentInst = B; |
1256 | |
1257 | if (DependentInst) { |
1258 | // A has a store dependence on B (or on some load within GroupB) and |
1259 | // is part of a store group. Release A's group to prevent illegal |
1260 | // sinking of A below B. A will then be free to form another group |
1261 | // with instructions that precede it. |
1262 | if (GroupA && StoreGroups.contains(key: GroupA)) { |
1263 | LLVM_DEBUG(dbgs() << "LV: Invalidated store group due to " |
1264 | "dependence between " |
1265 | << *A << " and " << *DependentInst << '\n'); |
1266 | StoreGroups.remove(X: GroupA); |
1267 | releaseGroup(Group: GroupA); |
1268 | } |
1269 | // If B is a load and part of an interleave group, no earlier loads |
1270 | // can be added to B's interleave group, because this would mean the |
1271 | // DependentInst would move across store A. Mark the interleave group |
1272 | // as complete. |
1273 | if (GroupB && LoadGroups.contains(key: GroupB)) { |
1274 | LLVM_DEBUG(dbgs() << "LV: Marking interleave group for " << *B |
1275 | << " as complete.\n" ); |
1276 | CompletedLoadGroups.insert(Ptr: GroupB); |
1277 | } |
1278 | } |
1279 | } |
1280 | if (CompletedLoadGroups.contains(Ptr: GroupB)) { |
1281 | // Skip trying to add A to B, continue to look for other conflicting A's |
1282 | // in groups to be released. |
1283 | continue; |
1284 | } |
1285 | |
1286 | // At this point, we've checked for illegal code motion. If either A or B |
1287 | // isn't strided, there's nothing left to do. |
1288 | if (!isStrided(Stride: DesA.Stride) || !isStrided(Stride: DesB.Stride)) |
1289 | continue; |
1290 | |
1291 | // Ignore A if it's already in a group or isn't the same kind of memory |
1292 | // operation as B. |
1293 | // Note that mayReadFromMemory() isn't mutually exclusive to |
1294 | // mayWriteToMemory in the case of atomic loads. We shouldn't see those |
1295 | // here, canVectorizeMemory() should have returned false - except for the |
1296 | // case we asked for optimization remarks. |
1297 | if (isInterleaved(Instr: A) || |
1298 | (A->mayReadFromMemory() != B->mayReadFromMemory()) || |
1299 | (A->mayWriteToMemory() != B->mayWriteToMemory())) |
1300 | continue; |
1301 | |
1302 | // Check rules 1 and 2. Ignore A if its stride or size is different from |
1303 | // that of B. |
1304 | if (DesA.Stride != DesB.Stride || DesA.Size != DesB.Size) |
1305 | continue; |
1306 | |
1307 | // Ignore A if the memory object of A and B don't belong to the same |
1308 | // address space |
1309 | if (getLoadStoreAddressSpace(I: A) != getLoadStoreAddressSpace(I: B)) |
1310 | continue; |
1311 | |
1312 | // Calculate the distance from A to B. |
1313 | const SCEVConstant *DistToB = dyn_cast<SCEVConstant>( |
1314 | Val: PSE.getSE()->getMinusSCEV(LHS: DesA.Scev, RHS: DesB.Scev)); |
1315 | if (!DistToB) |
1316 | continue; |
1317 | int64_t DistanceToB = DistToB->getAPInt().getSExtValue(); |
1318 | |
1319 | // Check rule 3. Ignore A if its distance to B is not a multiple of the |
1320 | // size. |
1321 | if (DistanceToB % static_cast<int64_t>(DesB.Size)) |
1322 | continue; |
1323 | |
1324 | // All members of a predicated interleave-group must have the same predicate, |
1325 | // and currently must reside in the same BB. |
1326 | BasicBlock *BlockA = A->getParent(); |
1327 | BasicBlock *BlockB = B->getParent(); |
1328 | if ((isPredicated(BB: BlockA) || isPredicated(BB: BlockB)) && |
1329 | (!EnablePredicatedInterleavedMemAccesses || BlockA != BlockB)) |
1330 | continue; |
1331 | |
1332 | // The index of A is the index of B plus A's distance to B in multiples |
1333 | // of the size. |
1334 | int IndexA = |
1335 | GroupB->getIndex(Instr: B) + DistanceToB / static_cast<int64_t>(DesB.Size); |
1336 | |
1337 | // Try to insert A into B's group. |
1338 | if (GroupB->insertMember(Instr: A, Index: IndexA, NewAlign: DesA.Alignment)) { |
1339 | LLVM_DEBUG(dbgs() << "LV: Inserted:" << *A << '\n' |
1340 | << " into the interleave group with" << *B |
1341 | << '\n'); |
1342 | InterleaveGroupMap[A] = GroupB; |
1343 | |
1344 | // Set the first load in program order as the insert position. |
1345 | if (A->mayReadFromMemory()) |
1346 | GroupB->setInsertPos(A); |
1347 | } |
1348 | } // Iteration over A accesses. |
1349 | } // Iteration over B accesses. |
1350 | |
1351 | auto InvalidateGroupIfMemberMayWrap = [&](InterleaveGroup<Instruction> *Group, |
1352 | int Index, |
1353 | std::string FirstOrLast) -> bool { |
1354 | Instruction *Member = Group->getMember(Index); |
1355 | assert(Member && "Group member does not exist" ); |
1356 | Value *MemberPtr = getLoadStorePointerOperand(V: Member); |
1357 | Type *AccessTy = getLoadStoreType(I: Member); |
1358 | if (getPtrStride(PSE, AccessTy, Ptr: MemberPtr, Lp: TheLoop, StridesMap: Strides, |
1359 | /*Assume=*/false, /*ShouldCheckWrap=*/true).value_or(u: 0)) |
1360 | return false; |
1361 | LLVM_DEBUG(dbgs() << "LV: Invalidate candidate interleaved group due to " |
1362 | << FirstOrLast |
1363 | << " group member potentially pointer-wrapping.\n" ); |
1364 | releaseGroup(Group); |
1365 | return true; |
1366 | }; |
1367 | |
1368 | // Remove interleaved groups with gaps whose memory |
1369 | // accesses may wrap around. We have to revisit the getPtrStride analysis, |
1370 | // this time with ShouldCheckWrap=true, since collectConstStrideAccesses does |
1371 | // not check wrapping (see documentation there). |
1372 | // FORNOW we use Assume=false; |
1373 | // TODO: Change to Assume=true but making sure we don't exceed the threshold |
1374 | // of runtime SCEV assumptions checks (thereby potentially failing to |
1375 | // vectorize altogether). |
1376 | // Additional optional optimizations: |
1377 | // TODO: If we are peeling the loop and we know that the first pointer doesn't |
1378 | // wrap then we can deduce that all pointers in the group don't wrap. |
1379 | // This means that we can forcefully peel the loop in order to only have to |
1380 | // check the first pointer for no-wrap. When we'll change to use Assume=true |
1381 | // we'll only need at most one runtime check per interleaved group. |
1382 | for (auto *Group : LoadGroups) { |
1383 | // Case 1: A full group. Can Skip the checks; For full groups, if the wide |
1384 | // load would wrap around the address space we would do a memory access at |
1385 | // nullptr even without the transformation. |
1386 | if (Group->getNumMembers() == Group->getFactor()) |
1387 | continue; |
1388 | |
1389 | // Case 2: If first and last members of the group don't wrap this implies |
1390 | // that all the pointers in the group don't wrap. |
1391 | // So we check only group member 0 (which is always guaranteed to exist), |
1392 | // and group member Factor - 1; If the latter doesn't exist we rely on |
1393 | // peeling (if it is a non-reversed accsess -- see Case 3). |
1394 | if (InvalidateGroupIfMemberMayWrap(Group, 0, std::string("first" ))) |
1395 | continue; |
1396 | if (Group->getMember(Index: Group->getFactor() - 1)) |
1397 | InvalidateGroupIfMemberMayWrap(Group, Group->getFactor() - 1, |
1398 | std::string("last" )); |
1399 | else { |
1400 | // Case 3: A non-reversed interleaved load group with gaps: We need |
1401 | // to execute at least one scalar epilogue iteration. This will ensure |
1402 | // we don't speculatively access memory out-of-bounds. We only need |
1403 | // to look for a member at index factor - 1, since every group must have |
1404 | // a member at index zero. |
1405 | if (Group->isReverse()) { |
1406 | LLVM_DEBUG( |
1407 | dbgs() << "LV: Invalidate candidate interleaved group due to " |
1408 | "a reverse access with gaps.\n" ); |
1409 | releaseGroup(Group); |
1410 | continue; |
1411 | } |
1412 | LLVM_DEBUG( |
1413 | dbgs() << "LV: Interleaved group requires epilogue iteration.\n" ); |
1414 | RequiresScalarEpilogue = true; |
1415 | } |
1416 | } |
1417 | |
1418 | for (auto *Group : StoreGroups) { |
1419 | // Case 1: A full group. Can Skip the checks; For full groups, if the wide |
1420 | // store would wrap around the address space we would do a memory access at |
1421 | // nullptr even without the transformation. |
1422 | if (Group->getNumMembers() == Group->getFactor()) |
1423 | continue; |
1424 | |
1425 | // Interleave-store-group with gaps is implemented using masked wide store. |
1426 | // Remove interleaved store groups with gaps if |
1427 | // masked-interleaved-accesses are not enabled by the target. |
1428 | if (!EnablePredicatedInterleavedMemAccesses) { |
1429 | LLVM_DEBUG( |
1430 | dbgs() << "LV: Invalidate candidate interleaved store group due " |
1431 | "to gaps.\n" ); |
1432 | releaseGroup(Group); |
1433 | continue; |
1434 | } |
1435 | |
1436 | // Case 2: If first and last members of the group don't wrap this implies |
1437 | // that all the pointers in the group don't wrap. |
1438 | // So we check only group member 0 (which is always guaranteed to exist), |
1439 | // and the last group member. Case 3 (scalar epilog) is not relevant for |
1440 | // stores with gaps, which are implemented with masked-store (rather than |
1441 | // speculative access, as in loads). |
1442 | if (InvalidateGroupIfMemberMayWrap(Group, 0, std::string("first" ))) |
1443 | continue; |
1444 | for (int Index = Group->getFactor() - 1; Index > 0; Index--) |
1445 | if (Group->getMember(Index)) { |
1446 | InvalidateGroupIfMemberMayWrap(Group, Index, std::string("last" )); |
1447 | break; |
1448 | } |
1449 | } |
1450 | } |
1451 | |
1452 | void InterleavedAccessInfo::invalidateGroupsRequiringScalarEpilogue() { |
1453 | // If no group had triggered the requirement to create an epilogue loop, |
1454 | // there is nothing to do. |
1455 | if (!requiresScalarEpilogue()) |
1456 | return; |
1457 | |
1458 | bool ReleasedGroup = false; |
1459 | // Release groups requiring scalar epilogues. Note that this also removes them |
1460 | // from InterleaveGroups. |
1461 | for (auto *Group : make_early_inc_range(Range&: InterleaveGroups)) { |
1462 | if (!Group->requiresScalarEpilogue()) |
1463 | continue; |
1464 | LLVM_DEBUG( |
1465 | dbgs() |
1466 | << "LV: Invalidate candidate interleaved group due to gaps that " |
1467 | "require a scalar epilogue (not allowed under optsize) and cannot " |
1468 | "be masked (not enabled). \n" ); |
1469 | releaseGroup(Group); |
1470 | ReleasedGroup = true; |
1471 | } |
1472 | assert(ReleasedGroup && "At least one group must be invalidated, as a " |
1473 | "scalar epilogue was required" ); |
1474 | (void)ReleasedGroup; |
1475 | RequiresScalarEpilogue = false; |
1476 | } |
1477 | |
1478 | template <typename InstT> |
1479 | void InterleaveGroup<InstT>::addMetadata(InstT *NewInst) const { |
1480 | llvm_unreachable("addMetadata can only be used for Instruction" ); |
1481 | } |
1482 | |
1483 | namespace llvm { |
1484 | template <> |
1485 | void InterleaveGroup<Instruction>::addMetadata(Instruction *NewInst) const { |
1486 | SmallVector<Value *, 4> VL; |
1487 | std::transform(first: Members.begin(), last: Members.end(), result: std::back_inserter(x&: VL), |
1488 | unary_op: [](std::pair<int, Instruction *> p) { return p.second; }); |
1489 | propagateMetadata(Inst: NewInst, VL); |
1490 | } |
1491 | } // namespace llvm |
1492 | |