1 | //===- CombinerHelperVectorOps.cpp-----------------------------------------===// |
2 | // |
3 | // Part of the LLVM Project, under the Apache License v2.0 with LLVM Exceptions. |
4 | // See https://llvm.org/LICENSE.txt for license information. |
5 | // SPDX-License-Identifier: Apache-2.0 WITH LLVM-exception |
6 | // |
7 | //===----------------------------------------------------------------------===// |
8 | // |
9 | // This file implements CombinerHelper for G_EXTRACT_VECTOR_ELT. |
10 | // |
11 | //===----------------------------------------------------------------------===// |
12 | #include "llvm/CodeGen/GlobalISel/CombinerHelper.h" |
13 | #include "llvm/CodeGen/GlobalISel/GenericMachineInstrs.h" |
14 | #include "llvm/CodeGen/GlobalISel/LegalizerHelper.h" |
15 | #include "llvm/CodeGen/GlobalISel/LegalizerInfo.h" |
16 | #include "llvm/CodeGen/GlobalISel/MIPatternMatch.h" |
17 | #include "llvm/CodeGen/GlobalISel/MachineIRBuilder.h" |
18 | #include "llvm/CodeGen/GlobalISel/Utils.h" |
19 | #include "llvm/CodeGen/LowLevelTypeUtils.h" |
20 | #include "llvm/CodeGen/MachineOperand.h" |
21 | #include "llvm/CodeGen/MachineRegisterInfo.h" |
22 | #include "llvm/CodeGen/TargetLowering.h" |
23 | #include "llvm/CodeGen/TargetOpcodes.h" |
24 | #include "llvm/Support/Casting.h" |
25 | #include <optional> |
26 | |
27 | #define DEBUG_TYPE "gi-combiner" |
28 | |
29 | using namespace llvm; |
30 | using namespace MIPatternMatch; |
31 | |
32 | bool CombinerHelper::(MachineInstr &MI, |
33 | BuildFnTy &MatchInfo) { |
34 | GExtractVectorElement * = cast<GExtractVectorElement>(Val: &MI); |
35 | |
36 | Register Dst = Extract->getReg(Idx: 0); |
37 | Register Vector = Extract->getVectorReg(); |
38 | Register Index = Extract->getIndexReg(); |
39 | LLT DstTy = MRI.getType(Reg: Dst); |
40 | LLT VectorTy = MRI.getType(Reg: Vector); |
41 | |
42 | // The vector register can be def'd by various ops that have vector as its |
43 | // type. They can all be used for constant folding, scalarizing, |
44 | // canonicalization, or combining based on symmetry. |
45 | // |
46 | // vector like ops |
47 | // * build vector |
48 | // * build vector trunc |
49 | // * shuffle vector |
50 | // * splat vector |
51 | // * concat vectors |
52 | // * insert/extract vector element |
53 | // * insert/extract subvector |
54 | // * vector loads |
55 | // * scalable vector loads |
56 | // |
57 | // compute like ops |
58 | // * binary ops |
59 | // * unary ops |
60 | // * exts and truncs |
61 | // * casts |
62 | // * fneg |
63 | // * select |
64 | // * phis |
65 | // * cmps |
66 | // * freeze |
67 | // * bitcast |
68 | // * undef |
69 | |
70 | // We try to get the value of the Index register. |
71 | std::optional<ValueAndVReg> MaybeIndex = |
72 | getIConstantVRegValWithLookThrough(VReg: Index, MRI); |
73 | std::optional<APInt> IndexC = std::nullopt; |
74 | |
75 | if (MaybeIndex) |
76 | IndexC = MaybeIndex->Value; |
77 | |
78 | // Fold extractVectorElement(Vector, TOOLARGE) -> undef |
79 | if (IndexC && VectorTy.isFixedVector() && |
80 | IndexC->getZExtValue() >= VectorTy.getNumElements() && |
81 | isLegalOrBeforeLegalizer(Query: {TargetOpcode::G_IMPLICIT_DEF, {DstTy}})) { |
82 | // For fixed-length vectors, it's invalid to extract out-of-range elements. |
83 | MatchInfo = [=](MachineIRBuilder &B) { B.buildUndef(Res: Dst); }; |
84 | return true; |
85 | } |
86 | |
87 | return false; |
88 | } |
89 | |
90 | bool CombinerHelper::matchExtractVectorElementWithDifferentIndices( |
91 | const MachineOperand &MO, BuildFnTy &MatchInfo) { |
92 | MachineInstr *Root = getDefIgnoringCopies(Reg: MO.getReg(), MRI); |
93 | GExtractVectorElement * = cast<GExtractVectorElement>(Val: Root); |
94 | |
95 | // |
96 | // %idx1:_(s64) = G_CONSTANT i64 1 |
97 | // %idx2:_(s64) = G_CONSTANT i64 2 |
98 | // %insert:_(<2 x s32>) = G_INSERT_VECTOR_ELT_ELT %bv(<2 x s32>), |
99 | // %value(s32), %idx2(s64) %extract:_(s32) = G_EXTRACT_VECTOR_ELT %insert(<2 |
100 | // x s32>), %idx1(s64) |
101 | // |
102 | // --> |
103 | // |
104 | // %insert:_(<2 x s32>) = G_INSERT_VECTOR_ELT_ELT %bv(<2 x s32>), |
105 | // %value(s32), %idx2(s64) %extract:_(s32) = G_EXTRACT_VECTOR_ELT %bv(<2 x |
106 | // s32>), %idx1(s64) |
107 | // |
108 | // |
109 | |
110 | Register Index = Extract->getIndexReg(); |
111 | |
112 | // We try to get the value of the Index register. |
113 | std::optional<ValueAndVReg> MaybeIndex = |
114 | getIConstantVRegValWithLookThrough(VReg: Index, MRI); |
115 | std::optional<APInt> IndexC = std::nullopt; |
116 | |
117 | if (!MaybeIndex) |
118 | return false; |
119 | else |
120 | IndexC = MaybeIndex->Value; |
121 | |
122 | Register Vector = Extract->getVectorReg(); |
123 | |
124 | GInsertVectorElement *Insert = |
125 | getOpcodeDef<GInsertVectorElement>(Reg: Vector, MRI); |
126 | if (!Insert) |
127 | return false; |
128 | |
129 | Register Dst = Extract->getReg(Idx: 0); |
130 | |
131 | std::optional<ValueAndVReg> MaybeInsertIndex = |
132 | getIConstantVRegValWithLookThrough(VReg: Insert->getIndexReg(), MRI); |
133 | |
134 | if (MaybeInsertIndex && MaybeInsertIndex->Value != *IndexC) { |
135 | // There is no one-use check. We have to keep the insert. When both Index |
136 | // registers are constants and not equal, we can look into the Vector |
137 | // register of the insert. |
138 | MatchInfo = [=](MachineIRBuilder &B) { |
139 | B.buildExtractVectorElement(Res: Dst, Val: Insert->getVectorReg(), Idx: Index); |
140 | }; |
141 | return true; |
142 | } |
143 | |
144 | return false; |
145 | } |
146 | |
147 | bool CombinerHelper::matchExtractVectorElementWithFreeze( |
148 | const MachineOperand &MO, BuildFnTy &MatchInfo) { |
149 | MachineInstr *Root = getDefIgnoringCopies(Reg: MO.getReg(), MRI); |
150 | GExtractVectorElement * = cast<GExtractVectorElement>(Val: Root); |
151 | |
152 | Register Vector = Extract->getVectorReg(); |
153 | |
154 | // |
155 | // %bv:_(<2 x s32>) = G_BUILD_VECTOR %arg1(s32), %arg2(s32) |
156 | // %freeze:_(<2 x s32>) = G_FREEZE %bv(<2 x s32>) |
157 | // %extract:_(s32) = G_EXTRACT_VECTOR_ELT %bv(<2 x s32>), %opaque(s64) |
158 | // |
159 | // --> |
160 | // |
161 | // %bv:_(<2 x s32>) = G_BUILD_VECTOR %arg1(s32), %arg2(s32) |
162 | // %extract:_(s32) = G_EXTRACT_VECTOR_ELT %bv(<2 x s32>), %opaque(s64) |
163 | // %freeze:_(s32) = G_FREEZE %extract(s32) |
164 | // |
165 | // |
166 | |
167 | // For G_FREEZE, the input and the output types are identical. Moving the |
168 | // freeze from the Vector into the front of the extract preserves the freeze |
169 | // semantics. The result is still freeze'd. Furthermore, the Vector register |
170 | // becomes easier to analyze. A build vector could have been hidden behind the |
171 | // freeze. |
172 | |
173 | // We expect a freeze on the Vector register. |
174 | GFreeze *Freeze = getOpcodeDef<GFreeze>(Reg: Vector, MRI); |
175 | if (!Freeze) |
176 | return false; |
177 | |
178 | Register Dst = Extract->getReg(Idx: 0); |
179 | LLT DstTy = MRI.getType(Reg: Dst); |
180 | |
181 | // We first have to check for one-use and legality of the freeze. |
182 | // The type of the extractVectorElement did not change. |
183 | if (!MRI.hasOneNonDBGUse(RegNo: Freeze->getReg(Idx: 0)) || |
184 | !isLegalOrBeforeLegalizer(Query: {TargetOpcode::G_FREEZE, {DstTy}})) |
185 | return false; |
186 | |
187 | Register Index = Extract->getIndexReg(); |
188 | |
189 | // We move the freeze from the Vector register in front of the |
190 | // extractVectorElement. |
191 | MatchInfo = [=](MachineIRBuilder &B) { |
192 | auto = |
193 | B.buildExtractVectorElement(Res: DstTy, Val: Freeze->getSourceReg(), Idx: Index); |
194 | B.buildFreeze(Dst, Src: Extract); |
195 | }; |
196 | |
197 | return true; |
198 | } |
199 | |
200 | bool CombinerHelper::matchExtractVectorElementWithBuildVector( |
201 | const MachineOperand &MO, BuildFnTy &MatchInfo) { |
202 | MachineInstr *Root = getDefIgnoringCopies(Reg: MO.getReg(), MRI); |
203 | GExtractVectorElement * = cast<GExtractVectorElement>(Val: Root); |
204 | |
205 | // |
206 | // %zero:_(s64) = G_CONSTANT i64 0 |
207 | // %bv:_(<2 x s32>) = G_BUILD_VECTOR %arg1(s32), %arg2(s32) |
208 | // %extract:_(s32) = G_EXTRACT_VECTOR_ELT %bv(<2 x s32>), %zero(s64) |
209 | // |
210 | // --> |
211 | // |
212 | // %extract:_(32) = COPY %arg1(s32) |
213 | // |
214 | // |
215 | // |
216 | // %bv:_(<2 x s32>) = G_BUILD_VECTOR %arg1(s32), %arg2(s32) |
217 | // %extract:_(s32) = G_EXTRACT_VECTOR_ELT %bv(<2 x s32>), %opaque(s64) |
218 | // |
219 | // --> |
220 | // |
221 | // %bv:_(<2 x s32>) = G_BUILD_VECTOR %arg1(s32), %arg2(s32) |
222 | // %extract:_(s32) = G_EXTRACT_VECTOR_ELT %bv(<2 x s32>), %opaque(s64) |
223 | // |
224 | |
225 | Register Vector = Extract->getVectorReg(); |
226 | |
227 | // We expect a buildVector on the Vector register. |
228 | GBuildVector *Build = getOpcodeDef<GBuildVector>(Reg: Vector, MRI); |
229 | if (!Build) |
230 | return false; |
231 | |
232 | LLT VectorTy = MRI.getType(Reg: Vector); |
233 | |
234 | // There is a one-use check. There are more combines on build vectors. |
235 | EVT Ty(getMVTForLLT(Ty: VectorTy)); |
236 | if (!MRI.hasOneNonDBGUse(RegNo: Build->getReg(Idx: 0)) || |
237 | !getTargetLowering().aggressivelyPreferBuildVectorSources(VecVT: Ty)) |
238 | return false; |
239 | |
240 | Register Index = Extract->getIndexReg(); |
241 | |
242 | // If the Index is constant, then we can extract the element from the given |
243 | // offset. |
244 | std::optional<ValueAndVReg> MaybeIndex = |
245 | getIConstantVRegValWithLookThrough(VReg: Index, MRI); |
246 | if (!MaybeIndex) |
247 | return false; |
248 | |
249 | // We now know that there is a buildVector def'd on the Vector register and |
250 | // the index is const. The combine will succeed. |
251 | |
252 | Register Dst = Extract->getReg(Idx: 0); |
253 | |
254 | MatchInfo = [=](MachineIRBuilder &B) { |
255 | B.buildCopy(Res: Dst, Op: Build->getSourceReg(I: MaybeIndex->Value.getZExtValue())); |
256 | }; |
257 | |
258 | return true; |
259 | } |
260 | |
261 | bool CombinerHelper::matchExtractVectorElementWithBuildVectorTrunc( |
262 | const MachineOperand &MO, BuildFnTy &MatchInfo) { |
263 | MachineInstr *Root = getDefIgnoringCopies(Reg: MO.getReg(), MRI); |
264 | GExtractVectorElement * = cast<GExtractVectorElement>(Val: Root); |
265 | |
266 | // |
267 | // %zero:_(s64) = G_CONSTANT i64 0 |
268 | // %bv:_(<2 x s32>) = G_BUILD_VECTOR_TRUNC %arg1(s64), %arg2(s64) |
269 | // %extract:_(s32) = G_EXTRACT_VECTOR_ELT %bv(<2 x s32>), %zero(s64) |
270 | // |
271 | // --> |
272 | // |
273 | // %extract:_(32) = G_TRUNC %arg1(s64) |
274 | // |
275 | // |
276 | // |
277 | // %bv:_(<2 x s32>) = G_BUILD_VECTOR_TRUNC %arg1(s64), %arg2(s64) |
278 | // %extract:_(s32) = G_EXTRACT_VECTOR_ELT %bv(<2 x s32>), %opaque(s64) |
279 | // |
280 | // --> |
281 | // |
282 | // %bv:_(<2 x s32>) = G_BUILD_VECTOR_TRUNC %arg1(s64), %arg2(s64) |
283 | // %extract:_(s32) = G_EXTRACT_VECTOR_ELT %bv(<2 x s32>), %opaque(s64) |
284 | // |
285 | |
286 | Register Vector = Extract->getVectorReg(); |
287 | |
288 | // We expect a buildVectorTrunc on the Vector register. |
289 | GBuildVectorTrunc *Build = getOpcodeDef<GBuildVectorTrunc>(Reg: Vector, MRI); |
290 | if (!Build) |
291 | return false; |
292 | |
293 | LLT VectorTy = MRI.getType(Reg: Vector); |
294 | |
295 | // There is a one-use check. There are more combines on build vectors. |
296 | EVT Ty(getMVTForLLT(Ty: VectorTy)); |
297 | if (!MRI.hasOneNonDBGUse(RegNo: Build->getReg(Idx: 0)) || |
298 | !getTargetLowering().aggressivelyPreferBuildVectorSources(VecVT: Ty)) |
299 | return false; |
300 | |
301 | Register Index = Extract->getIndexReg(); |
302 | |
303 | // If the Index is constant, then we can extract the element from the given |
304 | // offset. |
305 | std::optional<ValueAndVReg> MaybeIndex = |
306 | getIConstantVRegValWithLookThrough(VReg: Index, MRI); |
307 | if (!MaybeIndex) |
308 | return false; |
309 | |
310 | // We now know that there is a buildVectorTrunc def'd on the Vector register |
311 | // and the index is const. The combine will succeed. |
312 | |
313 | Register Dst = Extract->getReg(Idx: 0); |
314 | LLT DstTy = MRI.getType(Reg: Dst); |
315 | LLT SrcTy = MRI.getType(Reg: Build->getSourceReg(I: 0)); |
316 | |
317 | // For buildVectorTrunc, the inputs are truncated. |
318 | if (!isLegalOrBeforeLegalizer(Query: {TargetOpcode::G_TRUNC, {DstTy, SrcTy}})) |
319 | return false; |
320 | |
321 | MatchInfo = [=](MachineIRBuilder &B) { |
322 | B.buildTrunc(Res: Dst, Op: Build->getSourceReg(I: MaybeIndex->Value.getZExtValue())); |
323 | }; |
324 | |
325 | return true; |
326 | } |
327 | |