1//===- CombinerHelperVectorOps.cpp-----------------------------------------===//
2//
3// Part of the LLVM Project, under the Apache License v2.0 with LLVM Exceptions.
4// See https://llvm.org/LICENSE.txt for license information.
5// SPDX-License-Identifier: Apache-2.0 WITH LLVM-exception
6//
7//===----------------------------------------------------------------------===//
8//
9// This file implements CombinerHelper for G_EXTRACT_VECTOR_ELT.
10//
11//===----------------------------------------------------------------------===//
12#include "llvm/CodeGen/GlobalISel/CombinerHelper.h"
13#include "llvm/CodeGen/GlobalISel/GenericMachineInstrs.h"
14#include "llvm/CodeGen/GlobalISel/LegalizerHelper.h"
15#include "llvm/CodeGen/GlobalISel/LegalizerInfo.h"
16#include "llvm/CodeGen/GlobalISel/MIPatternMatch.h"
17#include "llvm/CodeGen/GlobalISel/MachineIRBuilder.h"
18#include "llvm/CodeGen/GlobalISel/Utils.h"
19#include "llvm/CodeGen/LowLevelTypeUtils.h"
20#include "llvm/CodeGen/MachineOperand.h"
21#include "llvm/CodeGen/MachineRegisterInfo.h"
22#include "llvm/CodeGen/TargetLowering.h"
23#include "llvm/CodeGen/TargetOpcodes.h"
24#include "llvm/Support/Casting.h"
25#include <optional>
26
27#define DEBUG_TYPE "gi-combiner"
28
29using namespace llvm;
30using namespace MIPatternMatch;
31
32bool CombinerHelper::matchExtractVectorElement(MachineInstr &MI,
33 BuildFnTy &MatchInfo) {
34 GExtractVectorElement *Extract = cast<GExtractVectorElement>(Val: &MI);
35
36 Register Dst = Extract->getReg(Idx: 0);
37 Register Vector = Extract->getVectorReg();
38 Register Index = Extract->getIndexReg();
39 LLT DstTy = MRI.getType(Reg: Dst);
40 LLT VectorTy = MRI.getType(Reg: Vector);
41
42 // The vector register can be def'd by various ops that have vector as its
43 // type. They can all be used for constant folding, scalarizing,
44 // canonicalization, or combining based on symmetry.
45 //
46 // vector like ops
47 // * build vector
48 // * build vector trunc
49 // * shuffle vector
50 // * splat vector
51 // * concat vectors
52 // * insert/extract vector element
53 // * insert/extract subvector
54 // * vector loads
55 // * scalable vector loads
56 //
57 // compute like ops
58 // * binary ops
59 // * unary ops
60 // * exts and truncs
61 // * casts
62 // * fneg
63 // * select
64 // * phis
65 // * cmps
66 // * freeze
67 // * bitcast
68 // * undef
69
70 // We try to get the value of the Index register.
71 std::optional<ValueAndVReg> MaybeIndex =
72 getIConstantVRegValWithLookThrough(VReg: Index, MRI);
73 std::optional<APInt> IndexC = std::nullopt;
74
75 if (MaybeIndex)
76 IndexC = MaybeIndex->Value;
77
78 // Fold extractVectorElement(Vector, TOOLARGE) -> undef
79 if (IndexC && VectorTy.isFixedVector() &&
80 IndexC->getZExtValue() >= VectorTy.getNumElements() &&
81 isLegalOrBeforeLegalizer(Query: {TargetOpcode::G_IMPLICIT_DEF, {DstTy}})) {
82 // For fixed-length vectors, it's invalid to extract out-of-range elements.
83 MatchInfo = [=](MachineIRBuilder &B) { B.buildUndef(Res: Dst); };
84 return true;
85 }
86
87 return false;
88}
89
90bool CombinerHelper::matchExtractVectorElementWithDifferentIndices(
91 const MachineOperand &MO, BuildFnTy &MatchInfo) {
92 MachineInstr *Root = getDefIgnoringCopies(Reg: MO.getReg(), MRI);
93 GExtractVectorElement *Extract = cast<GExtractVectorElement>(Val: Root);
94
95 //
96 // %idx1:_(s64) = G_CONSTANT i64 1
97 // %idx2:_(s64) = G_CONSTANT i64 2
98 // %insert:_(<2 x s32>) = G_INSERT_VECTOR_ELT_ELT %bv(<2 x s32>),
99 // %value(s32), %idx2(s64) %extract:_(s32) = G_EXTRACT_VECTOR_ELT %insert(<2
100 // x s32>), %idx1(s64)
101 //
102 // -->
103 //
104 // %insert:_(<2 x s32>) = G_INSERT_VECTOR_ELT_ELT %bv(<2 x s32>),
105 // %value(s32), %idx2(s64) %extract:_(s32) = G_EXTRACT_VECTOR_ELT %bv(<2 x
106 // s32>), %idx1(s64)
107 //
108 //
109
110 Register Index = Extract->getIndexReg();
111
112 // We try to get the value of the Index register.
113 std::optional<ValueAndVReg> MaybeIndex =
114 getIConstantVRegValWithLookThrough(VReg: Index, MRI);
115 std::optional<APInt> IndexC = std::nullopt;
116
117 if (!MaybeIndex)
118 return false;
119 else
120 IndexC = MaybeIndex->Value;
121
122 Register Vector = Extract->getVectorReg();
123
124 GInsertVectorElement *Insert =
125 getOpcodeDef<GInsertVectorElement>(Reg: Vector, MRI);
126 if (!Insert)
127 return false;
128
129 Register Dst = Extract->getReg(Idx: 0);
130
131 std::optional<ValueAndVReg> MaybeInsertIndex =
132 getIConstantVRegValWithLookThrough(VReg: Insert->getIndexReg(), MRI);
133
134 if (MaybeInsertIndex && MaybeInsertIndex->Value != *IndexC) {
135 // There is no one-use check. We have to keep the insert. When both Index
136 // registers are constants and not equal, we can look into the Vector
137 // register of the insert.
138 MatchInfo = [=](MachineIRBuilder &B) {
139 B.buildExtractVectorElement(Res: Dst, Val: Insert->getVectorReg(), Idx: Index);
140 };
141 return true;
142 }
143
144 return false;
145}
146
147bool CombinerHelper::matchExtractVectorElementWithFreeze(
148 const MachineOperand &MO, BuildFnTy &MatchInfo) {
149 MachineInstr *Root = getDefIgnoringCopies(Reg: MO.getReg(), MRI);
150 GExtractVectorElement *Extract = cast<GExtractVectorElement>(Val: Root);
151
152 Register Vector = Extract->getVectorReg();
153
154 //
155 // %bv:_(<2 x s32>) = G_BUILD_VECTOR %arg1(s32), %arg2(s32)
156 // %freeze:_(<2 x s32>) = G_FREEZE %bv(<2 x s32>)
157 // %extract:_(s32) = G_EXTRACT_VECTOR_ELT %bv(<2 x s32>), %opaque(s64)
158 //
159 // -->
160 //
161 // %bv:_(<2 x s32>) = G_BUILD_VECTOR %arg1(s32), %arg2(s32)
162 // %extract:_(s32) = G_EXTRACT_VECTOR_ELT %bv(<2 x s32>), %opaque(s64)
163 // %freeze:_(s32) = G_FREEZE %extract(s32)
164 //
165 //
166
167 // For G_FREEZE, the input and the output types are identical. Moving the
168 // freeze from the Vector into the front of the extract preserves the freeze
169 // semantics. The result is still freeze'd. Furthermore, the Vector register
170 // becomes easier to analyze. A build vector could have been hidden behind the
171 // freeze.
172
173 // We expect a freeze on the Vector register.
174 GFreeze *Freeze = getOpcodeDef<GFreeze>(Reg: Vector, MRI);
175 if (!Freeze)
176 return false;
177
178 Register Dst = Extract->getReg(Idx: 0);
179 LLT DstTy = MRI.getType(Reg: Dst);
180
181 // We first have to check for one-use and legality of the freeze.
182 // The type of the extractVectorElement did not change.
183 if (!MRI.hasOneNonDBGUse(RegNo: Freeze->getReg(Idx: 0)) ||
184 !isLegalOrBeforeLegalizer(Query: {TargetOpcode::G_FREEZE, {DstTy}}))
185 return false;
186
187 Register Index = Extract->getIndexReg();
188
189 // We move the freeze from the Vector register in front of the
190 // extractVectorElement.
191 MatchInfo = [=](MachineIRBuilder &B) {
192 auto Extract =
193 B.buildExtractVectorElement(Res: DstTy, Val: Freeze->getSourceReg(), Idx: Index);
194 B.buildFreeze(Dst, Src: Extract);
195 };
196
197 return true;
198}
199
200bool CombinerHelper::matchExtractVectorElementWithBuildVector(
201 const MachineOperand &MO, BuildFnTy &MatchInfo) {
202 MachineInstr *Root = getDefIgnoringCopies(Reg: MO.getReg(), MRI);
203 GExtractVectorElement *Extract = cast<GExtractVectorElement>(Val: Root);
204
205 //
206 // %zero:_(s64) = G_CONSTANT i64 0
207 // %bv:_(<2 x s32>) = G_BUILD_VECTOR %arg1(s32), %arg2(s32)
208 // %extract:_(s32) = G_EXTRACT_VECTOR_ELT %bv(<2 x s32>), %zero(s64)
209 //
210 // -->
211 //
212 // %extract:_(32) = COPY %arg1(s32)
213 //
214 //
215 //
216 // %bv:_(<2 x s32>) = G_BUILD_VECTOR %arg1(s32), %arg2(s32)
217 // %extract:_(s32) = G_EXTRACT_VECTOR_ELT %bv(<2 x s32>), %opaque(s64)
218 //
219 // -->
220 //
221 // %bv:_(<2 x s32>) = G_BUILD_VECTOR %arg1(s32), %arg2(s32)
222 // %extract:_(s32) = G_EXTRACT_VECTOR_ELT %bv(<2 x s32>), %opaque(s64)
223 //
224
225 Register Vector = Extract->getVectorReg();
226
227 // We expect a buildVector on the Vector register.
228 GBuildVector *Build = getOpcodeDef<GBuildVector>(Reg: Vector, MRI);
229 if (!Build)
230 return false;
231
232 LLT VectorTy = MRI.getType(Reg: Vector);
233
234 // There is a one-use check. There are more combines on build vectors.
235 EVT Ty(getMVTForLLT(Ty: VectorTy));
236 if (!MRI.hasOneNonDBGUse(RegNo: Build->getReg(Idx: 0)) ||
237 !getTargetLowering().aggressivelyPreferBuildVectorSources(VecVT: Ty))
238 return false;
239
240 Register Index = Extract->getIndexReg();
241
242 // If the Index is constant, then we can extract the element from the given
243 // offset.
244 std::optional<ValueAndVReg> MaybeIndex =
245 getIConstantVRegValWithLookThrough(VReg: Index, MRI);
246 if (!MaybeIndex)
247 return false;
248
249 // We now know that there is a buildVector def'd on the Vector register and
250 // the index is const. The combine will succeed.
251
252 Register Dst = Extract->getReg(Idx: 0);
253
254 MatchInfo = [=](MachineIRBuilder &B) {
255 B.buildCopy(Res: Dst, Op: Build->getSourceReg(I: MaybeIndex->Value.getZExtValue()));
256 };
257
258 return true;
259}
260
261bool CombinerHelper::matchExtractVectorElementWithBuildVectorTrunc(
262 const MachineOperand &MO, BuildFnTy &MatchInfo) {
263 MachineInstr *Root = getDefIgnoringCopies(Reg: MO.getReg(), MRI);
264 GExtractVectorElement *Extract = cast<GExtractVectorElement>(Val: Root);
265
266 //
267 // %zero:_(s64) = G_CONSTANT i64 0
268 // %bv:_(<2 x s32>) = G_BUILD_VECTOR_TRUNC %arg1(s64), %arg2(s64)
269 // %extract:_(s32) = G_EXTRACT_VECTOR_ELT %bv(<2 x s32>), %zero(s64)
270 //
271 // -->
272 //
273 // %extract:_(32) = G_TRUNC %arg1(s64)
274 //
275 //
276 //
277 // %bv:_(<2 x s32>) = G_BUILD_VECTOR_TRUNC %arg1(s64), %arg2(s64)
278 // %extract:_(s32) = G_EXTRACT_VECTOR_ELT %bv(<2 x s32>), %opaque(s64)
279 //
280 // -->
281 //
282 // %bv:_(<2 x s32>) = G_BUILD_VECTOR_TRUNC %arg1(s64), %arg2(s64)
283 // %extract:_(s32) = G_EXTRACT_VECTOR_ELT %bv(<2 x s32>), %opaque(s64)
284 //
285
286 Register Vector = Extract->getVectorReg();
287
288 // We expect a buildVectorTrunc on the Vector register.
289 GBuildVectorTrunc *Build = getOpcodeDef<GBuildVectorTrunc>(Reg: Vector, MRI);
290 if (!Build)
291 return false;
292
293 LLT VectorTy = MRI.getType(Reg: Vector);
294
295 // There is a one-use check. There are more combines on build vectors.
296 EVT Ty(getMVTForLLT(Ty: VectorTy));
297 if (!MRI.hasOneNonDBGUse(RegNo: Build->getReg(Idx: 0)) ||
298 !getTargetLowering().aggressivelyPreferBuildVectorSources(VecVT: Ty))
299 return false;
300
301 Register Index = Extract->getIndexReg();
302
303 // If the Index is constant, then we can extract the element from the given
304 // offset.
305 std::optional<ValueAndVReg> MaybeIndex =
306 getIConstantVRegValWithLookThrough(VReg: Index, MRI);
307 if (!MaybeIndex)
308 return false;
309
310 // We now know that there is a buildVectorTrunc def'd on the Vector register
311 // and the index is const. The combine will succeed.
312
313 Register Dst = Extract->getReg(Idx: 0);
314 LLT DstTy = MRI.getType(Reg: Dst);
315 LLT SrcTy = MRI.getType(Reg: Build->getSourceReg(I: 0));
316
317 // For buildVectorTrunc, the inputs are truncated.
318 if (!isLegalOrBeforeLegalizer(Query: {TargetOpcode::G_TRUNC, {DstTy, SrcTy}}))
319 return false;
320
321 MatchInfo = [=](MachineIRBuilder &B) {
322 B.buildTrunc(Res: Dst, Op: Build->getSourceReg(I: MaybeIndex->Value.getZExtValue()));
323 };
324
325 return true;
326}
327

source code of llvm/lib/CodeGen/GlobalISel/CombinerHelperVectorOps.cpp