1 | //===- VectorTransformOps.cpp - Implementation of Vector transform ops ----===// |
2 | // |
3 | // Part of the LLVM Project, under the Apache License v2.0 with LLVM Exceptions. |
4 | // See https://llvm.org/LICENSE.txt for license information. |
5 | // SPDX-License-Identifier: Apache-2.0 WITH LLVM-exception |
6 | // |
7 | //===----------------------------------------------------------------------===// |
8 | |
9 | #include "mlir/Dialect/Vector/TransformOps/VectorTransformOps.h" |
10 | |
11 | #include "mlir/Conversion/LLVMCommon/TypeConverter.h" |
12 | #include "mlir/Conversion/VectorToLLVM/ConvertVectorToLLVM.h" |
13 | #include "mlir/Conversion/VectorToSCF/VectorToSCF.h" |
14 | #include "mlir/Dialect/LLVMIR/LLVMDialect.h" |
15 | #include "mlir/Dialect/Transform/IR/TransformDialect.h" |
16 | #include "mlir/Dialect/Transform/IR/TransformOps.h" |
17 | #include "mlir/Dialect/Transform/Interfaces/TransformInterfaces.h" |
18 | #include "mlir/Dialect/Vector/IR/VectorOps.h" |
19 | #include "mlir/Dialect/Vector/Transforms/LoweringPatterns.h" |
20 | #include "mlir/Dialect/Vector/Transforms/VectorRewritePatterns.h" |
21 | #include "mlir/Dialect/Vector/Transforms/VectorTransforms.h" |
22 | #include "mlir/Dialect/X86Vector/Transforms.h" |
23 | #include "mlir/Transforms/GreedyPatternRewriteDriver.h" |
24 | |
25 | using namespace mlir; |
26 | using namespace mlir::vector; |
27 | using namespace mlir::transform; |
28 | |
29 | //===----------------------------------------------------------------------===// |
30 | // Apply...ConversionPatternsOp |
31 | //===----------------------------------------------------------------------===// |
32 | |
33 | void transform::ApplyVectorToLLVMConversionPatternsOp::populatePatterns( |
34 | TypeConverter &typeConverter, RewritePatternSet &patterns) { |
35 | populateVectorToLLVMConversionPatterns( |
36 | static_cast<LLVMTypeConverter &>(typeConverter), patterns, |
37 | getReassociateFpReductions(), getForce_32bitVectorIndices(), |
38 | getUseVectorAlignment()); |
39 | } |
40 | |
41 | LogicalResult |
42 | transform::ApplyVectorToLLVMConversionPatternsOp::verifyTypeConverter( |
43 | transform::TypeConverterBuilderOpInterface builder) { |
44 | if (builder.getTypeConverterType() != "LLVMTypeConverter" ) |
45 | return emitOpError("expected LLVMTypeConverter" ); |
46 | return success(); |
47 | } |
48 | |
49 | //===----------------------------------------------------------------------===// |
50 | // Apply...PatternsOp |
51 | //===----------------------------------------------------------------------===// |
52 | |
53 | void transform::ApplyCastAwayVectorLeadingOneDimPatternsOp::populatePatterns( |
54 | RewritePatternSet &patterns) { |
55 | vector::populateCastAwayVectorLeadingOneDimPatterns(patterns); |
56 | } |
57 | |
58 | void transform::ApplyFoldArithExtensionPatternsOp::populatePatterns( |
59 | RewritePatternSet &patterns) { |
60 | vector::populateFoldArithExtensionPatterns(patterns); |
61 | } |
62 | |
63 | void transform::ApplyFoldElementwiseToVectorPatternsOp::populatePatterns( |
64 | RewritePatternSet &patterns) { |
65 | vector::populateElementwiseToVectorOpsPatterns(patterns); |
66 | } |
67 | |
68 | void transform::ApplyVectorReductionToContractPatternsOp::populatePatterns( |
69 | RewritePatternSet &patterns) { |
70 | vector::populateVectorReductionToContractPatterns(patterns); |
71 | } |
72 | |
73 | void transform::ApplyLowerCreateMaskPatternsOp::populatePatterns( |
74 | RewritePatternSet &patterns) { |
75 | vector::populateVectorMaskOpLoweringPatterns(patterns); |
76 | } |
77 | |
78 | void transform::ApplyRankReducingSubviewPatternsOp::populatePatterns( |
79 | RewritePatternSet &patterns) { |
80 | vector::populateVectorTransferDropUnitDimsPatterns(patterns); |
81 | } |
82 | |
83 | void transform::ApplyTransferPermutationPatternsOp::populatePatterns( |
84 | RewritePatternSet &patterns) { |
85 | vector::populateVectorTransferPermutationMapLoweringPatterns(patterns); |
86 | } |
87 | |
88 | void transform::ApplyDropUnitDimWithShapeCastPatternsOp::populatePatterns( |
89 | RewritePatternSet &patterns) { |
90 | vector::populateDropUnitDimWithShapeCastPatterns(patterns); |
91 | } |
92 | |
93 | void transform::ApplyLowerBitCastPatternsOp::populatePatterns( |
94 | RewritePatternSet &patterns) { |
95 | vector::populateVectorBitCastLoweringPatterns(patterns); |
96 | } |
97 | |
98 | void transform::ApplyLowerBroadcastPatternsOp::populatePatterns( |
99 | RewritePatternSet &patterns) { |
100 | populateVectorBroadcastLoweringPatterns(patterns); |
101 | } |
102 | |
103 | void transform::ApplyLowerContractionPatternsOp::populatePatterns( |
104 | RewritePatternSet &patterns) { |
105 | populateVectorContractLoweringPatterns(patterns, getLoweringStrategy(), |
106 | /*benefit=*/1, |
107 | /*disableOuterProductLowering=*/true); |
108 | } |
109 | |
110 | void transform::ApplyLowerMasksPatternsOp::populatePatterns( |
111 | RewritePatternSet &patterns) { |
112 | populateVectorMaskOpLoweringPatterns(patterns); |
113 | } |
114 | |
115 | void transform::ApplyLowerMaskedTransfersPatternsOp::populatePatterns( |
116 | RewritePatternSet &patterns) { |
117 | populateVectorMaskLoweringPatternsForSideEffectingOps(patterns); |
118 | } |
119 | |
120 | void transform::ApplyMaterializeMasksPatternsOp::populatePatterns( |
121 | RewritePatternSet &patterns) { |
122 | populateVectorMaskMaterializationPatterns(patterns, |
123 | /*force32BitVectorIndices=*/false); |
124 | } |
125 | |
126 | void transform::ApplyLowerMultiReductionPatternsOp::populatePatterns( |
127 | RewritePatternSet &patterns) { |
128 | vector::VectorTransformsOptions vectorTransformOptions; |
129 | vectorTransformOptions.setVectorMultiReductionLowering(getLoweringStrategy()); |
130 | vector::populateVectorMultiReductionLoweringPatterns( |
131 | patterns, vectorTransformOptions.vectorMultiReductionLowering); |
132 | } |
133 | |
134 | void transform::ApplyLowerOuterProductPatternsOp::populatePatterns( |
135 | RewritePatternSet &patterns) { |
136 | populateVectorOuterProductLoweringPatterns(patterns); |
137 | } |
138 | |
139 | void transform::ApplyLowerGatherPatternsOp::populatePatterns( |
140 | RewritePatternSet &patterns) { |
141 | vector::populateVectorGatherLoweringPatterns(patterns); |
142 | } |
143 | |
144 | void transform::ApplyLowerScanPatternsOp::populatePatterns( |
145 | RewritePatternSet &patterns) { |
146 | vector::populateVectorScanLoweringPatterns(patterns); |
147 | } |
148 | |
149 | void transform::ApplyLowerShapeCastPatternsOp::populatePatterns( |
150 | RewritePatternSet &patterns) { |
151 | vector::populateVectorShapeCastLoweringPatterns(patterns); |
152 | } |
153 | |
154 | void transform::ApplyLowerTransferPatternsOp::populatePatterns( |
155 | RewritePatternSet &patterns) { |
156 | vector::populateVectorTransferLoweringPatterns(patterns, |
157 | getMaxTransferRank()); |
158 | } |
159 | |
160 | void transform::ApplyLowerTransposePatternsOp::populatePatterns( |
161 | RewritePatternSet &patterns) { |
162 | vector::populateVectorTransposeLoweringPatterns(patterns, |
163 | getLoweringStrategy()); |
164 | if (getAvx2LoweringStrategy()) { |
165 | auto avx2LoweringOptions = |
166 | x86vector::avx2::LoweringOptions().setTransposeOptions( |
167 | x86vector::avx2::TransposeLoweringOptions() |
168 | .lower4x8xf32(true) |
169 | .lower8x8xf32(true)); |
170 | x86vector::avx2::populateSpecializedTransposeLoweringPatterns( |
171 | patterns, avx2LoweringOptions, /*benefit=*/10); |
172 | } |
173 | } |
174 | |
175 | void transform::ApplyLowerInterleavePatternsOp::populatePatterns( |
176 | RewritePatternSet &patterns) { |
177 | vector::populateVectorInterleaveLoweringPatterns(patterns); |
178 | } |
179 | |
180 | void transform::ApplyInterleaveToShufflePatternsOp::populatePatterns( |
181 | RewritePatternSet &patterns) { |
182 | vector::populateVectorInterleaveToShufflePatterns(patterns); |
183 | } |
184 | |
185 | void transform::ApplyRewriteNarrowTypePatternsOp::populatePatterns( |
186 | RewritePatternSet &patterns) { |
187 | populateVectorNarrowTypeRewritePatterns(patterns); |
188 | populateVectorTransposeNarrowTypeRewritePatterns(patterns); |
189 | } |
190 | |
191 | void transform::ApplySplitTransferFullPartialPatternsOp::populatePatterns( |
192 | RewritePatternSet &patterns) { |
193 | vector::VectorTransformsOptions vectorTransformOptions; |
194 | vectorTransformOptions.setVectorTransferSplit(getSplitTransferStrategy()); |
195 | populateVectorTransferFullPartialPatterns(patterns, vectorTransformOptions); |
196 | } |
197 | |
198 | void transform::ApplyTransferToScfPatternsOp::populatePatterns( |
199 | RewritePatternSet &patterns) { |
200 | VectorTransferToSCFOptions vectorTransferToSCFOptions = |
201 | VectorTransferToSCFOptions() |
202 | .enableFullUnroll(getFullUnroll()) |
203 | .setTargetRank(getMaxTransferRank()); |
204 | populateVectorToSCFConversionPatterns(patterns, vectorTransferToSCFOptions); |
205 | } |
206 | |
207 | void transform::ApplySinkVectorPatternsOp::populatePatterns( |
208 | RewritePatternSet &patterns) { |
209 | vector::populateSinkVectorOpsPatterns(patterns); |
210 | } |
211 | |
212 | void transform::ApplySinkVectorMemPatternsOp::populatePatterns( |
213 | RewritePatternSet &patterns) { |
214 | vector::populateSinkVectorMemOpsPatterns(patterns); |
215 | } |
216 | |
217 | //===----------------------------------------------------------------------===// |
218 | // Transform op registration |
219 | //===----------------------------------------------------------------------===// |
220 | |
221 | namespace { |
222 | /// Registers new ops and declares PDL as dependent dialect since the additional |
223 | /// ops are using PDL types for operands and results. |
224 | class VectorTransformDialectExtension |
225 | : public transform::TransformDialectExtension< |
226 | VectorTransformDialectExtension> { |
227 | public: |
228 | MLIR_DEFINE_EXPLICIT_INTERNAL_INLINE_TYPE_ID(VectorTransformDialectExtension) |
229 | |
230 | VectorTransformDialectExtension() { |
231 | declareGeneratedDialect<vector::VectorDialect>(); |
232 | declareGeneratedDialect<LLVM::LLVMDialect>(); |
233 | registerTransformOps< |
234 | #define GET_OP_LIST |
235 | #include "mlir/Dialect/Vector/TransformOps/VectorTransformOps.cpp.inc" |
236 | >(); |
237 | } |
238 | }; |
239 | } // namespace |
240 | |
241 | #define GET_OP_CLASSES |
242 | #include "mlir/Dialect/Vector/TransformOps/VectorTransformOps.cpp.inc" |
243 | |
244 | void mlir::vector::registerTransformDialectExtension( |
245 | DialectRegistry ®istry) { |
246 | registry.addExtensions<VectorTransformDialectExtension>(); |
247 | } |
248 | |