1 | //===- VectorToLLVM.cpp - Conversion from Vector to the LLVM dialect ------===// |
2 | // |
3 | // Part of the LLVM Project, under the Apache License v2.0 with LLVM Exceptions. |
4 | // See https://llvm.org/LICENSE.txt for license information. |
5 | // SPDX-License-Identifier: Apache-2.0 WITH LLVM-exception |
6 | // |
7 | //===----------------------------------------------------------------------===// |
8 | |
9 | #include "mlir/Conversion/VectorToLLVM/ConvertVectorToLLVMPass.h" |
10 | |
11 | #include "mlir/Conversion/LLVMCommon/ConversionTarget.h" |
12 | #include "mlir/Conversion/LLVMCommon/TypeConverter.h" |
13 | #include "mlir/Dialect/AMX/AMXDialect.h" |
14 | #include "mlir/Dialect/AMX/Transforms.h" |
15 | #include "mlir/Dialect/Arith/IR/Arith.h" |
16 | #include "mlir/Dialect/ArmNeon/ArmNeonDialect.h" |
17 | #include "mlir/Dialect/ArmSVE/IR/ArmSVEDialect.h" |
18 | #include "mlir/Dialect/ArmSVE/Transforms/Transforms.h" |
19 | #include "mlir/Dialect/Func/IR/FuncOps.h" |
20 | #include "mlir/Dialect/LLVMIR/LLVMDialect.h" |
21 | #include "mlir/Dialect/MemRef/IR/MemRef.h" |
22 | #include "mlir/Dialect/Vector/Transforms/LoweringPatterns.h" |
23 | #include "mlir/Dialect/Vector/Transforms/VectorRewritePatterns.h" |
24 | #include "mlir/Dialect/X86Vector/Transforms.h" |
25 | #include "mlir/Dialect/X86Vector/X86VectorDialect.h" |
26 | #include "mlir/Pass/Pass.h" |
27 | #include "mlir/Transforms/GreedyPatternRewriteDriver.h" |
28 | |
29 | namespace mlir { |
30 | #define GEN_PASS_DEF_CONVERTVECTORTOLLVMPASS |
31 | #include "mlir/Conversion/Passes.h.inc" |
32 | } // namespace mlir |
33 | |
34 | using namespace mlir; |
35 | using namespace mlir::vector; |
36 | |
37 | namespace { |
38 | struct LowerVectorToLLVMPass |
39 | : public impl::ConvertVectorToLLVMPassBase<LowerVectorToLLVMPass> { |
40 | |
41 | using Base::Base; |
42 | |
43 | // Override explicitly to allow conditional dialect dependence. |
44 | void getDependentDialects(DialectRegistry ®istry) const override { |
45 | registry.insert<LLVM::LLVMDialect>(); |
46 | registry.insert<arith::ArithDialect>(); |
47 | registry.insert<memref::MemRefDialect>(); |
48 | if (armNeon) |
49 | registry.insert<arm_neon::ArmNeonDialect>(); |
50 | if (armSVE) |
51 | registry.insert<arm_sve::ArmSVEDialect>(); |
52 | if (amx) |
53 | registry.insert<amx::AMXDialect>(); |
54 | if (x86Vector) |
55 | registry.insert<x86vector::X86VectorDialect>(); |
56 | } |
57 | void runOnOperation() override; |
58 | }; |
59 | } // namespace |
60 | |
61 | void LowerVectorToLLVMPass::runOnOperation() { |
62 | // Perform progressive lowering of operations on slices and |
63 | // all contraction operations. Also applies folding and DCE. |
64 | { |
65 | RewritePatternSet patterns(&getContext()); |
66 | populateVectorToVectorCanonicalizationPatterns(patterns); |
67 | populateVectorBroadcastLoweringPatterns(patterns); |
68 | populateVectorContractLoweringPatterns(patterns, options: VectorTransformsOptions()); |
69 | populateVectorMaskOpLoweringPatterns(patterns); |
70 | populateVectorShapeCastLoweringPatterns(patterns); |
71 | populateVectorInterleaveLoweringPatterns(patterns); |
72 | populateVectorTransposeLoweringPatterns(patterns, |
73 | options: VectorTransformsOptions()); |
74 | // Vector transfer ops with rank > 1 should be lowered with VectorToSCF. |
75 | populateVectorTransferLoweringPatterns(patterns, /*maxTransferRank=*/1); |
76 | (void)applyPatternsAndFoldGreedily(getOperation(), std::move(patterns)); |
77 | } |
78 | |
79 | // Convert to the LLVM IR dialect. |
80 | LowerToLLVMOptions options(&getContext()); |
81 | LLVMTypeConverter converter(&getContext(), options); |
82 | RewritePatternSet patterns(&getContext()); |
83 | populateVectorMaskMaterializationPatterns(patterns, force32BitVectorIndices); |
84 | populateVectorTransferLoweringPatterns(patterns); |
85 | populateVectorToLLVMMatrixConversionPatterns(converter, patterns); |
86 | populateVectorToLLVMConversionPatterns( |
87 | converter, patterns, reassociateFPReductions, force32BitVectorIndices); |
88 | populateVectorToLLVMMatrixConversionPatterns(converter, patterns); |
89 | |
90 | // Architecture specific augmentations. |
91 | LLVMConversionTarget target(getContext()); |
92 | target.addLegalDialect<arith::ArithDialect>(); |
93 | target.addLegalDialect<memref::MemRefDialect>(); |
94 | target.addLegalOp<UnrealizedConversionCastOp>(); |
95 | |
96 | if (armNeon) { |
97 | // TODO: we may or may not want to include in-dialect lowering to |
98 | // LLVM-compatible operations here. So far, all operations in the dialect |
99 | // can be translated to LLVM IR so there is no conversion necessary. |
100 | target.addLegalDialect<arm_neon::ArmNeonDialect>(); |
101 | } |
102 | if (armSVE) { |
103 | configureArmSVELegalizeForExportTarget(target); |
104 | populateArmSVELegalizeForLLVMExportPatterns(converter, patterns); |
105 | } |
106 | if (amx) { |
107 | configureAMXLegalizeForExportTarget(target); |
108 | populateAMXLegalizeForLLVMExportPatterns(converter, patterns); |
109 | } |
110 | if (x86Vector) { |
111 | configureX86VectorLegalizeForExportTarget(target); |
112 | populateX86VectorLegalizeForLLVMExportPatterns(converter, patterns); |
113 | } |
114 | |
115 | if (failed( |
116 | applyPartialConversion(getOperation(), target, std::move(patterns)))) |
117 | signalPassFailure(); |
118 | } |
119 | |