1 | //===- ModuleImport.cpp - LLVM to MLIR conversion ---------------*- C++ -*-===// |
---|---|
2 | // |
3 | // Part of the LLVM Project, under the Apache License v2.0 with LLVM Exceptions. |
4 | // See https://llvm.org/LICENSE.txt for license information. |
5 | // SPDX-License-Identifier: Apache-2.0 WITH LLVM-exception |
6 | // |
7 | //===----------------------------------------------------------------------===// |
8 | // |
9 | // This file implements the import of an LLVM IR module into an LLVM dialect |
10 | // module. |
11 | // |
12 | //===----------------------------------------------------------------------===// |
13 | |
14 | #include "mlir/Target/LLVMIR/ModuleImport.h" |
15 | #include "mlir/IR/BuiltinAttributes.h" |
16 | #include "mlir/Target/LLVMIR/Import.h" |
17 | |
18 | #include "AttrKindDetail.h" |
19 | #include "DataLayoutImporter.h" |
20 | #include "DebugImporter.h" |
21 | #include "LoopAnnotationImporter.h" |
22 | |
23 | #include "mlir/Dialect/DLTI/DLTI.h" |
24 | #include "mlir/Dialect/LLVMIR/LLVMDialect.h" |
25 | #include "mlir/IR/Builders.h" |
26 | #include "mlir/IR/Matchers.h" |
27 | #include "mlir/Interfaces/DataLayoutInterfaces.h" |
28 | #include "mlir/Tools/mlir-translate/Translation.h" |
29 | |
30 | #include "llvm/ADT/DepthFirstIterator.h" |
31 | #include "llvm/ADT/PostOrderIterator.h" |
32 | #include "llvm/ADT/ScopeExit.h" |
33 | #include "llvm/ADT/StringSet.h" |
34 | #include "llvm/ADT/TypeSwitch.h" |
35 | #include "llvm/IR/Comdat.h" |
36 | #include "llvm/IR/Constants.h" |
37 | #include "llvm/IR/InlineAsm.h" |
38 | #include "llvm/IR/InstIterator.h" |
39 | #include "llvm/IR/Instructions.h" |
40 | #include "llvm/IR/IntrinsicInst.h" |
41 | #include "llvm/IR/Metadata.h" |
42 | #include "llvm/IR/Operator.h" |
43 | #include "llvm/Support/ModRef.h" |
44 | #include <optional> |
45 | |
46 | using namespace mlir; |
47 | using namespace mlir::LLVM; |
48 | using namespace mlir::LLVM::detail; |
49 | |
50 | #include "mlir/Dialect/LLVMIR/LLVMConversionEnumsFromLLVM.inc" |
51 | |
52 | // Utility to print an LLVM value as a string for passing to emitError(). |
53 | // FIXME: Diagnostic should be able to natively handle types that have |
54 | // operator << (raw_ostream&) defined. |
55 | static std::string diag(const llvm::Value &value) { |
56 | std::string str; |
57 | llvm::raw_string_ostream os(str); |
58 | os << value; |
59 | return str; |
60 | } |
61 | |
62 | // Utility to print an LLVM metadata node as a string for passing |
63 | // to emitError(). The module argument is needed to print the nodes |
64 | // canonically numbered. |
65 | static std::string diagMD(const llvm::Metadata *node, |
66 | const llvm::Module *module) { |
67 | std::string str; |
68 | llvm::raw_string_ostream os(str); |
69 | node->print(OS&: os, M: module, /*IsForDebug=*/true); |
70 | return str; |
71 | } |
72 | |
73 | /// Returns the name of the global_ctors global variables. |
74 | static constexpr StringRef getGlobalCtorsVarName() { |
75 | return "llvm.global_ctors"; |
76 | } |
77 | |
78 | /// Prefix used for symbols of nameless llvm globals. |
79 | static constexpr StringRef getNamelessGlobalPrefix() { |
80 | return "mlir.llvm.nameless_global"; |
81 | } |
82 | |
83 | /// Returns the name of the global_dtors global variables. |
84 | static constexpr StringRef getGlobalDtorsVarName() { |
85 | return "llvm.global_dtors"; |
86 | } |
87 | |
88 | /// Returns the symbol name for the module-level comdat operation. It must not |
89 | /// conflict with the user namespace. |
90 | static constexpr StringRef getGlobalComdatOpName() { |
91 | return "__llvm_global_comdat"; |
92 | } |
93 | |
94 | /// Converts the sync scope identifier of `inst` to the string representation |
95 | /// necessary to build an atomic LLVM dialect operation. Returns the empty |
96 | /// string if the operation has either no sync scope or the default system-level |
97 | /// sync scope attached. The atomic operations only set their sync scope |
98 | /// attribute if they have a non-default sync scope attached. |
99 | static StringRef getLLVMSyncScope(llvm::Instruction *inst) { |
100 | std::optional<llvm::SyncScope::ID> syncScopeID = |
101 | llvm::getAtomicSyncScopeID(I: inst); |
102 | if (!syncScopeID) |
103 | return ""; |
104 | |
105 | // Search the sync scope name for the given identifier. The default |
106 | // system-level sync scope thereby maps to the empty string. |
107 | SmallVector<StringRef> syncScopeName; |
108 | llvm::LLVMContext &llvmContext = inst->getContext(); |
109 | llvmContext.getSyncScopeNames(SSNs&: syncScopeName); |
110 | auto *it = llvm::find_if(Range&: syncScopeName, P: [&](StringRef name) { |
111 | return *syncScopeID == llvmContext.getOrInsertSyncScopeID(SSN: name); |
112 | }); |
113 | if (it != syncScopeName.end()) |
114 | return *it; |
115 | llvm_unreachable("incorrect sync scope identifier"); |
116 | } |
117 | |
118 | /// Converts an array of unsigned indices to a signed integer position array. |
119 | static SmallVector<int64_t> getPositionFromIndices(ArrayRef<unsigned> indices) { |
120 | SmallVector<int64_t> position; |
121 | llvm::append_range(C&: position, R&: indices); |
122 | return position; |
123 | } |
124 | |
125 | /// Converts the LLVM instructions that have a generated MLIR builder. Using a |
126 | /// static implementation method called from the module import ensures the |
127 | /// builders have to use the `moduleImport` argument and cannot directly call |
128 | /// import methods. As a result, both the intrinsic and the instruction MLIR |
129 | /// builders have to use the `moduleImport` argument and none of them has direct |
130 | /// access to the private module import methods. |
131 | static LogicalResult convertInstructionImpl(OpBuilder &odsBuilder, |
132 | llvm::Instruction *inst, |
133 | ModuleImport &moduleImport, |
134 | LLVMImportInterface &iface) { |
135 | // Copy the operands to an LLVM operands array reference for conversion. |
136 | SmallVector<llvm::Value *> operands(inst->operands()); |
137 | ArrayRef<llvm::Value *> llvmOperands(operands); |
138 | |
139 | // Convert all instructions that provide an MLIR builder. |
140 | if (iface.isConvertibleInstruction(id: inst->getOpcode())) |
141 | return iface.convertInstruction(builder&: odsBuilder, inst, llvmOperands, |
142 | moduleImport); |
143 | // TODO: Implement the `convertInstruction` hooks in the |
144 | // `LLVMDialectLLVMIRImportInterface` and move the following include there. |
145 | #include "mlir/Dialect/LLVMIR/LLVMOpFromLLVMIRConversions.inc" |
146 | return failure(); |
147 | } |
148 | |
149 | /// Get a topologically sorted list of blocks for the given basic blocks. |
150 | static SetVector<llvm::BasicBlock *> |
151 | getTopologicallySortedBlocks(ArrayRef<llvm::BasicBlock *> basicBlocks) { |
152 | SetVector<llvm::BasicBlock *> blocks; |
153 | for (llvm::BasicBlock *basicBlock : basicBlocks) { |
154 | if (!blocks.contains(basicBlock)) { |
155 | llvm::ReversePostOrderTraversal<llvm::BasicBlock *> traversal(basicBlock); |
156 | blocks.insert_range(traversal); |
157 | } |
158 | } |
159 | assert(blocks.size() == basicBlocks.size() && "some blocks are not sorted"); |
160 | return blocks; |
161 | } |
162 | |
163 | ModuleImport::ModuleImport(ModuleOp mlirModule, |
164 | std::unique_ptr<llvm::Module> llvmModule, |
165 | bool emitExpensiveWarnings, |
166 | bool importEmptyDICompositeTypes, |
167 | bool preferUnregisteredIntrinsics, |
168 | bool importStructsAsLiterals) |
169 | : builder(mlirModule->getContext()), context(mlirModule->getContext()), |
170 | mlirModule(mlirModule), llvmModule(std::move(llvmModule)), |
171 | iface(mlirModule->getContext()), |
172 | typeTranslator(*mlirModule->getContext(), importStructsAsLiterals), |
173 | debugImporter(std::make_unique<DebugImporter>( |
174 | mlirModule, importEmptyDICompositeTypes)), |
175 | loopAnnotationImporter( |
176 | std::make_unique<LoopAnnotationImporter>(args&: *this, args&: builder)), |
177 | emitExpensiveWarnings(emitExpensiveWarnings), |
178 | preferUnregisteredIntrinsics(preferUnregisteredIntrinsics) { |
179 | builder.setInsertionPointToStart(mlirModule.getBody()); |
180 | } |
181 | |
182 | ComdatOp ModuleImport::getGlobalComdatOp() { |
183 | if (globalComdatOp) |
184 | return globalComdatOp; |
185 | |
186 | OpBuilder::InsertionGuard guard(builder); |
187 | builder.setInsertionPointToEnd(mlirModule.getBody()); |
188 | globalComdatOp = |
189 | builder.create<ComdatOp>(mlirModule.getLoc(), getGlobalComdatOpName()); |
190 | globalInsertionOp = globalComdatOp; |
191 | return globalComdatOp; |
192 | } |
193 | |
194 | LogicalResult ModuleImport::processTBAAMetadata(const llvm::MDNode *node) { |
195 | Location loc = mlirModule.getLoc(); |
196 | |
197 | // If `node` is a valid TBAA root node, then return its optional identity |
198 | // string, otherwise return failure. |
199 | auto getIdentityIfRootNode = |
200 | [&](const llvm::MDNode *node) -> FailureOr<std::optional<StringRef>> { |
201 | // Root node, e.g.: |
202 | // !0 = !{!"Simple C/C++ TBAA"} |
203 | // !1 = !{} |
204 | if (node->getNumOperands() > 1) |
205 | return failure(); |
206 | // If the operand is MDString, then assume that this is a root node. |
207 | if (node->getNumOperands() == 1) |
208 | if (const auto *op0 = dyn_cast<const llvm::MDString>(Val: node->getOperand(I: 0))) |
209 | return std::optional<StringRef>{op0->getString()}; |
210 | return std::optional<StringRef>{}; |
211 | }; |
212 | |
213 | // If `node` looks like a TBAA type descriptor metadata, |
214 | // then return true, if it is a valid node, and false otherwise. |
215 | // If it does not look like a TBAA type descriptor metadata, then |
216 | // return std::nullopt. |
217 | // If `identity` and `memberTypes/Offsets` are non-null, then they will |
218 | // contain the converted metadata operands for a valid TBAA node (i.e. when |
219 | // true is returned). |
220 | auto isTypeDescriptorNode = [&](const llvm::MDNode *node, |
221 | StringRef *identity = nullptr, |
222 | SmallVectorImpl<TBAAMemberAttr> *members = |
223 | nullptr) -> std::optional<bool> { |
224 | unsigned numOperands = node->getNumOperands(); |
225 | // Type descriptor, e.g.: |
226 | // !1 = !{!"int", !0, /*optional*/i64 0} /* scalar int type */ |
227 | // !2 = !{!"agg_t", !1, i64 0} /* struct agg_t { int x; } */ |
228 | if (numOperands < 2) |
229 | return std::nullopt; |
230 | |
231 | // TODO: support "new" format (D41501) for type descriptors, |
232 | // where the first operand is an MDNode. |
233 | const auto *identityNode = |
234 | dyn_cast<const llvm::MDString>(Val: node->getOperand(I: 0)); |
235 | if (!identityNode) |
236 | return std::nullopt; |
237 | |
238 | // This should be a type descriptor node. |
239 | if (identity) |
240 | *identity = identityNode->getString(); |
241 | |
242 | for (unsigned pairNum = 0, e = numOperands / 2; pairNum < e; ++pairNum) { |
243 | const auto *memberNode = |
244 | dyn_cast<const llvm::MDNode>(Val: node->getOperand(I: 2 * pairNum + 1)); |
245 | if (!memberNode) { |
246 | emitError(loc) << "operand '"<< 2 * pairNum + 1 << "' must be MDNode: " |
247 | << diagMD(node, module: llvmModule.get()); |
248 | return false; |
249 | } |
250 | int64_t offset = 0; |
251 | if (2 * pairNum + 2 >= numOperands) { |
252 | // Allow for optional 0 offset in 2-operand nodes. |
253 | if (numOperands != 2) { |
254 | emitError(loc) << "missing member offset: " |
255 | << diagMD(node, module: llvmModule.get()); |
256 | return false; |
257 | } |
258 | } else { |
259 | auto *offsetCI = llvm::mdconst::dyn_extract<llvm::ConstantInt>( |
260 | MD: node->getOperand(I: 2 * pairNum + 2)); |
261 | if (!offsetCI) { |
262 | emitError(loc) << "operand '"<< 2 * pairNum + 2 |
263 | << "' must be ConstantInt: " |
264 | << diagMD(node, module: llvmModule.get()); |
265 | return false; |
266 | } |
267 | offset = offsetCI->getZExtValue(); |
268 | } |
269 | |
270 | if (members) |
271 | members->push_back(TBAAMemberAttr::get( |
272 | cast<TBAANodeAttr>(tbaaMapping.lookup(memberNode)), offset)); |
273 | } |
274 | |
275 | return true; |
276 | }; |
277 | |
278 | // If `node` looks like a TBAA access tag metadata, |
279 | // then return true, if it is a valid node, and false otherwise. |
280 | // If it does not look like a TBAA access tag metadata, then |
281 | // return std::nullopt. |
282 | // If the other arguments are non-null, then they will contain |
283 | // the converted metadata operands for a valid TBAA node (i.e. when true is |
284 | // returned). |
285 | auto isTagNode = [&](const llvm::MDNode *node, |
286 | TBAATypeDescriptorAttr *baseAttr = nullptr, |
287 | TBAATypeDescriptorAttr *accessAttr = nullptr, |
288 | int64_t *offset = nullptr, |
289 | bool *isConstant = nullptr) -> std::optional<bool> { |
290 | // Access tag, e.g.: |
291 | // !3 = !{!1, !1, i64 0} /* scalar int access */ |
292 | // !4 = !{!2, !1, i64 0} /* agg_t::x access */ |
293 | // |
294 | // Optional 4th argument is ConstantInt 0/1 identifying whether |
295 | // the location being accessed is "constant" (see for details: |
296 | // https://llvm.org/docs/LangRef.html#representation). |
297 | unsigned numOperands = node->getNumOperands(); |
298 | if (numOperands != 3 && numOperands != 4) |
299 | return std::nullopt; |
300 | const auto *baseMD = dyn_cast<const llvm::MDNode>(Val: node->getOperand(I: 0)); |
301 | const auto *accessMD = dyn_cast<const llvm::MDNode>(Val: node->getOperand(I: 1)); |
302 | auto *offsetCI = |
303 | llvm::mdconst::dyn_extract<llvm::ConstantInt>(MD: node->getOperand(I: 2)); |
304 | if (!baseMD || !accessMD || !offsetCI) |
305 | return std::nullopt; |
306 | // TODO: support "new" TBAA format, if needed (see D41501). |
307 | // In the "old" format the first operand of the access type |
308 | // metadata is MDString. We have to distinguish the formats, |
309 | // because access tags have the same structure, but different |
310 | // meaning for the operands. |
311 | if (accessMD->getNumOperands() < 1 || |
312 | !isa<llvm::MDString>(Val: accessMD->getOperand(I: 0))) |
313 | return std::nullopt; |
314 | bool isConst = false; |
315 | if (numOperands == 4) { |
316 | auto *isConstantCI = |
317 | llvm::mdconst::dyn_extract<llvm::ConstantInt>(MD: node->getOperand(I: 3)); |
318 | if (!isConstantCI) { |
319 | emitError(loc) << "operand '3' must be ConstantInt: " |
320 | << diagMD(node, module: llvmModule.get()); |
321 | return false; |
322 | } |
323 | isConst = isConstantCI->getValue()[0]; |
324 | } |
325 | if (baseAttr) |
326 | *baseAttr = cast<TBAATypeDescriptorAttr>(tbaaMapping.lookup(baseMD)); |
327 | if (accessAttr) |
328 | *accessAttr = cast<TBAATypeDescriptorAttr>(tbaaMapping.lookup(accessMD)); |
329 | if (offset) |
330 | *offset = offsetCI->getZExtValue(); |
331 | if (isConstant) |
332 | *isConstant = isConst; |
333 | return true; |
334 | }; |
335 | |
336 | // Do a post-order walk over the TBAA Graph. Since a correct TBAA Graph is a |
337 | // DAG, a post-order walk guarantees that we convert any metadata node we |
338 | // depend on, prior to converting the current node. |
339 | DenseSet<const llvm::MDNode *> seen; |
340 | SmallVector<const llvm::MDNode *> workList; |
341 | workList.push_back(Elt: node); |
342 | while (!workList.empty()) { |
343 | const llvm::MDNode *current = workList.back(); |
344 | if (tbaaMapping.contains(Val: current)) { |
345 | // Already converted. Just pop from the worklist. |
346 | workList.pop_back(); |
347 | continue; |
348 | } |
349 | |
350 | // If any child of this node is not yet converted, don't pop the current |
351 | // node from the worklist but push the not-yet-converted children in the |
352 | // front of the worklist. |
353 | bool anyChildNotConverted = false; |
354 | for (const llvm::MDOperand &operand : current->operands()) |
355 | if (auto *childNode = dyn_cast_or_null<const llvm::MDNode>(Val: operand.get())) |
356 | if (!tbaaMapping.contains(Val: childNode)) { |
357 | workList.push_back(Elt: childNode); |
358 | anyChildNotConverted = true; |
359 | } |
360 | |
361 | if (anyChildNotConverted) { |
362 | // If this is the second time we failed to convert an element in the |
363 | // worklist it must be because a child is dependent on it being converted |
364 | // and we have a cycle in the graph. Cycles are not allowed in TBAA |
365 | // graphs. |
366 | if (!seen.insert(V: current).second) |
367 | return emitError(loc) << "has cycle in TBAA graph: " |
368 | << diagMD(node: current, module: llvmModule.get()); |
369 | |
370 | continue; |
371 | } |
372 | |
373 | // Otherwise simply import the current node. |
374 | workList.pop_back(); |
375 | |
376 | FailureOr<std::optional<StringRef>> rootNodeIdentity = |
377 | getIdentityIfRootNode(current); |
378 | if (succeeded(Result: rootNodeIdentity)) { |
379 | StringAttr stringAttr = *rootNodeIdentity |
380 | ? builder.getStringAttr(**rootNodeIdentity) |
381 | : nullptr; |
382 | // The root nodes do not have operands, so we can create |
383 | // the TBAARootAttr on the first walk. |
384 | tbaaMapping.insert({current, builder.getAttr<TBAARootAttr>(stringAttr)}); |
385 | continue; |
386 | } |
387 | |
388 | StringRef identity; |
389 | SmallVector<TBAAMemberAttr> members; |
390 | if (std::optional<bool> isValid = |
391 | isTypeDescriptorNode(current, &identity, &members)) { |
392 | assert(isValid.value() && "type descriptor node must be valid"); |
393 | |
394 | tbaaMapping.insert({current, builder.getAttr<TBAATypeDescriptorAttr>( |
395 | identity, members)}); |
396 | continue; |
397 | } |
398 | |
399 | TBAATypeDescriptorAttr baseAttr, accessAttr; |
400 | int64_t offset; |
401 | bool isConstant; |
402 | if (std::optional<bool> isValid = |
403 | isTagNode(current, &baseAttr, &accessAttr, &offset, &isConstant)) { |
404 | assert(isValid.value() && "access tag node must be valid"); |
405 | tbaaMapping.insert( |
406 | {current, builder.getAttr<TBAATagAttr>(baseAttr, accessAttr, offset, |
407 | isConstant)}); |
408 | continue; |
409 | } |
410 | |
411 | return emitError(loc) << "unsupported TBAA node format: " |
412 | << diagMD(node: current, module: llvmModule.get()); |
413 | } |
414 | return success(); |
415 | } |
416 | |
417 | LogicalResult |
418 | ModuleImport::processAccessGroupMetadata(const llvm::MDNode *node) { |
419 | Location loc = mlirModule.getLoc(); |
420 | if (failed(Result: loopAnnotationImporter->translateAccessGroup(node, loc))) |
421 | return emitError(loc) << "unsupported access group node: " |
422 | << diagMD(node, module: llvmModule.get()); |
423 | return success(); |
424 | } |
425 | |
426 | LogicalResult |
427 | ModuleImport::processAliasScopeMetadata(const llvm::MDNode *node) { |
428 | Location loc = mlirModule.getLoc(); |
429 | // Helper that verifies the node has a self reference operand. |
430 | auto verifySelfRef = [](const llvm::MDNode *node) { |
431 | return node->getNumOperands() != 0 && |
432 | node == dyn_cast<llvm::MDNode>(Val: node->getOperand(I: 0)); |
433 | }; |
434 | auto verifySelfRefOrString = [](const llvm::MDNode *node) { |
435 | return node->getNumOperands() != 0 && |
436 | (node == dyn_cast<llvm::MDNode>(Val: node->getOperand(I: 0)) || |
437 | isa<llvm::MDString>(Val: node->getOperand(I: 0))); |
438 | }; |
439 | // Helper that verifies the given operand is a string or does not exist. |
440 | auto verifyDescription = [](const llvm::MDNode *node, unsigned idx) { |
441 | return idx >= node->getNumOperands() || |
442 | isa<llvm::MDString>(Val: node->getOperand(I: idx)); |
443 | }; |
444 | |
445 | auto getIdAttr = [&](const llvm::MDNode *node) -> Attribute { |
446 | if (verifySelfRef(node)) |
447 | return DistinctAttr::create(builder.getUnitAttr()); |
448 | |
449 | auto name = cast<llvm::MDString>(Val: node->getOperand(I: 0)); |
450 | return builder.getStringAttr(name->getString()); |
451 | }; |
452 | |
453 | // Helper that creates an alias scope domain attribute. |
454 | auto createAliasScopeDomainOp = [&](const llvm::MDNode *aliasDomain) { |
455 | StringAttr description = nullptr; |
456 | if (aliasDomain->getNumOperands() >= 2) |
457 | if (auto *operand = dyn_cast<llvm::MDString>(Val: aliasDomain->getOperand(I: 1))) |
458 | description = builder.getStringAttr(operand->getString()); |
459 | Attribute idAttr = getIdAttr(aliasDomain); |
460 | return builder.getAttr<AliasScopeDomainAttr>(idAttr, description); |
461 | }; |
462 | |
463 | // Collect the alias scopes and domains to translate them. |
464 | for (const llvm::MDOperand &operand : node->operands()) { |
465 | if (const auto *scope = dyn_cast<llvm::MDNode>(Val: operand)) { |
466 | llvm::AliasScopeNode aliasScope(scope); |
467 | const llvm::MDNode *domain = aliasScope.getDomain(); |
468 | |
469 | // Verify the scope node points to valid scope metadata which includes |
470 | // verifying its domain. Perform the verification before looking it up in |
471 | // the alias scope mapping since it could have been inserted as a domain |
472 | // node before. |
473 | if (!verifySelfRefOrString(scope) || !domain || |
474 | !verifyDescription(scope, 2)) |
475 | return emitError(loc) << "unsupported alias scope node: " |
476 | << diagMD(node: scope, module: llvmModule.get()); |
477 | if (!verifySelfRefOrString(domain) || !verifyDescription(domain, 1)) |
478 | return emitError(loc) << "unsupported alias domain node: " |
479 | << diagMD(node: domain, module: llvmModule.get()); |
480 | |
481 | if (aliasScopeMapping.contains(Val: scope)) |
482 | continue; |
483 | |
484 | // Convert the domain metadata node if it has not been translated before. |
485 | auto it = aliasScopeMapping.find(Val: aliasScope.getDomain()); |
486 | if (it == aliasScopeMapping.end()) { |
487 | auto aliasScopeDomainOp = createAliasScopeDomainOp(domain); |
488 | it = aliasScopeMapping.try_emplace(domain, aliasScopeDomainOp).first; |
489 | } |
490 | |
491 | // Convert the scope metadata node if it has not been converted before. |
492 | StringAttr description = nullptr; |
493 | if (!aliasScope.getName().empty()) |
494 | description = builder.getStringAttr(aliasScope.getName()); |
495 | Attribute idAttr = getIdAttr(scope); |
496 | auto aliasScopeOp = builder.getAttr<AliasScopeAttr>( |
497 | idAttr, cast<AliasScopeDomainAttr>(it->second), description); |
498 | |
499 | aliasScopeMapping.try_emplace(aliasScope.getNode(), aliasScopeOp); |
500 | } |
501 | } |
502 | return success(); |
503 | } |
504 | |
505 | FailureOr<SmallVector<AliasScopeAttr>> |
506 | ModuleImport::lookupAliasScopeAttrs(const llvm::MDNode *node) const { |
507 | SmallVector<AliasScopeAttr> aliasScopes; |
508 | aliasScopes.reserve(node->getNumOperands()); |
509 | for (const llvm::MDOperand &operand : node->operands()) { |
510 | auto *node = cast<llvm::MDNode>(Val: operand.get()); |
511 | aliasScopes.push_back( |
512 | dyn_cast_or_null<AliasScopeAttr>(aliasScopeMapping.lookup(node))); |
513 | } |
514 | // Return failure if one of the alias scope lookups failed. |
515 | if (llvm::is_contained(aliasScopes, nullptr)) |
516 | return failure(); |
517 | return aliasScopes; |
518 | } |
519 | |
520 | void ModuleImport::addDebugIntrinsic(llvm::CallInst *intrinsic) { |
521 | debugIntrinsics.insert(X: intrinsic); |
522 | } |
523 | |
524 | static Attribute convertCGProfileModuleFlagValue(ModuleOp mlirModule, |
525 | llvm::MDTuple *mdTuple) { |
526 | auto getLLVMFunction = |
527 | [&](const llvm::MDOperand &funcMDO) -> llvm::Function * { |
528 | auto *f = cast_or_null<llvm::ValueAsMetadata>(Val: funcMDO); |
529 | // nullptr is a valid value for the function pointer. |
530 | if (!f) |
531 | return nullptr; |
532 | auto *llvmFn = cast<llvm::Function>(Val: f->getValue()->stripPointerCasts()); |
533 | return llvmFn; |
534 | }; |
535 | |
536 | // Each tuple element becomes one ModuleFlagCGProfileEntryAttr. |
537 | SmallVector<Attribute> cgProfile; |
538 | for (unsigned i = 0; i < mdTuple->getNumOperands(); i++) { |
539 | const llvm::MDOperand &mdo = mdTuple->getOperand(I: i); |
540 | auto *cgEntry = cast<llvm::MDNode>(Val: mdo); |
541 | llvm::Constant *llvmConstant = |
542 | cast<llvm::ConstantAsMetadata>(Val: cgEntry->getOperand(I: 2))->getValue(); |
543 | uint64_t count = cast<llvm::ConstantInt>(Val: llvmConstant)->getZExtValue(); |
544 | auto *fromFn = getLLVMFunction(cgEntry->getOperand(I: 0)); |
545 | auto *toFn = getLLVMFunction(cgEntry->getOperand(I: 1)); |
546 | // FlatSymbolRefAttr::get(mlirModule->getContext(), llvmFn->getName()); |
547 | cgProfile.push_back(ModuleFlagCGProfileEntryAttr::get( |
548 | mlirModule->getContext(), |
549 | fromFn ? FlatSymbolRefAttr::get(mlirModule->getContext(), |
550 | fromFn->getName()) |
551 | : nullptr, |
552 | toFn ? FlatSymbolRefAttr::get(mlirModule->getContext(), toFn->getName()) |
553 | : nullptr, |
554 | count)); |
555 | } |
556 | return ArrayAttr::get(mlirModule->getContext(), cgProfile); |
557 | } |
558 | |
559 | /// Extract a two element `MDTuple` from a `MDOperand`. Emit a warning in case |
560 | /// something else is found. |
561 | static llvm::MDTuple *getTwoElementMDTuple(ModuleOp mlirModule, |
562 | const llvm::Module *llvmModule, |
563 | const llvm::MDOperand &md) { |
564 | auto *tupleEntry = dyn_cast_or_null<llvm::MDTuple>(Val: md); |
565 | if (!tupleEntry || tupleEntry->getNumOperands() != 2) |
566 | emitWarning(mlirModule.getLoc()) |
567 | << "expected 2-element tuple metadata: "<< diagMD(node: md, module: llvmModule); |
568 | return tupleEntry; |
569 | } |
570 | |
571 | /// Extract a constant metadata value from a two element tuple (<key, value>). |
572 | /// Return nullptr if requirements are not met. A warning is emitted if the |
573 | /// `matchKey` is different from the tuple's key. |
574 | static llvm::ConstantAsMetadata *getConstantMDFromKeyValueTuple( |
575 | ModuleOp mlirModule, const llvm::Module *llvmModule, |
576 | const llvm::MDOperand &md, StringRef matchKey, bool optional = false) { |
577 | llvm::MDTuple *tupleEntry = getTwoElementMDTuple(mlirModule, llvmModule, md); |
578 | if (!tupleEntry) |
579 | return nullptr; |
580 | auto *keyMD = dyn_cast<llvm::MDString>(Val: tupleEntry->getOperand(I: 0)); |
581 | if (!keyMD || keyMD->getString() != matchKey) { |
582 | if (!optional) |
583 | emitWarning(mlirModule.getLoc()) |
584 | << "expected '"<< matchKey << "' key, but found: " |
585 | << diagMD(node: tupleEntry->getOperand(I: 0), module: llvmModule); |
586 | return nullptr; |
587 | } |
588 | |
589 | return dyn_cast<llvm::ConstantAsMetadata>(Val: tupleEntry->getOperand(I: 1)); |
590 | } |
591 | |
592 | /// Extract an integer value from a two element tuple (<key, value>). |
593 | /// Fail if requirements are not met. A warning is emitted if the |
594 | /// found value isn't a LLVM constant integer. |
595 | static FailureOr<uint64_t> |
596 | convertInt64FromKeyValueTuple(ModuleOp mlirModule, |
597 | const llvm::Module *llvmModule, |
598 | const llvm::MDOperand &md, StringRef matchKey) { |
599 | llvm::ConstantAsMetadata *valMD = |
600 | getConstantMDFromKeyValueTuple(mlirModule, llvmModule, md, matchKey); |
601 | if (!valMD) |
602 | return failure(); |
603 | |
604 | if (auto *cstInt = dyn_cast<llvm::ConstantInt>(valMD->getValue())) |
605 | return cstInt->getZExtValue(); |
606 | |
607 | emitWarning(mlirModule.getLoc()) |
608 | << "expected integer metadata value for key '"<< matchKey |
609 | << "': "<< diagMD(node: md, module: llvmModule); |
610 | return failure(); |
611 | } |
612 | |
613 | static std::optional<ProfileSummaryFormatKind> |
614 | convertProfileSummaryFormat(ModuleOp mlirModule, const llvm::Module *llvmModule, |
615 | const llvm::MDOperand &formatMD) { |
616 | auto *tupleEntry = getTwoElementMDTuple(mlirModule, llvmModule, formatMD); |
617 | if (!tupleEntry) |
618 | return std::nullopt; |
619 | |
620 | llvm::MDString *keyMD = dyn_cast<llvm::MDString>(tupleEntry->getOperand(0)); |
621 | if (!keyMD || keyMD->getString() != "ProfileFormat") { |
622 | emitWarning(mlirModule.getLoc()) |
623 | << "expected 'ProfileFormat' key: " |
624 | << diagMD(tupleEntry->getOperand(0), llvmModule); |
625 | return std::nullopt; |
626 | } |
627 | |
628 | llvm::MDString *valMD = dyn_cast<llvm::MDString>(tupleEntry->getOperand(1)); |
629 | std::optional<ProfileSummaryFormatKind> fmtKind = |
630 | symbolizeProfileSummaryFormatKind(valMD->getString()); |
631 | if (!fmtKind) { |
632 | emitWarning(mlirModule.getLoc()) |
633 | << "expected 'SampleProfile', 'InstrProf' or 'CSInstrProf' values, " |
634 | "but found: " |
635 | << diagMD(node: valMD, module: llvmModule); |
636 | return std::nullopt; |
637 | } |
638 | |
639 | return fmtKind; |
640 | } |
641 | |
642 | static FailureOr<SmallVector<ModuleFlagProfileSummaryDetailedAttr>> |
643 | convertProfileSummaryDetailed(ModuleOp mlirModule, |
644 | const llvm::Module *llvmModule, |
645 | const llvm::MDOperand &summaryMD) { |
646 | auto *tupleEntry = getTwoElementMDTuple(mlirModule, llvmModule, summaryMD); |
647 | if (!tupleEntry) |
648 | return failure(); |
649 | |
650 | llvm::MDString *keyMD = dyn_cast<llvm::MDString>(tupleEntry->getOperand(0)); |
651 | if (!keyMD || keyMD->getString() != "DetailedSummary") { |
652 | emitWarning(mlirModule.getLoc()) |
653 | << "expected 'DetailedSummary' key: " |
654 | << diagMD(tupleEntry->getOperand(0), llvmModule); |
655 | return failure(); |
656 | } |
657 | |
658 | llvm::MDTuple *entriesMD = dyn_cast<llvm::MDTuple>(tupleEntry->getOperand(1)); |
659 | if (!entriesMD) { |
660 | emitWarning(mlirModule.getLoc()) |
661 | << "expected tuple value for 'DetailedSummary' key: " |
662 | << diagMD(tupleEntry->getOperand(1), llvmModule); |
663 | return failure(); |
664 | } |
665 | |
666 | SmallVector<ModuleFlagProfileSummaryDetailedAttr> detailedSummary; |
667 | for (auto &&entry : entriesMD->operands()) { |
668 | llvm::MDTuple *entryMD = dyn_cast<llvm::MDTuple>(entry); |
669 | if (!entryMD || entryMD->getNumOperands() != 3) { |
670 | emitWarning(mlirModule.getLoc()) |
671 | << "'DetailedSummary' entry expects 3 operands: " |
672 | << diagMD(entry, llvmModule); |
673 | return failure(); |
674 | } |
675 | |
676 | auto *op0 = dyn_cast<llvm::ConstantAsMetadata>(entryMD->getOperand(0)); |
677 | auto *op1 = dyn_cast<llvm::ConstantAsMetadata>(entryMD->getOperand(1)); |
678 | auto *op2 = dyn_cast<llvm::ConstantAsMetadata>(entryMD->getOperand(2)); |
679 | if (!op0 || !op1 || !op2) { |
680 | emitWarning(mlirModule.getLoc()) |
681 | << "expected only integer entries in 'DetailedSummary': " |
682 | << diagMD(entry, llvmModule); |
683 | return failure(); |
684 | } |
685 | |
686 | auto detaildSummaryEntry = ModuleFlagProfileSummaryDetailedAttr::get( |
687 | mlirModule->getContext(), |
688 | cast<llvm::ConstantInt>(op0->getValue())->getZExtValue(), |
689 | cast<llvm::ConstantInt>(op1->getValue())->getZExtValue(), |
690 | cast<llvm::ConstantInt>(op2->getValue())->getZExtValue()); |
691 | detailedSummary.push_back(detaildSummaryEntry); |
692 | } |
693 | return detailedSummary; |
694 | } |
695 | |
696 | static Attribute |
697 | convertProfileSummaryModuleFlagValue(ModuleOp mlirModule, |
698 | const llvm::Module *llvmModule, |
699 | llvm::MDTuple *mdTuple) { |
700 | unsigned profileNumEntries = mdTuple->getNumOperands(); |
701 | if (profileNumEntries < 8) { |
702 | emitWarning(mlirModule.getLoc()) |
703 | << "expected at 8 entries in 'ProfileSummary': " |
704 | << diagMD(node: mdTuple, module: llvmModule); |
705 | return nullptr; |
706 | } |
707 | |
708 | unsigned summayIdx = 0; |
709 | auto checkOptionalPosition = [&](const llvm::MDOperand &md, |
710 | StringRef matchKey) -> LogicalResult { |
711 | // Make sure we won't step over the bound of the array of summary entries. |
712 | // Since (non-optional) DetailedSummary always comes last, the next entry in |
713 | // the tuple operand array must exist. |
714 | if (summayIdx + 1 >= profileNumEntries) { |
715 | emitWarning(mlirModule.getLoc()) |
716 | << "the last summary entry is '"<< matchKey |
717 | << "', expected 'DetailedSummary': "<< diagMD(node: md, module: llvmModule); |
718 | return failure(); |
719 | } |
720 | |
721 | return success(); |
722 | }; |
723 | |
724 | auto getOptIntValue = |
725 | [&](const llvm::MDOperand &md, |
726 | StringRef matchKey) -> FailureOr<std::optional<uint64_t>> { |
727 | if (!getConstantMDFromKeyValueTuple(mlirModule, llvmModule, md, matchKey, |
728 | /*optional=*/true)) |
729 | return FailureOr<std::optional<uint64_t>>(std::nullopt); |
730 | if (checkOptionalPosition(md, matchKey).failed()) |
731 | return failure(); |
732 | FailureOr<uint64_t> val = |
733 | convertInt64FromKeyValueTuple(mlirModule, llvmModule, md, matchKey); |
734 | if (failed(Result: val)) |
735 | return failure(); |
736 | return val; |
737 | }; |
738 | |
739 | auto getOptDoubleValue = [&](const llvm::MDOperand &md, |
740 | StringRef matchKey) -> FailureOr<FloatAttr> { |
741 | auto *valMD = getConstantMDFromKeyValueTuple(mlirModule, llvmModule, md, |
742 | matchKey, /*optional=*/true); |
743 | if (!valMD) |
744 | return FloatAttr{}; |
745 | if (auto *cstFP = dyn_cast<llvm::ConstantFP>(valMD->getValue())) { |
746 | if (checkOptionalPosition(md, matchKey).failed()) |
747 | return failure(); |
748 | return FloatAttr::get(Float64Type::get(mlirModule.getContext()), |
749 | cstFP->getValueAPF()); |
750 | } |
751 | emitWarning(mlirModule.getLoc()) |
752 | << "expected double metadata value for key '"<< matchKey |
753 | << "': "<< diagMD(node: md, module: llvmModule); |
754 | return failure(); |
755 | }; |
756 | |
757 | // Build ModuleFlagProfileSummaryAttr by sequentially fetching elements in |
758 | // a fixed order: format, total count, etc. |
759 | std::optional<ProfileSummaryFormatKind> format = convertProfileSummaryFormat( |
760 | mlirModule, llvmModule, mdTuple->getOperand(summayIdx++)); |
761 | if (!format.has_value()) |
762 | return nullptr; |
763 | |
764 | FailureOr<uint64_t> totalCount = convertInt64FromKeyValueTuple( |
765 | mlirModule, llvmModule, mdTuple->getOperand(I: summayIdx++), "TotalCount"); |
766 | if (failed(Result: totalCount)) |
767 | return nullptr; |
768 | |
769 | FailureOr<uint64_t> maxCount = convertInt64FromKeyValueTuple( |
770 | mlirModule, llvmModule, mdTuple->getOperand(I: summayIdx++), "MaxCount"); |
771 | if (failed(Result: maxCount)) |
772 | return nullptr; |
773 | |
774 | FailureOr<uint64_t> maxInternalCount = convertInt64FromKeyValueTuple( |
775 | mlirModule, llvmModule, mdTuple->getOperand(I: summayIdx++), |
776 | "MaxInternalCount"); |
777 | if (failed(Result: maxInternalCount)) |
778 | return nullptr; |
779 | |
780 | FailureOr<uint64_t> maxFunctionCount = convertInt64FromKeyValueTuple( |
781 | mlirModule, llvmModule, mdTuple->getOperand(I: summayIdx++), |
782 | "MaxFunctionCount"); |
783 | if (failed(Result: maxFunctionCount)) |
784 | return nullptr; |
785 | |
786 | FailureOr<uint64_t> numCounts = convertInt64FromKeyValueTuple( |
787 | mlirModule, llvmModule, mdTuple->getOperand(I: summayIdx++), "NumCounts"); |
788 | if (failed(Result: numCounts)) |
789 | return nullptr; |
790 | |
791 | FailureOr<uint64_t> numFunctions = convertInt64FromKeyValueTuple( |
792 | mlirModule, llvmModule, mdTuple->getOperand(I: summayIdx++), "NumFunctions"); |
793 | if (failed(Result: numFunctions)) |
794 | return nullptr; |
795 | |
796 | // Handle optional keys. |
797 | FailureOr<std::optional<uint64_t>> isPartialProfile = |
798 | getOptIntValue(mdTuple->getOperand(I: summayIdx), "IsPartialProfile"); |
799 | if (failed(Result: isPartialProfile)) |
800 | return nullptr; |
801 | if (isPartialProfile->has_value()) |
802 | summayIdx++; |
803 | |
804 | FailureOr<FloatAttr> partialProfileRatio = |
805 | getOptDoubleValue(mdTuple->getOperand(I: summayIdx), "PartialProfileRatio"); |
806 | if (failed(Result: partialProfileRatio)) |
807 | return nullptr; |
808 | if (*partialProfileRatio) |
809 | summayIdx++; |
810 | |
811 | // Handle detailed summary. |
812 | FailureOr<SmallVector<ModuleFlagProfileSummaryDetailedAttr>> detailed = |
813 | convertProfileSummaryDetailed(mlirModule, llvmModule, |
814 | mdTuple->getOperand(summayIdx)); |
815 | if (failed(detailed)) |
816 | return nullptr; |
817 | |
818 | // Build the final profile summary attribute. |
819 | return ModuleFlagProfileSummaryAttr::get( |
820 | mlirModule->getContext(), *format, *totalCount, *maxCount, |
821 | *maxInternalCount, *maxFunctionCount, *numCounts, *numFunctions, |
822 | *isPartialProfile, *partialProfileRatio, *detailed); |
823 | } |
824 | |
825 | /// Invoke specific handlers for each known module flag value, returns nullptr |
826 | /// if the key is unknown or unimplemented. |
827 | static Attribute |
828 | convertModuleFlagValueFromMDTuple(ModuleOp mlirModule, |
829 | const llvm::Module *llvmModule, StringRef key, |
830 | llvm::MDTuple *mdTuple) { |
831 | if (key == LLVMDialect::getModuleFlagKeyCGProfileName()) |
832 | return convertCGProfileModuleFlagValue(mlirModule, mdTuple); |
833 | if (key == LLVMDialect::getModuleFlagKeyProfileSummaryName()) |
834 | return convertProfileSummaryModuleFlagValue(mlirModule, llvmModule, |
835 | mdTuple); |
836 | return nullptr; |
837 | } |
838 | |
839 | LogicalResult ModuleImport::convertModuleFlagsMetadata() { |
840 | SmallVector<llvm::Module::ModuleFlagEntry> llvmModuleFlags; |
841 | llvmModule->getModuleFlagsMetadata(Flags&: llvmModuleFlags); |
842 | |
843 | SmallVector<Attribute> moduleFlags; |
844 | for (const auto [behavior, key, val] : llvmModuleFlags) { |
845 | Attribute valAttr = nullptr; |
846 | if (auto *constInt = llvm::mdconst::dyn_extract<llvm::ConstantInt>(MD: val)) { |
847 | valAttr = builder.getI32IntegerAttr(constInt->getZExtValue()); |
848 | } else if (auto *mdString = dyn_cast<llvm::MDString>(Val: val)) { |
849 | valAttr = builder.getStringAttr(mdString->getString()); |
850 | } else if (auto *mdTuple = dyn_cast<llvm::MDTuple>(Val: val)) { |
851 | valAttr = convertModuleFlagValueFromMDTuple(mlirModule, llvmModule.get(), |
852 | key->getString(), mdTuple); |
853 | } |
854 | |
855 | if (!valAttr) { |
856 | emitWarning(mlirModule.getLoc()) |
857 | << "unsupported module flag value for key '"<< key->getString() |
858 | << "' : "<< diagMD(val, llvmModule.get()); |
859 | continue; |
860 | } |
861 | |
862 | moduleFlags.push_back(builder.getAttr<ModuleFlagAttr>( |
863 | convertModFlagBehaviorFromLLVM(behavior), |
864 | builder.getStringAttr(key->getString()), valAttr)); |
865 | } |
866 | |
867 | if (!moduleFlags.empty()) |
868 | builder.create<LLVM::ModuleFlagsOp>(mlirModule.getLoc(), |
869 | builder.getArrayAttr(moduleFlags)); |
870 | |
871 | return success(); |
872 | } |
873 | |
874 | LogicalResult ModuleImport::convertLinkerOptionsMetadata() { |
875 | for (const llvm::NamedMDNode &named : llvmModule->named_metadata()) { |
876 | if (named.getName() != "llvm.linker.options") |
877 | continue; |
878 | // llvm.linker.options operands are lists of strings. |
879 | for (const llvm::MDNode *node : named.operands()) { |
880 | SmallVector<StringRef> options; |
881 | options.reserve(N: node->getNumOperands()); |
882 | for (const llvm::MDOperand &option : node->operands()) |
883 | options.push_back(Elt: cast<llvm::MDString>(Val: option)->getString()); |
884 | builder.create<LLVM::LinkerOptionsOp>(mlirModule.getLoc(), |
885 | builder.getStrArrayAttr(options)); |
886 | } |
887 | } |
888 | return success(); |
889 | } |
890 | |
891 | LogicalResult ModuleImport::convertDependentLibrariesMetadata() { |
892 | for (const llvm::NamedMDNode &named : llvmModule->named_metadata()) { |
893 | if (named.getName() != "llvm.dependent-libraries") |
894 | continue; |
895 | SmallVector<StringRef> libraries; |
896 | for (const llvm::MDNode *node : named.operands()) { |
897 | if (node->getNumOperands() == 1) |
898 | if (auto *mdString = dyn_cast<llvm::MDString>(Val: node->getOperand(I: 0))) |
899 | libraries.push_back(Elt: mdString->getString()); |
900 | } |
901 | if (!libraries.empty()) |
902 | mlirModule->setAttr(LLVM::LLVMDialect::getDependentLibrariesAttrName(), |
903 | builder.getStrArrayAttr(libraries)); |
904 | } |
905 | return success(); |
906 | } |
907 | |
908 | LogicalResult ModuleImport::convertIdentMetadata() { |
909 | for (const llvm::NamedMDNode &named : llvmModule->named_metadata()) { |
910 | // llvm.ident should have a single operand. That operand is itself an |
911 | // MDNode with a single string operand. |
912 | if (named.getName() != LLVMDialect::getIdentAttrName()) |
913 | continue; |
914 | |
915 | if (named.getNumOperands() == 1) |
916 | if (auto *md = dyn_cast<llvm::MDNode>(named.getOperand(0))) |
917 | if (md->getNumOperands() == 1) |
918 | if (auto *mdStr = dyn_cast<llvm::MDString>(md->getOperand(0))) |
919 | mlirModule->setAttr(LLVMDialect::getIdentAttrName(), |
920 | builder.getStringAttr(mdStr->getString())); |
921 | } |
922 | return success(); |
923 | } |
924 | |
925 | LogicalResult ModuleImport::convertCommandlineMetadata() { |
926 | for (const llvm::NamedMDNode &nmd : llvmModule->named_metadata()) { |
927 | // llvm.commandline should have a single operand. That operand is itself an |
928 | // MDNode with a single string operand. |
929 | if (nmd.getName() != LLVMDialect::getCommandlineAttrName()) |
930 | continue; |
931 | |
932 | if (nmd.getNumOperands() == 1) |
933 | if (auto *md = dyn_cast<llvm::MDNode>(nmd.getOperand(0))) |
934 | if (md->getNumOperands() == 1) |
935 | if (auto *mdStr = dyn_cast<llvm::MDString>(md->getOperand(0))) |
936 | mlirModule->setAttr(LLVMDialect::getCommandlineAttrName(), |
937 | builder.getStringAttr(mdStr->getString())); |
938 | } |
939 | return success(); |
940 | } |
941 | |
942 | LogicalResult ModuleImport::convertMetadata() { |
943 | OpBuilder::InsertionGuard guard(builder); |
944 | builder.setInsertionPointToEnd(mlirModule.getBody()); |
945 | for (const llvm::Function &func : llvmModule->functions()) { |
946 | for (const llvm::Instruction &inst : llvm::instructions(F: func)) { |
947 | // Convert access group metadata nodes. |
948 | if (llvm::MDNode *node = |
949 | inst.getMetadata(KindID: llvm::LLVMContext::MD_access_group)) |
950 | if (failed(Result: processAccessGroupMetadata(node))) |
951 | return failure(); |
952 | |
953 | // Convert alias analysis metadata nodes. |
954 | llvm::AAMDNodes aliasAnalysisNodes = inst.getAAMetadata(); |
955 | if (!aliasAnalysisNodes) |
956 | continue; |
957 | if (aliasAnalysisNodes.TBAA) |
958 | if (failed(Result: processTBAAMetadata(node: aliasAnalysisNodes.TBAA))) |
959 | return failure(); |
960 | if (aliasAnalysisNodes.Scope) |
961 | if (failed(Result: processAliasScopeMetadata(node: aliasAnalysisNodes.Scope))) |
962 | return failure(); |
963 | if (aliasAnalysisNodes.NoAlias) |
964 | if (failed(Result: processAliasScopeMetadata(node: aliasAnalysisNodes.NoAlias))) |
965 | return failure(); |
966 | } |
967 | } |
968 | if (failed(Result: convertLinkerOptionsMetadata())) |
969 | return failure(); |
970 | if (failed(Result: convertDependentLibrariesMetadata())) |
971 | return failure(); |
972 | if (failed(Result: convertModuleFlagsMetadata())) |
973 | return failure(); |
974 | if (failed(Result: convertIdentMetadata())) |
975 | return failure(); |
976 | if (failed(Result: convertCommandlineMetadata())) |
977 | return failure(); |
978 | return success(); |
979 | } |
980 | |
981 | void ModuleImport::processComdat(const llvm::Comdat *comdat) { |
982 | if (comdatMapping.contains(Val: comdat)) |
983 | return; |
984 | |
985 | ComdatOp comdatOp = getGlobalComdatOp(); |
986 | OpBuilder::InsertionGuard guard(builder); |
987 | builder.setInsertionPointToEnd(&comdatOp.getBody().back()); |
988 | auto selectorOp = builder.create<ComdatSelectorOp>( |
989 | mlirModule.getLoc(), comdat->getName(), |
990 | convertComdatFromLLVM(comdat->getSelectionKind())); |
991 | auto symbolRef = |
992 | SymbolRefAttr::get(builder.getContext(), getGlobalComdatOpName(), |
993 | FlatSymbolRefAttr::get(selectorOp.getSymNameAttr())); |
994 | comdatMapping.try_emplace(comdat, symbolRef); |
995 | } |
996 | |
997 | LogicalResult ModuleImport::convertComdats() { |
998 | for (llvm::GlobalVariable &globalVar : llvmModule->globals()) |
999 | if (globalVar.hasComdat()) |
1000 | processComdat(comdat: globalVar.getComdat()); |
1001 | for (llvm::Function &func : llvmModule->functions()) |
1002 | if (func.hasComdat()) |
1003 | processComdat(comdat: func.getComdat()); |
1004 | return success(); |
1005 | } |
1006 | |
1007 | LogicalResult ModuleImport::convertGlobals() { |
1008 | for (llvm::GlobalVariable &globalVar : llvmModule->globals()) { |
1009 | if (globalVar.getName() == getGlobalCtorsVarName() || |
1010 | globalVar.getName() == getGlobalDtorsVarName()) { |
1011 | if (failed(Result: convertGlobalCtorsAndDtors(globalVar: &globalVar))) { |
1012 | return emitError(UnknownLoc::get(context)) |
1013 | << "unhandled global variable: "<< diag(globalVar); |
1014 | } |
1015 | continue; |
1016 | } |
1017 | if (failed(Result: convertGlobal(globalVar: &globalVar))) { |
1018 | return emitError(UnknownLoc::get(context)) |
1019 | << "unhandled global variable: "<< diag(globalVar); |
1020 | } |
1021 | } |
1022 | return success(); |
1023 | } |
1024 | |
1025 | LogicalResult ModuleImport::convertAliases() { |
1026 | for (llvm::GlobalAlias &alias : llvmModule->aliases()) { |
1027 | if (failed(Result: convertAlias(alias: &alias))) { |
1028 | return emitError(UnknownLoc::get(context)) |
1029 | << "unhandled global alias: "<< diag(alias); |
1030 | } |
1031 | } |
1032 | return success(); |
1033 | } |
1034 | |
1035 | LogicalResult ModuleImport::convertDataLayout() { |
1036 | Location loc = mlirModule.getLoc(); |
1037 | DataLayoutImporter dataLayoutImporter(context, llvmModule->getDataLayout()); |
1038 | if (!dataLayoutImporter.getDataLayout()) |
1039 | return emitError(loc, message: "cannot translate data layout: ") |
1040 | << dataLayoutImporter.getLastToken(); |
1041 | |
1042 | for (StringRef token : dataLayoutImporter.getUnhandledTokens()) |
1043 | emitWarning(loc, message: "unhandled data layout token: ") << token; |
1044 | |
1045 | mlirModule->setAttr(DLTIDialect::kDataLayoutAttrName, |
1046 | dataLayoutImporter.getDataLayout()); |
1047 | return success(); |
1048 | } |
1049 | |
1050 | void ModuleImport::convertTargetTriple() { |
1051 | mlirModule->setAttr( |
1052 | LLVM::LLVMDialect::getTargetTripleAttrName(), |
1053 | builder.getStringAttr(llvmModule->getTargetTriple().str())); |
1054 | } |
1055 | |
1056 | LogicalResult ModuleImport::convertFunctions() { |
1057 | for (llvm::Function &func : llvmModule->functions()) |
1058 | if (failed(Result: processFunction(func: &func))) |
1059 | return failure(); |
1060 | return success(); |
1061 | } |
1062 | |
1063 | void ModuleImport::setNonDebugMetadataAttrs(llvm::Instruction *inst, |
1064 | Operation *op) { |
1065 | SmallVector<std::pair<unsigned, llvm::MDNode *>> allMetadata; |
1066 | inst->getAllMetadataOtherThanDebugLoc(MDs&: allMetadata); |
1067 | for (auto &[kind, node] : allMetadata) { |
1068 | if (!iface.isConvertibleMetadata(kind)) |
1069 | continue; |
1070 | if (failed(Result: iface.setMetadataAttrs(builder, kind, node, op, moduleImport&: *this))) { |
1071 | if (emitExpensiveWarnings) { |
1072 | Location loc = debugImporter->translateLoc(loc: inst->getDebugLoc()); |
1073 | emitWarning(loc) << "unhandled metadata: " |
1074 | << diagMD(node, module: llvmModule.get()) << " on " |
1075 | << diag(value: *inst); |
1076 | } |
1077 | } |
1078 | } |
1079 | } |
1080 | |
1081 | void ModuleImport::setIntegerOverflowFlags(llvm::Instruction *inst, |
1082 | Operation *op) const { |
1083 | auto iface = cast<IntegerOverflowFlagsInterface>(op); |
1084 | |
1085 | IntegerOverflowFlags value = {}; |
1086 | value = bitEnumSet(value, IntegerOverflowFlags::nsw, inst->hasNoSignedWrap()); |
1087 | value = |
1088 | bitEnumSet(value, IntegerOverflowFlags::nuw, inst->hasNoUnsignedWrap()); |
1089 | |
1090 | iface.setOverflowFlags(value); |
1091 | } |
1092 | |
1093 | void ModuleImport::setExactFlag(llvm::Instruction *inst, Operation *op) const { |
1094 | auto iface = cast<ExactFlagInterface>(op); |
1095 | |
1096 | iface.setIsExact(inst->isExact()); |
1097 | } |
1098 | |
1099 | void ModuleImport::setDisjointFlag(llvm::Instruction *inst, |
1100 | Operation *op) const { |
1101 | auto iface = cast<DisjointFlagInterface>(op); |
1102 | auto instDisjoint = cast<llvm::PossiblyDisjointInst>(Val: inst); |
1103 | |
1104 | iface.setIsDisjoint(instDisjoint->isDisjoint()); |
1105 | } |
1106 | |
1107 | void ModuleImport::setNonNegFlag(llvm::Instruction *inst, Operation *op) const { |
1108 | auto iface = cast<NonNegFlagInterface>(op); |
1109 | |
1110 | iface.setNonNeg(inst->hasNonNeg()); |
1111 | } |
1112 | |
1113 | void ModuleImport::setFastmathFlagsAttr(llvm::Instruction *inst, |
1114 | Operation *op) const { |
1115 | auto iface = cast<FastmathFlagsInterface>(op); |
1116 | |
1117 | // Even if the imported operation implements the fastmath interface, the |
1118 | // original instruction may not have fastmath flags set. Exit if an |
1119 | // instruction, such as a non floating-point function call, does not have |
1120 | // fastmath flags. |
1121 | if (!isa<llvm::FPMathOperator>(Val: inst)) |
1122 | return; |
1123 | llvm::FastMathFlags flags = inst->getFastMathFlags(); |
1124 | |
1125 | // Set the fastmath bits flag-by-flag. |
1126 | FastmathFlags value = {}; |
1127 | value = bitEnumSet(value, FastmathFlags::nnan, flags.noNaNs()); |
1128 | value = bitEnumSet(value, FastmathFlags::ninf, flags.noInfs()); |
1129 | value = bitEnumSet(value, FastmathFlags::nsz, flags.noSignedZeros()); |
1130 | value = bitEnumSet(value, FastmathFlags::arcp, flags.allowReciprocal()); |
1131 | value = bitEnumSet(value, FastmathFlags::contract, flags.allowContract()); |
1132 | value = bitEnumSet(value, FastmathFlags::afn, flags.approxFunc()); |
1133 | value = bitEnumSet(value, FastmathFlags::reassoc, flags.allowReassoc()); |
1134 | FastmathFlagsAttr attr = FastmathFlagsAttr::get(builder.getContext(), value); |
1135 | iface->setAttr(iface.getFastmathAttrName(), attr); |
1136 | } |
1137 | |
1138 | /// Returns `type` if it is a builtin integer or floating-point vector type that |
1139 | /// can be used to create an attribute or nullptr otherwise. If provided, |
1140 | /// `arrayShape` is added to the shape of the vector to create an attribute that |
1141 | /// matches an array of vectors. |
1142 | static Type getVectorTypeForAttr(Type type, ArrayRef<int64_t> arrayShape = {}) { |
1143 | if (!LLVM::isCompatibleVectorType(type)) |
1144 | return {}; |
1145 | |
1146 | llvm::ElementCount numElements = LLVM::getVectorNumElements(type); |
1147 | if (numElements.isScalable()) { |
1148 | emitError(UnknownLoc::get(type.getContext())) |
1149 | << "scalable vectors not supported"; |
1150 | return {}; |
1151 | } |
1152 | |
1153 | // An LLVM dialect vector can only contain scalars. |
1154 | Type elementType = cast<VectorType>(type).getElementType(); |
1155 | if (!elementType.isIntOrFloat()) |
1156 | return {}; |
1157 | |
1158 | SmallVector<int64_t> shape(arrayShape); |
1159 | shape.push_back(Elt: numElements.getKnownMinValue()); |
1160 | return VectorType::get(shape, elementType); |
1161 | } |
1162 | |
1163 | Type ModuleImport::getBuiltinTypeForAttr(Type type) { |
1164 | if (!type) |
1165 | return {}; |
1166 | |
1167 | // Return builtin integer and floating-point types as is. |
1168 | if (type.isIntOrFloat()) |
1169 | return type; |
1170 | |
1171 | // Return builtin vectors of integer and floating-point types as is. |
1172 | if (Type vectorType = getVectorTypeForAttr(type)) |
1173 | return vectorType; |
1174 | |
1175 | // Multi-dimensional array types are converted to tensors or vectors, |
1176 | // depending on the innermost type being a scalar or a vector. |
1177 | SmallVector<int64_t> arrayShape; |
1178 | while (auto arrayType = dyn_cast<LLVMArrayType>(type)) { |
1179 | arrayShape.push_back(Elt: arrayType.getNumElements()); |
1180 | type = arrayType.getElementType(); |
1181 | } |
1182 | if (type.isIntOrFloat()) |
1183 | return RankedTensorType::get(arrayShape, type); |
1184 | return getVectorTypeForAttr(type, arrayShape); |
1185 | } |
1186 | |
1187 | /// Returns an integer or float attribute for the provided scalar constant |
1188 | /// `constScalar` or nullptr if the conversion fails. |
1189 | static TypedAttr getScalarConstantAsAttr(OpBuilder &builder, |
1190 | llvm::Constant *constScalar) { |
1191 | MLIRContext *context = builder.getContext(); |
1192 | |
1193 | // Convert scalar intergers. |
1194 | if (auto *constInt = dyn_cast<llvm::ConstantInt>(Val: constScalar)) { |
1195 | return builder.getIntegerAttr( |
1196 | IntegerType::get(context, constInt->getBitWidth()), |
1197 | constInt->getValue()); |
1198 | } |
1199 | |
1200 | // Convert scalar floats. |
1201 | if (auto *constFloat = dyn_cast<llvm::ConstantFP>(Val: constScalar)) { |
1202 | llvm::Type *type = constFloat->getType(); |
1203 | FloatType floatType = |
1204 | type->isBFloatTy() |
1205 | ? BFloat16Type::get(context) |
1206 | : LLVM::detail::getFloatType(context, type->getScalarSizeInBits()); |
1207 | if (!floatType) { |
1208 | emitError(UnknownLoc::get(builder.getContext())) |
1209 | << "unexpected floating-point type"; |
1210 | return {}; |
1211 | } |
1212 | return builder.getFloatAttr(floatType, constFloat->getValueAPF()); |
1213 | } |
1214 | return {}; |
1215 | } |
1216 | |
1217 | /// Returns an integer or float attribute array for the provided constant |
1218 | /// sequence `constSequence` or nullptr if the conversion fails. |
1219 | static SmallVector<Attribute> |
1220 | getSequenceConstantAsAttrs(OpBuilder &builder, |
1221 | llvm::ConstantDataSequential *constSequence) { |
1222 | SmallVector<Attribute> elementAttrs; |
1223 | elementAttrs.reserve(N: constSequence->getNumElements()); |
1224 | for (auto idx : llvm::seq<int64_t>(Begin: 0, End: constSequence->getNumElements())) { |
1225 | llvm::Constant *constElement = constSequence->getElementAsConstant(i: idx); |
1226 | elementAttrs.push_back(getScalarConstantAsAttr(builder, constElement)); |
1227 | } |
1228 | return elementAttrs; |
1229 | } |
1230 | |
1231 | Attribute ModuleImport::getConstantAsAttr(llvm::Constant *constant) { |
1232 | // Convert scalar constants. |
1233 | if (Attribute scalarAttr = getScalarConstantAsAttr(builder, constant)) |
1234 | return scalarAttr; |
1235 | |
1236 | // Returns the static shape of the provided type if possible. |
1237 | auto getConstantShape = [&](llvm::Type *type) { |
1238 | return llvm::dyn_cast_if_present<ShapedType>( |
1239 | getBuiltinTypeForAttr(convertType(type))); |
1240 | }; |
1241 | |
1242 | // Convert one-dimensional constant arrays or vectors that store 1/2/4/8-byte |
1243 | // integer or half/bfloat/float/double values. |
1244 | if (auto *constArray = dyn_cast<llvm::ConstantDataSequential>(Val: constant)) { |
1245 | if (constArray->isString()) |
1246 | return builder.getStringAttr(constArray->getAsString()); |
1247 | auto shape = getConstantShape(constArray->getType()); |
1248 | if (!shape) |
1249 | return {}; |
1250 | // Convert splat constants to splat elements attributes. |
1251 | auto *constVector = dyn_cast<llvm::ConstantDataVector>(Val: constant); |
1252 | if (constVector && constVector->isSplat()) { |
1253 | // A vector is guaranteed to have at least size one. |
1254 | Attribute splatAttr = getScalarConstantAsAttr( |
1255 | builder, constVector->getElementAsConstant(0)); |
1256 | return SplatElementsAttr::get(shape, splatAttr); |
1257 | } |
1258 | // Convert non-splat constants to dense elements attributes. |
1259 | SmallVector<Attribute> elementAttrs = |
1260 | getSequenceConstantAsAttrs(builder, constSequence: constArray); |
1261 | return DenseElementsAttr::get(shape, elementAttrs); |
1262 | } |
1263 | |
1264 | // Convert multi-dimensional constant aggregates that store all kinds of |
1265 | // integer and floating-point types. |
1266 | if (auto *constAggregate = dyn_cast<llvm::ConstantAggregate>(Val: constant)) { |
1267 | auto shape = getConstantShape(constAggregate->getType()); |
1268 | if (!shape) |
1269 | return {}; |
1270 | // Collect the aggregate elements in depths first order. |
1271 | SmallVector<Attribute> elementAttrs; |
1272 | SmallVector<llvm::Constant *> workList = {constAggregate}; |
1273 | while (!workList.empty()) { |
1274 | llvm::Constant *current = workList.pop_back_val(); |
1275 | // Append any nested aggregates in reverse order to ensure the head |
1276 | // element of the nested aggregates is at the back of the work list. |
1277 | if (auto *constAggregate = dyn_cast<llvm::ConstantAggregate>(Val: current)) { |
1278 | for (auto idx : |
1279 | reverse(C: llvm::seq<int64_t>(Begin: 0, End: constAggregate->getNumOperands()))) |
1280 | workList.push_back(Elt: constAggregate->getAggregateElement(Elt: idx)); |
1281 | continue; |
1282 | } |
1283 | // Append the elements of nested constant arrays or vectors that store |
1284 | // 1/2/4/8-byte integer or half/bfloat/float/double values. |
1285 | if (auto *constArray = dyn_cast<llvm::ConstantDataSequential>(Val: current)) { |
1286 | SmallVector<Attribute> attrs = |
1287 | getSequenceConstantAsAttrs(builder, constSequence: constArray); |
1288 | elementAttrs.append(in_start: attrs.begin(), in_end: attrs.end()); |
1289 | continue; |
1290 | } |
1291 | // Append nested scalar constants that store all kinds of integer and |
1292 | // floating-point types. |
1293 | if (Attribute scalarAttr = getScalarConstantAsAttr(builder, current)) { |
1294 | elementAttrs.push_back(Elt: scalarAttr); |
1295 | continue; |
1296 | } |
1297 | // Bail if the aggregate contains a unsupported constant type such as a |
1298 | // constant expression. |
1299 | return {}; |
1300 | } |
1301 | return DenseElementsAttr::get(shape, elementAttrs); |
1302 | } |
1303 | |
1304 | // Convert zero aggregates. |
1305 | if (auto *constZero = dyn_cast<llvm::ConstantAggregateZero>(Val: constant)) { |
1306 | auto shape = llvm::dyn_cast_if_present<ShapedType>( |
1307 | getBuiltinTypeForAttr(convertType(constZero->getType()))); |
1308 | if (!shape) |
1309 | return {}; |
1310 | // Convert zero aggregates with a static shape to splat elements attributes. |
1311 | Attribute splatAttr = builder.getZeroAttr(type: shape.getElementType()); |
1312 | assert(splatAttr && "expected non-null zero attribute for scalar types"); |
1313 | return SplatElementsAttr::get(shape, splatAttr); |
1314 | } |
1315 | return {}; |
1316 | } |
1317 | |
1318 | FlatSymbolRefAttr |
1319 | ModuleImport::getOrCreateNamelessSymbolName(llvm::GlobalVariable *globalVar) { |
1320 | assert(globalVar->getName().empty() && |
1321 | "expected to work with a nameless global"); |
1322 | auto [it, success] = namelessGlobals.try_emplace(Key: globalVar); |
1323 | if (!success) |
1324 | return it->second; |
1325 | |
1326 | // Make sure the symbol name does not clash with an existing symbol. |
1327 | SmallString<128> globalName = SymbolTable::generateSymbolName<128>( |
1328 | name: getNamelessGlobalPrefix(), |
1329 | uniqueChecker: [this](StringRef newName) { return llvmModule->getNamedValue(Name: newName); }, |
1330 | uniquingCounter&: namelessGlobalId); |
1331 | auto symbolRef = FlatSymbolRefAttr::get(ctx: context, value: globalName); |
1332 | it->getSecond() = symbolRef; |
1333 | return symbolRef; |
1334 | } |
1335 | |
1336 | OpBuilder::InsertionGuard ModuleImport::setGlobalInsertionPoint() { |
1337 | OpBuilder::InsertionGuard guard(builder); |
1338 | if (globalInsertionOp) |
1339 | builder.setInsertionPointAfter(globalInsertionOp); |
1340 | else |
1341 | builder.setInsertionPointToStart(mlirModule.getBody()); |
1342 | return guard; |
1343 | } |
1344 | |
1345 | LogicalResult ModuleImport::convertAlias(llvm::GlobalAlias *alias) { |
1346 | // Insert the alias after the last one or at the start of the module. |
1347 | OpBuilder::InsertionGuard guard = setGlobalInsertionPoint(); |
1348 | |
1349 | Type type = convertType(type: alias->getValueType()); |
1350 | AliasOp aliasOp = builder.create<AliasOp>( |
1351 | mlirModule.getLoc(), type, convertLinkageFromLLVM(alias->getLinkage()), |
1352 | alias->getName(), |
1353 | /*dso_local=*/alias->isDSOLocal(), |
1354 | /*thread_local=*/alias->isThreadLocal(), |
1355 | /*attrs=*/ArrayRef<NamedAttribute>()); |
1356 | globalInsertionOp = aliasOp; |
1357 | |
1358 | clearRegionState(); |
1359 | Block *block = builder.createBlock(&aliasOp.getInitializerRegion()); |
1360 | setConstantInsertionPointToStart(block); |
1361 | FailureOr<Value> initializer = convertConstantExpr(constant: alias->getAliasee()); |
1362 | if (failed(Result: initializer)) |
1363 | return failure(); |
1364 | builder.create<ReturnOp>(aliasOp.getLoc(), *initializer); |
1365 | |
1366 | if (alias->hasAtLeastLocalUnnamedAddr()) |
1367 | aliasOp.setUnnamedAddr(convertUnnamedAddrFromLLVM(alias->getUnnamedAddr())); |
1368 | aliasOp.setVisibility_(convertVisibilityFromLLVM(alias->getVisibility())); |
1369 | |
1370 | return success(); |
1371 | } |
1372 | |
1373 | LogicalResult ModuleImport::convertGlobal(llvm::GlobalVariable *globalVar) { |
1374 | // Insert the global after the last one or at the start of the module. |
1375 | OpBuilder::InsertionGuard guard = setGlobalInsertionPoint(); |
1376 | |
1377 | Attribute valueAttr; |
1378 | if (globalVar->hasInitializer()) |
1379 | valueAttr = getConstantAsAttr(constant: globalVar->getInitializer()); |
1380 | Type type = convertType(type: globalVar->getValueType()); |
1381 | |
1382 | uint64_t alignment = 0; |
1383 | llvm::MaybeAlign maybeAlign = globalVar->getAlign(); |
1384 | if (maybeAlign.has_value()) { |
1385 | llvm::Align align = *maybeAlign; |
1386 | alignment = align.value(); |
1387 | } |
1388 | |
1389 | // Get the global expression associated with this global variable and convert |
1390 | // it. |
1391 | SmallVector<Attribute> globalExpressionAttrs; |
1392 | SmallVector<llvm::DIGlobalVariableExpression *> globalExpressions; |
1393 | globalVar->getDebugInfo(GVs&: globalExpressions); |
1394 | |
1395 | for (llvm::DIGlobalVariableExpression *expr : globalExpressions) { |
1396 | DIGlobalVariableExpressionAttr globalExpressionAttr = |
1397 | debugImporter->translateGlobalVariableExpression(expr); |
1398 | globalExpressionAttrs.push_back(Elt: globalExpressionAttr); |
1399 | } |
1400 | |
1401 | // Workaround to support LLVM's nameless globals. MLIR, in contrast to LLVM, |
1402 | // always requires a symbol name. |
1403 | StringRef globalName = globalVar->getName(); |
1404 | if (globalName.empty()) |
1405 | globalName = getOrCreateNamelessSymbolName(globalVar).getValue(); |
1406 | |
1407 | GlobalOp globalOp = builder.create<GlobalOp>( |
1408 | mlirModule.getLoc(), type, globalVar->isConstant(), |
1409 | convertLinkageFromLLVM(globalVar->getLinkage()), StringRef(globalName), |
1410 | valueAttr, alignment, /*addr_space=*/globalVar->getAddressSpace(), |
1411 | /*dso_local=*/globalVar->isDSOLocal(), |
1412 | /*thread_local=*/globalVar->isThreadLocal(), /*comdat=*/SymbolRefAttr(), |
1413 | /*attrs=*/ArrayRef<NamedAttribute>(), /*dbgExprs=*/globalExpressionAttrs); |
1414 | globalInsertionOp = globalOp; |
1415 | |
1416 | if (globalVar->hasInitializer() && !valueAttr) { |
1417 | clearRegionState(); |
1418 | Block *block = builder.createBlock(&globalOp.getInitializerRegion()); |
1419 | setConstantInsertionPointToStart(block); |
1420 | FailureOr<Value> initializer = |
1421 | convertConstantExpr(constant: globalVar->getInitializer()); |
1422 | if (failed(Result: initializer)) |
1423 | return failure(); |
1424 | builder.create<ReturnOp>(globalOp.getLoc(), *initializer); |
1425 | } |
1426 | if (globalVar->hasAtLeastLocalUnnamedAddr()) { |
1427 | globalOp.setUnnamedAddr( |
1428 | convertUnnamedAddrFromLLVM(globalVar->getUnnamedAddr())); |
1429 | } |
1430 | if (globalVar->hasSection()) |
1431 | globalOp.setSection(globalVar->getSection()); |
1432 | globalOp.setVisibility_( |
1433 | convertVisibilityFromLLVM(globalVar->getVisibility())); |
1434 | |
1435 | if (globalVar->hasComdat()) |
1436 | globalOp.setComdatAttr(comdatMapping.lookup(globalVar->getComdat())); |
1437 | |
1438 | return success(); |
1439 | } |
1440 | |
1441 | LogicalResult |
1442 | ModuleImport::convertGlobalCtorsAndDtors(llvm::GlobalVariable *globalVar) { |
1443 | if (!globalVar->hasInitializer() || !globalVar->hasAppendingLinkage()) |
1444 | return failure(); |
1445 | llvm::Constant *initializer = globalVar->getInitializer(); |
1446 | |
1447 | bool knownInit = isa<llvm::ConstantArray>(Val: initializer) || |
1448 | isa<llvm::ConstantAggregateZero>(Val: initializer); |
1449 | if (!knownInit) |
1450 | return failure(); |
1451 | |
1452 | // ConstantAggregateZero does not engage with the operand initialization |
1453 | // in the loop that follows - there should be no operands. This implies |
1454 | // empty ctor/dtor lists. |
1455 | if (auto *caz = dyn_cast<llvm::ConstantAggregateZero>(Val: initializer)) { |
1456 | if (caz->getElementCount().getFixedValue() != 0) |
1457 | return failure(); |
1458 | } |
1459 | |
1460 | SmallVector<Attribute> funcs; |
1461 | SmallVector<int32_t> priorities; |
1462 | SmallVector<Attribute> dataList; |
1463 | for (llvm::Value *operand : initializer->operands()) { |
1464 | auto *aggregate = dyn_cast<llvm::ConstantAggregate>(Val: operand); |
1465 | if (!aggregate || aggregate->getNumOperands() != 3) |
1466 | return failure(); |
1467 | |
1468 | auto *priority = dyn_cast<llvm::ConstantInt>(Val: aggregate->getOperand(i_nocapture: 0)); |
1469 | auto *func = dyn_cast<llvm::Function>(Val: aggregate->getOperand(i_nocapture: 1)); |
1470 | auto *data = dyn_cast<llvm::Constant>(Val: aggregate->getOperand(i_nocapture: 2)); |
1471 | if (!priority || !func || !data) |
1472 | return failure(); |
1473 | |
1474 | auto *gv = dyn_cast_or_null<llvm::GlobalValue>(Val: data); |
1475 | Attribute dataAttr; |
1476 | if (gv) |
1477 | dataAttr = FlatSymbolRefAttr::get(ctx: context, value: gv->getName()); |
1478 | else if (data->isNullValue()) |
1479 | dataAttr = ZeroAttr::get(context); |
1480 | else |
1481 | return failure(); |
1482 | |
1483 | funcs.push_back(FlatSymbolRefAttr::get(ctx: context, value: func->getName())); |
1484 | priorities.push_back(Elt: priority->getValue().getZExtValue()); |
1485 | dataList.push_back(Elt: dataAttr); |
1486 | } |
1487 | |
1488 | // Insert the global after the last one or at the start of the module. |
1489 | OpBuilder::InsertionGuard guard = setGlobalInsertionPoint(); |
1490 | |
1491 | if (globalVar->getName() == getGlobalCtorsVarName()) { |
1492 | globalInsertionOp = builder.create<LLVM::GlobalCtorsOp>( |
1493 | mlirModule.getLoc(), builder.getArrayAttr(funcs), |
1494 | builder.getI32ArrayAttr(priorities), builder.getArrayAttr(dataList)); |
1495 | return success(); |
1496 | } |
1497 | globalInsertionOp = builder.create<LLVM::GlobalDtorsOp>( |
1498 | mlirModule.getLoc(), builder.getArrayAttr(funcs), |
1499 | builder.getI32ArrayAttr(priorities), builder.getArrayAttr(dataList)); |
1500 | return success(); |
1501 | } |
1502 | |
1503 | SetVector<llvm::Constant *> |
1504 | ModuleImport::getConstantsToConvert(llvm::Constant *constant) { |
1505 | // Return the empty set if the constant has been translated before. |
1506 | if (valueMapping.contains(Val: constant)) |
1507 | return {}; |
1508 | |
1509 | // Traverse the constants in post-order and stop the traversal if a constant |
1510 | // already has a `valueMapping` from an earlier constant translation or if the |
1511 | // constant is traversed a second time. |
1512 | SetVector<llvm::Constant *> orderedSet; |
1513 | SetVector<llvm::Constant *> workList; |
1514 | DenseMap<llvm::Constant *, SmallVector<llvm::Constant *>> adjacencyLists; |
1515 | workList.insert(X: constant); |
1516 | while (!workList.empty()) { |
1517 | llvm::Constant *current = workList.back(); |
1518 | // References of global objects are just pointers to the object. Avoid |
1519 | // walking the elements of these here. |
1520 | if (isa<llvm::GlobalObject>(Val: current) || isa<llvm::GlobalAlias>(Val: current)) { |
1521 | orderedSet.insert(X: current); |
1522 | workList.pop_back(); |
1523 | continue; |
1524 | } |
1525 | |
1526 | // Collect all dependencies of the current constant and add them to the |
1527 | // adjacency list if none has been computed before. |
1528 | auto [adjacencyIt, inserted] = adjacencyLists.try_emplace(Key: current); |
1529 | if (inserted) { |
1530 | // Add all constant operands to the adjacency list and skip any other |
1531 | // values such as basic block addresses. |
1532 | for (llvm::Value *operand : current->operands()) |
1533 | if (auto *constDependency = dyn_cast<llvm::Constant>(Val: operand)) |
1534 | adjacencyIt->getSecond().push_back(Elt: constDependency); |
1535 | // Use the getElementValue method to add the dependencies of zero |
1536 | // initialized aggregate constants since they do not take any operands. |
1537 | if (auto *constAgg = dyn_cast<llvm::ConstantAggregateZero>(Val: current)) { |
1538 | unsigned numElements = constAgg->getElementCount().getFixedValue(); |
1539 | for (unsigned i = 0, e = numElements; i != e; ++i) |
1540 | adjacencyIt->getSecond().push_back(Elt: constAgg->getElementValue(Idx: i)); |
1541 | } |
1542 | } |
1543 | // Add the current constant to the `orderedSet` of the traversed nodes if |
1544 | // all its dependencies have been traversed before. Additionally, remove the |
1545 | // constant from the `workList` and continue the traversal. |
1546 | if (adjacencyIt->getSecond().empty()) { |
1547 | orderedSet.insert(X: current); |
1548 | workList.pop_back(); |
1549 | continue; |
1550 | } |
1551 | // Add the next dependency from the adjacency list to the `workList` and |
1552 | // continue the traversal. Remove the dependency from the adjacency list to |
1553 | // mark that it has been processed. Only enqueue the dependency if it has no |
1554 | // `valueMapping` from an earlier translation and if it has not been |
1555 | // enqueued before. |
1556 | llvm::Constant *dependency = adjacencyIt->getSecond().pop_back_val(); |
1557 | if (valueMapping.contains(Val: dependency) || workList.contains(key: dependency) || |
1558 | orderedSet.contains(key: dependency)) |
1559 | continue; |
1560 | workList.insert(X: dependency); |
1561 | } |
1562 | |
1563 | return orderedSet; |
1564 | } |
1565 | |
1566 | FailureOr<Value> ModuleImport::convertConstant(llvm::Constant *constant) { |
1567 | Location loc = UnknownLoc::get(context); |
1568 | |
1569 | // Convert constants that can be represented as attributes. |
1570 | if (Attribute attr = getConstantAsAttr(constant)) { |
1571 | Type type = convertType(type: constant->getType()); |
1572 | if (auto symbolRef = dyn_cast<FlatSymbolRefAttr>(attr)) { |
1573 | return builder.create<AddressOfOp>(loc, type, symbolRef.getValue()) |
1574 | .getResult(); |
1575 | } |
1576 | return builder.create<ConstantOp>(loc, type, attr).getResult(); |
1577 | } |
1578 | |
1579 | // Convert null pointer constants. |
1580 | if (auto *nullPtr = dyn_cast<llvm::ConstantPointerNull>(Val: constant)) { |
1581 | Type type = convertType(type: nullPtr->getType()); |
1582 | return builder.create<ZeroOp>(loc, type).getResult(); |
1583 | } |
1584 | |
1585 | // Convert none token constants. |
1586 | if (isa<llvm::ConstantTokenNone>(Val: constant)) { |
1587 | return builder.create<NoneTokenOp>(loc).getResult(); |
1588 | } |
1589 | |
1590 | // Convert poison. |
1591 | if (auto *poisonVal = dyn_cast<llvm::PoisonValue>(Val: constant)) { |
1592 | Type type = convertType(type: poisonVal->getType()); |
1593 | return builder.create<PoisonOp>(loc, type).getResult(); |
1594 | } |
1595 | |
1596 | // Convert undef. |
1597 | if (auto *undefVal = dyn_cast<llvm::UndefValue>(Val: constant)) { |
1598 | Type type = convertType(type: undefVal->getType()); |
1599 | return builder.create<UndefOp>(loc, type).getResult(); |
1600 | } |
1601 | |
1602 | // Convert dso_local_equivalent. |
1603 | if (auto *dsoLocalEquivalent = dyn_cast<llvm::DSOLocalEquivalent>(Val: constant)) { |
1604 | Type type = convertType(type: dsoLocalEquivalent->getType()); |
1605 | return builder |
1606 | .create<DSOLocalEquivalentOp>( |
1607 | loc, type, |
1608 | FlatSymbolRefAttr::get( |
1609 | builder.getContext(), |
1610 | dsoLocalEquivalent->getGlobalValue()->getName())) |
1611 | .getResult(); |
1612 | } |
1613 | |
1614 | // Convert global variable accesses. |
1615 | if (auto *globalObj = dyn_cast<llvm::GlobalObject>(Val: constant)) { |
1616 | Type type = convertType(type: globalObj->getType()); |
1617 | StringRef globalName = globalObj->getName(); |
1618 | FlatSymbolRefAttr symbolRef; |
1619 | // Empty names are only allowed for global variables. |
1620 | if (globalName.empty()) |
1621 | symbolRef = |
1622 | getOrCreateNamelessSymbolName(globalVar: cast<llvm::GlobalVariable>(Val: globalObj)); |
1623 | else |
1624 | symbolRef = FlatSymbolRefAttr::get(ctx: context, value: globalName); |
1625 | return builder.create<AddressOfOp>(loc, type, symbolRef).getResult(); |
1626 | } |
1627 | |
1628 | // Convert global alias accesses. |
1629 | if (auto *globalAliasObj = dyn_cast<llvm::GlobalAlias>(Val: constant)) { |
1630 | Type type = convertType(type: globalAliasObj->getType()); |
1631 | StringRef aliaseeName = globalAliasObj->getName(); |
1632 | FlatSymbolRefAttr symbolRef = FlatSymbolRefAttr::get(ctx: context, value: aliaseeName); |
1633 | return builder.create<AddressOfOp>(loc, type, symbolRef).getResult(); |
1634 | } |
1635 | |
1636 | // Convert constant expressions. |
1637 | if (auto *constExpr = dyn_cast<llvm::ConstantExpr>(Val: constant)) { |
1638 | // Convert the constant expression to a temporary LLVM instruction and |
1639 | // translate it using the `processInstruction` method. Delete the |
1640 | // instruction after the translation and remove it from `valueMapping`, |
1641 | // since later calls to `getAsInstruction` may return the same address |
1642 | // resulting in a conflicting `valueMapping` entry. |
1643 | llvm::Instruction *inst = constExpr->getAsInstruction(); |
1644 | auto guard = llvm::make_scope_exit(F: [&]() { |
1645 | assert(!noResultOpMapping.contains(inst) && |
1646 | "expected constant expression to return a result"); |
1647 | valueMapping.erase(Val: inst); |
1648 | inst->deleteValue(); |
1649 | }); |
1650 | // Note: `processInstruction` does not call `convertConstant` recursively |
1651 | // since all constant dependencies have been converted before. |
1652 | assert(llvm::all_of(inst->operands(), [&](llvm::Value *value) { |
1653 | return valueMapping.contains(value); |
1654 | })); |
1655 | if (failed(Result: processInstruction(inst))) |
1656 | return failure(); |
1657 | return lookupValue(value: inst); |
1658 | } |
1659 | |
1660 | // Convert aggregate constants. |
1661 | if (isa<llvm::ConstantAggregate>(Val: constant) || |
1662 | isa<llvm::ConstantAggregateZero>(Val: constant)) { |
1663 | // Lookup the aggregate elements that have been converted before. |
1664 | SmallVector<Value> elementValues; |
1665 | if (auto *constAgg = dyn_cast<llvm::ConstantAggregate>(Val: constant)) { |
1666 | elementValues.reserve(N: constAgg->getNumOperands()); |
1667 | for (llvm::Value *operand : constAgg->operands()) |
1668 | elementValues.push_back(Elt: lookupValue(value: operand)); |
1669 | } |
1670 | if (auto *constAgg = dyn_cast<llvm::ConstantAggregateZero>(Val: constant)) { |
1671 | unsigned numElements = constAgg->getElementCount().getFixedValue(); |
1672 | elementValues.reserve(N: numElements); |
1673 | for (unsigned i = 0, e = numElements; i != e; ++i) |
1674 | elementValues.push_back(Elt: lookupValue(value: constAgg->getElementValue(Idx: i))); |
1675 | } |
1676 | assert(llvm::count(elementValues, nullptr) == 0 && |
1677 | "expected all elements have been converted before"); |
1678 | |
1679 | // Generate an UndefOp as root value and insert the aggregate elements. |
1680 | Type rootType = convertType(type: constant->getType()); |
1681 | bool isArrayOrStruct = isa<LLVMArrayType, LLVMStructType>(rootType); |
1682 | assert((isArrayOrStruct || LLVM::isCompatibleVectorType(rootType)) && |
1683 | "unrecognized aggregate type"); |
1684 | Value root = builder.create<UndefOp>(loc, rootType); |
1685 | for (const auto &it : llvm::enumerate(First&: elementValues)) { |
1686 | if (isArrayOrStruct) { |
1687 | root = builder.create<InsertValueOp>(loc, root, it.value(), it.index()); |
1688 | } else { |
1689 | Attribute indexAttr = builder.getI32IntegerAttr(it.index()); |
1690 | Value indexValue = |
1691 | builder.create<ConstantOp>(loc, builder.getI32Type(), indexAttr); |
1692 | root = builder.create<InsertElementOp>(loc, rootType, root, it.value(), |
1693 | indexValue); |
1694 | } |
1695 | } |
1696 | return root; |
1697 | } |
1698 | |
1699 | if (auto *constTargetNone = dyn_cast<llvm::ConstantTargetNone>(Val: constant)) { |
1700 | LLVMTargetExtType targetExtType = |
1701 | cast<LLVMTargetExtType>(convertType(constTargetNone->getType())); |
1702 | assert(targetExtType.hasProperty(LLVMTargetExtType::HasZeroInit) && |
1703 | "target extension type does not support zero-initialization"); |
1704 | // Create llvm.mlir.zero operation to represent zero-initialization of |
1705 | // target extension type. |
1706 | return builder.create<LLVM::ZeroOp>(loc, targetExtType).getRes(); |
1707 | } |
1708 | |
1709 | if (auto *blockAddr = dyn_cast<llvm::BlockAddress>(Val: constant)) { |
1710 | auto fnSym = |
1711 | FlatSymbolRefAttr::get(ctx: context, value: blockAddr->getFunction()->getName()); |
1712 | auto blockTag = |
1713 | BlockTagAttr::get(context, blockAddr->getBasicBlock()->getNumber()); |
1714 | return builder |
1715 | .create<BlockAddressOp>(loc, convertType(blockAddr->getType()), |
1716 | BlockAddressAttr::get(context, fnSym, blockTag)) |
1717 | .getRes(); |
1718 | } |
1719 | |
1720 | StringRef error = ""; |
1721 | |
1722 | if (isa<llvm::ConstantPtrAuth>(Val: constant)) |
1723 | error = " since ptrauth(...) is unsupported"; |
1724 | |
1725 | if (isa<llvm::NoCFIValue>(Val: constant)) |
1726 | error = " since no_cfi is unsupported"; |
1727 | |
1728 | if (isa<llvm::GlobalValue>(Val: constant)) |
1729 | error = " since global value is unsupported"; |
1730 | |
1731 | return emitError(loc) << "unhandled constant: "<< diag(value: *constant) << error; |
1732 | } |
1733 | |
1734 | FailureOr<Value> ModuleImport::convertConstantExpr(llvm::Constant *constant) { |
1735 | // Only call the function for constants that have not been translated before |
1736 | // since it updates the constant insertion point assuming the converted |
1737 | // constant has been introduced at the end of the constant section. |
1738 | assert(!valueMapping.contains(constant) && |
1739 | "expected constant has not been converted before"); |
1740 | assert(constantInsertionBlock && |
1741 | "expected the constant insertion block to be non-null"); |
1742 | |
1743 | // Insert the constant after the last one or at the start of the entry block. |
1744 | OpBuilder::InsertionGuard guard(builder); |
1745 | if (!constantInsertionOp) |
1746 | builder.setInsertionPointToStart(constantInsertionBlock); |
1747 | else |
1748 | builder.setInsertionPointAfter(constantInsertionOp); |
1749 | |
1750 | // Convert all constants of the expression and add them to `valueMapping`. |
1751 | SetVector<llvm::Constant *> constantsToConvert = |
1752 | getConstantsToConvert(constant); |
1753 | for (llvm::Constant *constantToConvert : constantsToConvert) { |
1754 | FailureOr<Value> converted = convertConstant(constant: constantToConvert); |
1755 | if (failed(Result: converted)) |
1756 | return failure(); |
1757 | mapValue(llvm: constantToConvert, mlir: *converted); |
1758 | } |
1759 | |
1760 | // Update the constant insertion point and return the converted constant. |
1761 | Value result = lookupValue(value: constant); |
1762 | constantInsertionOp = result.getDefiningOp(); |
1763 | return result; |
1764 | } |
1765 | |
1766 | FailureOr<Value> ModuleImport::convertValue(llvm::Value *value) { |
1767 | assert(!isa<llvm::MetadataAsValue>(value) && |
1768 | "expected value to not be metadata"); |
1769 | |
1770 | // Return the mapped value if it has been converted before. |
1771 | auto it = valueMapping.find(Val: value); |
1772 | if (it != valueMapping.end()) |
1773 | return it->getSecond(); |
1774 | |
1775 | // Convert constants such as immediate values that have no mapping yet. |
1776 | if (auto *constant = dyn_cast<llvm::Constant>(Val: value)) |
1777 | return convertConstantExpr(constant); |
1778 | |
1779 | Location loc = UnknownLoc::get(context); |
1780 | if (auto *inst = dyn_cast<llvm::Instruction>(Val: value)) |
1781 | loc = translateLoc(loc: inst->getDebugLoc()); |
1782 | return emitError(loc) << "unhandled value: "<< diag(value: *value); |
1783 | } |
1784 | |
1785 | FailureOr<Value> ModuleImport::convertMetadataValue(llvm::Value *value) { |
1786 | // A value may be wrapped as metadata, for example, when passed to a debug |
1787 | // intrinsic. Unwrap these values before the conversion. |
1788 | auto *nodeAsVal = dyn_cast<llvm::MetadataAsValue>(Val: value); |
1789 | if (!nodeAsVal) |
1790 | return failure(); |
1791 | auto *node = dyn_cast<llvm::ValueAsMetadata>(Val: nodeAsVal->getMetadata()); |
1792 | if (!node) |
1793 | return failure(); |
1794 | value = node->getValue(); |
1795 | |
1796 | // Return the mapped value if it has been converted before. |
1797 | auto it = valueMapping.find(Val: value); |
1798 | if (it != valueMapping.end()) |
1799 | return it->getSecond(); |
1800 | |
1801 | // Convert constants such as immediate values that have no mapping yet. |
1802 | if (auto *constant = dyn_cast<llvm::Constant>(Val: value)) |
1803 | return convertConstantExpr(constant); |
1804 | return failure(); |
1805 | } |
1806 | |
1807 | FailureOr<SmallVector<Value>> |
1808 | ModuleImport::convertValues(ArrayRef<llvm::Value *> values) { |
1809 | SmallVector<Value> remapped; |
1810 | remapped.reserve(N: values.size()); |
1811 | for (llvm::Value *value : values) { |
1812 | FailureOr<Value> converted = convertValue(value); |
1813 | if (failed(Result: converted)) |
1814 | return failure(); |
1815 | remapped.push_back(Elt: *converted); |
1816 | } |
1817 | return remapped; |
1818 | } |
1819 | |
1820 | LogicalResult ModuleImport::convertIntrinsicArguments( |
1821 | ArrayRef<llvm::Value *> values, ArrayRef<llvm::OperandBundleUse> opBundles, |
1822 | bool requiresOpBundles, ArrayRef<unsigned> immArgPositions, |
1823 | ArrayRef<StringLiteral> immArgAttrNames, SmallVectorImpl<Value> &valuesOut, |
1824 | SmallVectorImpl<NamedAttribute> &attrsOut) { |
1825 | assert(immArgPositions.size() == immArgAttrNames.size() && |
1826 | "LLVM `immArgPositions` and MLIR `immArgAttrNames` should have equal " |
1827 | "length"); |
1828 | |
1829 | SmallVector<llvm::Value *> operands(values); |
1830 | for (auto [immArgPos, immArgName] : |
1831 | llvm::zip(t&: immArgPositions, u&: immArgAttrNames)) { |
1832 | auto &value = operands[immArgPos]; |
1833 | auto *constant = llvm::cast<llvm::Constant>(Val: value); |
1834 | auto attr = getScalarConstantAsAttr(builder, constant); |
1835 | assert(attr && attr.getType().isIntOrFloat() && |
1836 | "expected immarg to be float or integer constant"); |
1837 | auto nameAttr = StringAttr::get(attr.getContext(), immArgName); |
1838 | attrsOut.push_back(Elt: {nameAttr, attr}); |
1839 | // Mark matched attribute values as null (so they can be removed below). |
1840 | value = nullptr; |
1841 | } |
1842 | |
1843 | for (llvm::Value *value : operands) { |
1844 | if (!value) |
1845 | continue; |
1846 | auto mlirValue = convertValue(value); |
1847 | if (failed(Result: mlirValue)) |
1848 | return failure(); |
1849 | valuesOut.push_back(Elt: *mlirValue); |
1850 | } |
1851 | |
1852 | SmallVector<int> opBundleSizes; |
1853 | SmallVector<Attribute> opBundleTagAttrs; |
1854 | if (requiresOpBundles) { |
1855 | opBundleSizes.reserve(N: opBundles.size()); |
1856 | opBundleTagAttrs.reserve(N: opBundles.size()); |
1857 | |
1858 | for (const llvm::OperandBundleUse &bundle : opBundles) { |
1859 | opBundleSizes.push_back(Elt: bundle.Inputs.size()); |
1860 | opBundleTagAttrs.push_back(StringAttr::get(context, bundle.getTagName())); |
1861 | |
1862 | for (const llvm::Use &opBundleOperand : bundle.Inputs) { |
1863 | auto operandMlirValue = convertValue(value: opBundleOperand.get()); |
1864 | if (failed(Result: operandMlirValue)) |
1865 | return failure(); |
1866 | valuesOut.push_back(Elt: *operandMlirValue); |
1867 | } |
1868 | } |
1869 | |
1870 | auto opBundleSizesAttr = DenseI32ArrayAttr::get(context, opBundleSizes); |
1871 | auto opBundleSizesAttrNameAttr = |
1872 | StringAttr::get(context, LLVMDialect::getOpBundleSizesAttrName()); |
1873 | attrsOut.push_back(Elt: {opBundleSizesAttrNameAttr, opBundleSizesAttr}); |
1874 | |
1875 | auto opBundleTagsAttr = ArrayAttr::get(context, opBundleTagAttrs); |
1876 | auto opBundleTagsAttrNameAttr = |
1877 | StringAttr::get(context, LLVMDialect::getOpBundleTagsAttrName()); |
1878 | attrsOut.push_back(Elt: {opBundleTagsAttrNameAttr, opBundleTagsAttr}); |
1879 | } |
1880 | |
1881 | return success(); |
1882 | } |
1883 | |
1884 | IntegerAttr ModuleImport::matchIntegerAttr(llvm::Value *value) { |
1885 | IntegerAttr integerAttr; |
1886 | FailureOr<Value> converted = convertValue(value); |
1887 | bool success = succeeded(Result: converted) && |
1888 | matchPattern(*converted, m_Constant(&integerAttr)); |
1889 | assert(success && "expected a constant integer value"); |
1890 | (void)success; |
1891 | return integerAttr; |
1892 | } |
1893 | |
1894 | FloatAttr ModuleImport::matchFloatAttr(llvm::Value *value) { |
1895 | FloatAttr floatAttr; |
1896 | FailureOr<Value> converted = convertValue(value); |
1897 | bool success = |
1898 | succeeded(Result: converted) && matchPattern(*converted, m_Constant(&floatAttr)); |
1899 | assert(success && "expected a constant float value"); |
1900 | (void)success; |
1901 | return floatAttr; |
1902 | } |
1903 | |
1904 | DILocalVariableAttr ModuleImport::matchLocalVariableAttr(llvm::Value *value) { |
1905 | auto *nodeAsVal = cast<llvm::MetadataAsValue>(Val: value); |
1906 | auto *node = cast<llvm::DILocalVariable>(Val: nodeAsVal->getMetadata()); |
1907 | return debugImporter->translate(node); |
1908 | } |
1909 | |
1910 | DILabelAttr ModuleImport::matchLabelAttr(llvm::Value *value) { |
1911 | auto *nodeAsVal = cast<llvm::MetadataAsValue>(Val: value); |
1912 | auto *node = cast<llvm::DILabel>(Val: nodeAsVal->getMetadata()); |
1913 | return debugImporter->translate(node); |
1914 | } |
1915 | |
1916 | FPExceptionBehaviorAttr |
1917 | ModuleImport::matchFPExceptionBehaviorAttr(llvm::Value *value) { |
1918 | auto *metadata = cast<llvm::MetadataAsValue>(Val: value); |
1919 | auto *mdstr = cast<llvm::MDString>(Val: metadata->getMetadata()); |
1920 | std::optional<llvm::fp::ExceptionBehavior> optLLVM = |
1921 | llvm::convertStrToExceptionBehavior(mdstr->getString()); |
1922 | assert(optLLVM && "Expecting FP exception behavior"); |
1923 | return builder.getAttr<FPExceptionBehaviorAttr>( |
1924 | convertFPExceptionBehaviorFromLLVM(*optLLVM)); |
1925 | } |
1926 | |
1927 | RoundingModeAttr ModuleImport::matchRoundingModeAttr(llvm::Value *value) { |
1928 | auto *metadata = cast<llvm::MetadataAsValue>(Val: value); |
1929 | auto *mdstr = cast<llvm::MDString>(Val: metadata->getMetadata()); |
1930 | std::optional<llvm::RoundingMode> optLLVM = |
1931 | llvm::convertStrToRoundingMode(mdstr->getString()); |
1932 | assert(optLLVM && "Expecting rounding mode"); |
1933 | return builder.getAttr<RoundingModeAttr>( |
1934 | convertRoundingModeFromLLVM(*optLLVM)); |
1935 | } |
1936 | |
1937 | FailureOr<SmallVector<AliasScopeAttr>> |
1938 | ModuleImport::matchAliasScopeAttrs(llvm::Value *value) { |
1939 | auto *nodeAsVal = cast<llvm::MetadataAsValue>(Val: value); |
1940 | auto *node = cast<llvm::MDNode>(Val: nodeAsVal->getMetadata()); |
1941 | return lookupAliasScopeAttrs(node); |
1942 | } |
1943 | |
1944 | Location ModuleImport::translateLoc(llvm::DILocation *loc) { |
1945 | return debugImporter->translateLoc(loc); |
1946 | } |
1947 | |
1948 | LogicalResult |
1949 | ModuleImport::convertBranchArgs(llvm::Instruction *branch, |
1950 | llvm::BasicBlock *target, |
1951 | SmallVectorImpl<Value> &blockArguments) { |
1952 | for (auto inst = target->begin(); isa<llvm::PHINode>(Val: inst); ++inst) { |
1953 | auto *phiInst = cast<llvm::PHINode>(Val: &*inst); |
1954 | llvm::Value *value = phiInst->getIncomingValueForBlock(BB: branch->getParent()); |
1955 | FailureOr<Value> converted = convertValue(value); |
1956 | if (failed(Result: converted)) |
1957 | return failure(); |
1958 | blockArguments.push_back(Elt: *converted); |
1959 | } |
1960 | return success(); |
1961 | } |
1962 | |
1963 | FailureOr<SmallVector<Value>> |
1964 | ModuleImport::convertCallOperands(llvm::CallBase *callInst, |
1965 | bool allowInlineAsm) { |
1966 | bool isInlineAsm = callInst->isInlineAsm(); |
1967 | if (isInlineAsm && !allowInlineAsm) |
1968 | return failure(); |
1969 | |
1970 | SmallVector<Value> operands; |
1971 | |
1972 | // Cannot use isIndirectCall() here because we need to handle Constant callees |
1973 | // that are not considered indirect calls by LLVM. However, in MLIR, they are |
1974 | // treated as indirect calls to constant operands that need to be converted. |
1975 | // Skip the callee operand if it's inline assembly, as it's handled separately |
1976 | // in InlineAsmOp. |
1977 | if (!isa<llvm::Function>(Val: callInst->getCalledOperand()) && !isInlineAsm) { |
1978 | FailureOr<Value> called = convertValue(value: callInst->getCalledOperand()); |
1979 | if (failed(Result: called)) |
1980 | return failure(); |
1981 | operands.push_back(Elt: *called); |
1982 | } |
1983 | |
1984 | SmallVector<llvm::Value *> args(callInst->args()); |
1985 | FailureOr<SmallVector<Value>> arguments = convertValues(values: args); |
1986 | if (failed(Result: arguments)) |
1987 | return failure(); |
1988 | |
1989 | llvm::append_range(C&: operands, R&: *arguments); |
1990 | return operands; |
1991 | } |
1992 | |
1993 | /// Checks if `callType` and `calleeType` are compatible and can be represented |
1994 | /// in MLIR. |
1995 | static LogicalResult |
1996 | checkFunctionTypeCompatibility(LLVMFunctionType callType, |
1997 | LLVMFunctionType calleeType) { |
1998 | if (callType.getReturnType() != calleeType.getReturnType()) |
1999 | return failure(); |
2000 | |
2001 | if (calleeType.isVarArg()) { |
2002 | // For variadic functions, the call can have more types than the callee |
2003 | // specifies. |
2004 | if (callType.getNumParams() < calleeType.getNumParams()) |
2005 | return failure(); |
2006 | } else { |
2007 | // For non-variadic functions, the number of parameters needs to be the |
2008 | // same. |
2009 | if (callType.getNumParams() != calleeType.getNumParams()) |
2010 | return failure(); |
2011 | } |
2012 | |
2013 | // Check that all operands match. |
2014 | for (auto [operandType, argumentType] : |
2015 | llvm::zip(callType.getParams(), calleeType.getParams())) |
2016 | if (operandType != argumentType) |
2017 | return failure(); |
2018 | |
2019 | return success(); |
2020 | } |
2021 | |
2022 | FailureOr<LLVMFunctionType> |
2023 | ModuleImport::convertFunctionType(llvm::CallBase *callInst, |
2024 | bool &isIncompatibleCall) { |
2025 | isIncompatibleCall = false; |
2026 | auto castOrFailure = [](Type convertedType) -> FailureOr<LLVMFunctionType> { |
2027 | auto funcTy = dyn_cast_or_null<LLVMFunctionType>(convertedType); |
2028 | if (!funcTy) |
2029 | return failure(); |
2030 | return funcTy; |
2031 | }; |
2032 | |
2033 | llvm::Value *calledOperand = callInst->getCalledOperand(); |
2034 | FailureOr<LLVMFunctionType> callType = |
2035 | castOrFailure(convertType(callInst->getFunctionType())); |
2036 | if (failed(callType)) |
2037 | return failure(); |
2038 | auto *callee = dyn_cast<llvm::Function>(Val: calledOperand); |
2039 | // For indirect calls, return the type of the call itself. |
2040 | if (!callee) |
2041 | return callType; |
2042 | |
2043 | FailureOr<LLVMFunctionType> calleeType = |
2044 | castOrFailure(convertType(callee->getFunctionType())); |
2045 | if (failed(calleeType)) |
2046 | return failure(); |
2047 | |
2048 | // Compare the types and notify users via `isIncompatibleCall` if they are not |
2049 | // compatible. |
2050 | if (failed(checkFunctionTypeCompatibility(*callType, *calleeType))) { |
2051 | isIncompatibleCall = true; |
2052 | Location loc = translateLoc(loc: callInst->getDebugLoc()); |
2053 | emitWarning(loc) << "incompatible call and callee types: "<< *callType |
2054 | << " and "<< *calleeType; |
2055 | return callType; |
2056 | } |
2057 | |
2058 | return calleeType; |
2059 | } |
2060 | |
2061 | FlatSymbolRefAttr ModuleImport::convertCalleeName(llvm::CallBase *callInst) { |
2062 | llvm::Value *calledOperand = callInst->getCalledOperand(); |
2063 | if (auto *callee = dyn_cast<llvm::Function>(calledOperand)) |
2064 | return SymbolRefAttr::get(context, callee->getName()); |
2065 | return {}; |
2066 | } |
2067 | |
2068 | LogicalResult ModuleImport::convertIntrinsic(llvm::CallInst *inst) { |
2069 | if (succeeded(Result: iface.convertIntrinsic(builder, inst, moduleImport&: *this))) |
2070 | return success(); |
2071 | |
2072 | Location loc = translateLoc(loc: inst->getDebugLoc()); |
2073 | return emitError(loc) << "unhandled intrinsic: "<< diag(value: *inst); |
2074 | } |
2075 | |
2076 | ArrayAttr |
2077 | ModuleImport::convertAsmInlineOperandAttrs(const llvm::CallBase &llvmCall) { |
2078 | const auto *ia = cast<llvm::InlineAsm>(Val: llvmCall.getCalledOperand()); |
2079 | unsigned argIdx = 0; |
2080 | SmallVector<mlir::Attribute> opAttrs; |
2081 | bool hasIndirect = false; |
2082 | |
2083 | for (const llvm::InlineAsm::ConstraintInfo &ci : ia->ParseConstraints()) { |
2084 | // Only deal with constraints that correspond to call arguments. |
2085 | if (ci.Type == llvm::InlineAsm::isLabel || !ci.hasArg()) |
2086 | continue; |
2087 | |
2088 | // Only increment `argIdx` in terms of constraints containing arguments, |
2089 | // which are guaranteed to happen in the same order of the call arguments. |
2090 | if (ci.isIndirect) { |
2091 | if (llvm::Type *paramEltType = llvmCall.getParamElementType(ArgNo: argIdx)) { |
2092 | SmallVector<mlir::NamedAttribute> attrs; |
2093 | attrs.push_back(builder.getNamedAttr( |
2094 | mlir::LLVM::InlineAsmOp::getElementTypeAttrName(), |
2095 | mlir::TypeAttr::get(convertType(paramEltType)))); |
2096 | opAttrs.push_back(builder.getDictionaryAttr(attrs)); |
2097 | hasIndirect = true; |
2098 | } |
2099 | } else { |
2100 | opAttrs.push_back(builder.getDictionaryAttr({})); |
2101 | } |
2102 | argIdx++; |
2103 | } |
2104 | |
2105 | // Avoid emitting an array where all entries are empty dictionaries. |
2106 | return hasIndirect ? ArrayAttr::get(mlirModule->getContext(), opAttrs) |
2107 | : nullptr; |
2108 | } |
2109 | |
2110 | LogicalResult ModuleImport::convertInstruction(llvm::Instruction *inst) { |
2111 | // Convert all instructions that do not provide an MLIR builder. |
2112 | Location loc = translateLoc(loc: inst->getDebugLoc()); |
2113 | if (inst->getOpcode() == llvm::Instruction::Br) { |
2114 | auto *brInst = cast<llvm::BranchInst>(Val: inst); |
2115 | |
2116 | SmallVector<Block *> succBlocks; |
2117 | SmallVector<SmallVector<Value>> succBlockArgs; |
2118 | for (auto i : llvm::seq<unsigned>(Begin: 0, End: brInst->getNumSuccessors())) { |
2119 | llvm::BasicBlock *succ = brInst->getSuccessor(i); |
2120 | SmallVector<Value> blockArgs; |
2121 | if (failed(Result: convertBranchArgs(branch: brInst, target: succ, blockArguments&: blockArgs))) |
2122 | return failure(); |
2123 | succBlocks.push_back(Elt: lookupBlock(block: succ)); |
2124 | succBlockArgs.push_back(Elt: blockArgs); |
2125 | } |
2126 | |
2127 | if (!brInst->isConditional()) { |
2128 | auto brOp = builder.create<LLVM::BrOp>(loc, succBlockArgs.front(), |
2129 | succBlocks.front()); |
2130 | mapNoResultOp(inst, brOp); |
2131 | return success(); |
2132 | } |
2133 | FailureOr<Value> condition = convertValue(value: brInst->getCondition()); |
2134 | if (failed(Result: condition)) |
2135 | return failure(); |
2136 | auto condBrOp = builder.create<LLVM::CondBrOp>( |
2137 | loc, *condition, succBlocks.front(), succBlockArgs.front(), |
2138 | succBlocks.back(), succBlockArgs.back()); |
2139 | mapNoResultOp(inst, condBrOp); |
2140 | return success(); |
2141 | } |
2142 | if (inst->getOpcode() == llvm::Instruction::Switch) { |
2143 | auto *swInst = cast<llvm::SwitchInst>(Val: inst); |
2144 | // Process the condition value. |
2145 | FailureOr<Value> condition = convertValue(value: swInst->getCondition()); |
2146 | if (failed(Result: condition)) |
2147 | return failure(); |
2148 | SmallVector<Value> defaultBlockArgs; |
2149 | // Process the default case. |
2150 | llvm::BasicBlock *defaultBB = swInst->getDefaultDest(); |
2151 | if (failed(Result: convertBranchArgs(branch: swInst, target: defaultBB, blockArguments&: defaultBlockArgs))) |
2152 | return failure(); |
2153 | |
2154 | // Process the cases. |
2155 | unsigned numCases = swInst->getNumCases(); |
2156 | SmallVector<SmallVector<Value>> caseOperands(numCases); |
2157 | SmallVector<ValueRange> caseOperandRefs(numCases); |
2158 | SmallVector<APInt> caseValues(numCases); |
2159 | SmallVector<Block *> caseBlocks(numCases); |
2160 | for (const auto &it : llvm::enumerate(First: swInst->cases())) { |
2161 | const llvm::SwitchInst::CaseHandle &caseHandle = it.value(); |
2162 | llvm::BasicBlock *succBB = caseHandle.getCaseSuccessor(); |
2163 | if (failed(Result: convertBranchArgs(branch: swInst, target: succBB, blockArguments&: caseOperands[it.index()]))) |
2164 | return failure(); |
2165 | caseOperandRefs[it.index()] = caseOperands[it.index()]; |
2166 | caseValues[it.index()] = caseHandle.getCaseValue()->getValue(); |
2167 | caseBlocks[it.index()] = lookupBlock(block: succBB); |
2168 | } |
2169 | |
2170 | auto switchOp = builder.create<SwitchOp>( |
2171 | loc, *condition, lookupBlock(defaultBB), defaultBlockArgs, caseValues, |
2172 | caseBlocks, caseOperandRefs); |
2173 | mapNoResultOp(inst, switchOp); |
2174 | return success(); |
2175 | } |
2176 | if (inst->getOpcode() == llvm::Instruction::PHI) { |
2177 | Type type = convertType(type: inst->getType()); |
2178 | mapValue(llvm: inst, mlir: builder.getInsertionBlock()->addArgument( |
2179 | type, loc: translateLoc(loc: inst->getDebugLoc()))); |
2180 | return success(); |
2181 | } |
2182 | if (inst->getOpcode() == llvm::Instruction::Call) { |
2183 | auto *callInst = cast<llvm::CallInst>(Val: inst); |
2184 | llvm::Value *calledOperand = callInst->getCalledOperand(); |
2185 | |
2186 | FailureOr<SmallVector<Value>> operands = |
2187 | convertCallOperands(callInst, /*allowInlineAsm=*/true); |
2188 | if (failed(Result: operands)) |
2189 | return failure(); |
2190 | |
2191 | auto callOp = [&]() -> FailureOr<Operation *> { |
2192 | if (auto *asmI = dyn_cast<llvm::InlineAsm>(Val: calledOperand)) { |
2193 | Type resultTy = convertType(type: callInst->getType()); |
2194 | if (!resultTy) |
2195 | return failure(); |
2196 | ArrayAttr operandAttrs = convertAsmInlineOperandAttrs(*callInst); |
2197 | return builder |
2198 | .create<InlineAsmOp>( |
2199 | loc, resultTy, *operands, |
2200 | builder.getStringAttr(asmI->getAsmString()), |
2201 | builder.getStringAttr(asmI->getConstraintString()), |
2202 | asmI->hasSideEffects(), asmI->isAlignStack(), |
2203 | convertTailCallKindFromLLVM(callInst->getTailCallKind()), |
2204 | AsmDialectAttr::get( |
2205 | mlirModule.getContext(), |
2206 | convertAsmDialectFromLLVM(asmI->getDialect())), |
2207 | operandAttrs) |
2208 | .getOperation(); |
2209 | } |
2210 | bool isIncompatibleCall; |
2211 | FailureOr<LLVMFunctionType> funcTy = |
2212 | convertFunctionType(callInst, isIncompatibleCall); |
2213 | if (failed(funcTy)) |
2214 | return failure(); |
2215 | |
2216 | FlatSymbolRefAttr callee = nullptr; |
2217 | if (isIncompatibleCall) { |
2218 | // Use an indirect call (in order to represent valid and verifiable LLVM |
2219 | // IR). Build the indirect call by passing an empty `callee` operand and |
2220 | // insert into `operands` to include the indirect call target. |
2221 | FlatSymbolRefAttr calleeSym = convertCalleeName(callInst); |
2222 | Value indirectCallVal = builder.create<LLVM::AddressOfOp>( |
2223 | loc, LLVM::LLVMPointerType::get(context), calleeSym); |
2224 | operands->insert(I: operands->begin(), Elt: indirectCallVal); |
2225 | } else { |
2226 | // Regular direct call using callee name. |
2227 | callee = convertCalleeName(callInst); |
2228 | } |
2229 | CallOp callOp = builder.create<CallOp>(loc, *funcTy, callee, *operands); |
2230 | |
2231 | if (failed(convertCallAttributes(callInst, callOp))) |
2232 | return failure(); |
2233 | |
2234 | // Handle parameter and result attributes unless it's an incompatible |
2235 | // call. |
2236 | if (!isIncompatibleCall) |
2237 | convertParameterAttributes(callInst, callOp, builder); |
2238 | return callOp.getOperation(); |
2239 | }(); |
2240 | |
2241 | if (failed(Result: callOp)) |
2242 | return failure(); |
2243 | |
2244 | if (!callInst->getType()->isVoidTy()) |
2245 | mapValue(llvm: inst, mlir: (*callOp)->getResult(idx: 0)); |
2246 | else |
2247 | mapNoResultOp(llvm: inst, mlir: *callOp); |
2248 | return success(); |
2249 | } |
2250 | if (inst->getOpcode() == llvm::Instruction::LandingPad) { |
2251 | auto *lpInst = cast<llvm::LandingPadInst>(Val: inst); |
2252 | |
2253 | SmallVector<Value> operands; |
2254 | operands.reserve(N: lpInst->getNumClauses()); |
2255 | for (auto i : llvm::seq<unsigned>(Begin: 0, End: lpInst->getNumClauses())) { |
2256 | FailureOr<Value> operand = convertValue(value: lpInst->getClause(Idx: i)); |
2257 | if (failed(Result: operand)) |
2258 | return failure(); |
2259 | operands.push_back(Elt: *operand); |
2260 | } |
2261 | |
2262 | Type type = convertType(type: lpInst->getType()); |
2263 | auto lpOp = |
2264 | builder.create<LandingpadOp>(loc, type, lpInst->isCleanup(), operands); |
2265 | mapValue(inst, lpOp); |
2266 | return success(); |
2267 | } |
2268 | if (inst->getOpcode() == llvm::Instruction::Invoke) { |
2269 | auto *invokeInst = cast<llvm::InvokeInst>(Val: inst); |
2270 | |
2271 | if (invokeInst->isInlineAsm()) |
2272 | return emitError(loc) << "invoke of inline assembly is not supported"; |
2273 | |
2274 | FailureOr<SmallVector<Value>> operands = convertCallOperands(callInst: invokeInst); |
2275 | if (failed(Result: operands)) |
2276 | return failure(); |
2277 | |
2278 | // Check whether the invoke result is an argument to the normal destination |
2279 | // block. |
2280 | bool invokeResultUsedInPhi = llvm::any_of( |
2281 | Range: invokeInst->getNormalDest()->phis(), P: [&](const llvm::PHINode &phi) { |
2282 | return phi.getIncomingValueForBlock(BB: invokeInst->getParent()) == |
2283 | invokeInst; |
2284 | }); |
2285 | |
2286 | Block *normalDest = lookupBlock(block: invokeInst->getNormalDest()); |
2287 | Block *directNormalDest = normalDest; |
2288 | if (invokeResultUsedInPhi) { |
2289 | // The invoke result cannot be an argument to the normal destination |
2290 | // block, as that would imply using the invoke operation result in its |
2291 | // definition, so we need to create a dummy block to serve as an |
2292 | // intermediate destination. |
2293 | OpBuilder::InsertionGuard g(builder); |
2294 | directNormalDest = builder.createBlock(insertBefore: normalDest); |
2295 | } |
2296 | |
2297 | SmallVector<Value> unwindArgs; |
2298 | if (failed(Result: convertBranchArgs(branch: invokeInst, target: invokeInst->getUnwindDest(), |
2299 | blockArguments&: unwindArgs))) |
2300 | return failure(); |
2301 | |
2302 | bool isIncompatibleInvoke; |
2303 | FailureOr<LLVMFunctionType> funcTy = |
2304 | convertFunctionType(invokeInst, isIncompatibleInvoke); |
2305 | if (failed(funcTy)) |
2306 | return failure(); |
2307 | |
2308 | FlatSymbolRefAttr calleeName = nullptr; |
2309 | if (isIncompatibleInvoke) { |
2310 | // Use an indirect invoke (in order to represent valid and verifiable LLVM |
2311 | // IR). Build the indirect invoke by passing an empty `callee` operand and |
2312 | // insert into `operands` to include the indirect invoke target. |
2313 | FlatSymbolRefAttr calleeSym = convertCalleeName(callInst: invokeInst); |
2314 | Value indirectInvokeVal = builder.create<LLVM::AddressOfOp>( |
2315 | loc, LLVM::LLVMPointerType::get(context), calleeSym); |
2316 | operands->insert(I: operands->begin(), Elt: indirectInvokeVal); |
2317 | } else { |
2318 | // Regular direct invoke using callee name. |
2319 | calleeName = convertCalleeName(callInst: invokeInst); |
2320 | } |
2321 | // Create the invoke operation. Normal destination block arguments will be |
2322 | // added later on to handle the case in which the operation result is |
2323 | // included in this list. |
2324 | auto invokeOp = builder.create<InvokeOp>( |
2325 | loc, *funcTy, calleeName, *operands, directNormalDest, ValueRange(), |
2326 | lookupBlock(invokeInst->getUnwindDest()), unwindArgs); |
2327 | |
2328 | if (failed(convertInvokeAttributes(invokeInst, invokeOp))) |
2329 | return failure(); |
2330 | |
2331 | // Handle parameter and result attributes unless it's an incompatible |
2332 | // invoke. |
2333 | if (!isIncompatibleInvoke) |
2334 | convertParameterAttributes(invokeInst, invokeOp, builder); |
2335 | |
2336 | if (!invokeInst->getType()->isVoidTy()) |
2337 | mapValue(inst, invokeOp.getResults().front()); |
2338 | else |
2339 | mapNoResultOp(inst, invokeOp); |
2340 | |
2341 | SmallVector<Value> normalArgs; |
2342 | if (failed(Result: convertBranchArgs(branch: invokeInst, target: invokeInst->getNormalDest(), |
2343 | blockArguments&: normalArgs))) |
2344 | return failure(); |
2345 | |
2346 | if (invokeResultUsedInPhi) { |
2347 | // The dummy normal dest block will just host an unconditional branch |
2348 | // instruction to the normal destination block passing the required block |
2349 | // arguments (including the invoke operation's result). |
2350 | OpBuilder::InsertionGuard g(builder); |
2351 | builder.setInsertionPointToStart(directNormalDest); |
2352 | builder.create<LLVM::BrOp>(loc, normalArgs, normalDest); |
2353 | } else { |
2354 | // If the invoke operation's result is not a block argument to the normal |
2355 | // destination block, just add the block arguments as usual. |
2356 | assert(llvm::none_of( |
2357 | normalArgs, |
2358 | [&](Value val) { return val.getDefiningOp() == invokeOp; }) && |
2359 | "An llvm.invoke operation cannot pass its result as a block " |
2360 | "argument."); |
2361 | invokeOp.getNormalDestOperandsMutable().append(normalArgs); |
2362 | } |
2363 | |
2364 | return success(); |
2365 | } |
2366 | if (inst->getOpcode() == llvm::Instruction::GetElementPtr) { |
2367 | auto *gepInst = cast<llvm::GetElementPtrInst>(Val: inst); |
2368 | Type sourceElementType = convertType(type: gepInst->getSourceElementType()); |
2369 | FailureOr<Value> basePtr = convertValue(value: gepInst->getOperand(i_nocapture: 0)); |
2370 | if (failed(Result: basePtr)) |
2371 | return failure(); |
2372 | |
2373 | // Treat every indices as dynamic since GEPOp::build will refine those |
2374 | // indices into static attributes later. One small downside of this |
2375 | // approach is that many unused `llvm.mlir.constant` would be emitted |
2376 | // at first place. |
2377 | SmallVector<GEPArg> indices; |
2378 | for (llvm::Value *operand : llvm::drop_begin(RangeOrContainer: gepInst->operand_values())) { |
2379 | FailureOr<Value> index = convertValue(value: operand); |
2380 | if (failed(Result: index)) |
2381 | return failure(); |
2382 | indices.push_back(Elt: *index); |
2383 | } |
2384 | |
2385 | Type type = convertType(type: inst->getType()); |
2386 | auto gepOp = builder.create<GEPOp>( |
2387 | loc, type, sourceElementType, *basePtr, indices, |
2388 | static_cast<GEPNoWrapFlags>(gepInst->getNoWrapFlags().getRaw())); |
2389 | mapValue(inst, gepOp); |
2390 | return success(); |
2391 | } |
2392 | |
2393 | if (inst->getOpcode() == llvm::Instruction::IndirectBr) { |
2394 | auto *indBrInst = cast<llvm::IndirectBrInst>(Val: inst); |
2395 | |
2396 | FailureOr<Value> basePtr = convertValue(value: indBrInst->getAddress()); |
2397 | if (failed(Result: basePtr)) |
2398 | return failure(); |
2399 | |
2400 | SmallVector<Block *> succBlocks; |
2401 | SmallVector<SmallVector<Value>> succBlockArgs; |
2402 | for (auto i : llvm::seq<unsigned>(Begin: 0, End: indBrInst->getNumSuccessors())) { |
2403 | llvm::BasicBlock *succ = indBrInst->getSuccessor(i); |
2404 | SmallVector<Value> blockArgs; |
2405 | if (failed(Result: convertBranchArgs(branch: indBrInst, target: succ, blockArguments&: blockArgs))) |
2406 | return failure(); |
2407 | succBlocks.push_back(Elt: lookupBlock(block: succ)); |
2408 | succBlockArgs.push_back(Elt: blockArgs); |
2409 | } |
2410 | SmallVector<ValueRange> succBlockArgsRange = |
2411 | llvm::to_vector_of<ValueRange>(Range&: succBlockArgs); |
2412 | Location loc = translateLoc(loc: inst->getDebugLoc()); |
2413 | auto indBrOp = builder.create<LLVM::IndirectBrOp>( |
2414 | loc, *basePtr, succBlockArgsRange, succBlocks); |
2415 | |
2416 | mapNoResultOp(inst, indBrOp); |
2417 | return success(); |
2418 | } |
2419 | |
2420 | // Convert all instructions that have an mlirBuilder. |
2421 | if (succeeded(Result: convertInstructionImpl(odsBuilder&: builder, inst, moduleImport&: *this, iface))) |
2422 | return success(); |
2423 | |
2424 | return emitError(loc) << "unhandled instruction: "<< diag(value: *inst); |
2425 | } |
2426 | |
2427 | LogicalResult ModuleImport::processInstruction(llvm::Instruction *inst) { |
2428 | // FIXME: Support uses of SubtargetData. |
2429 | // FIXME: Add support for call / operand attributes. |
2430 | // FIXME: Add support for the cleanupret, catchret, catchswitch, callbr, |
2431 | // vaarg, catchpad, cleanuppad instructions. |
2432 | |
2433 | // Convert LLVM intrinsics calls to MLIR intrinsics. |
2434 | if (auto *intrinsic = dyn_cast<llvm::IntrinsicInst>(Val: inst)) |
2435 | return convertIntrinsic(inst: intrinsic); |
2436 | |
2437 | // Convert all remaining LLVM instructions to MLIR operations. |
2438 | return convertInstruction(inst); |
2439 | } |
2440 | |
2441 | FlatSymbolRefAttr ModuleImport::getPersonalityAsAttr(llvm::Function *f) { |
2442 | if (!f->hasPersonalityFn()) |
2443 | return nullptr; |
2444 | |
2445 | llvm::Constant *pf = f->getPersonalityFn(); |
2446 | |
2447 | // If it directly has a name, we can use it. |
2448 | if (pf->hasName()) |
2449 | return SymbolRefAttr::get(builder.getContext(), pf->getName()); |
2450 | |
2451 | // If it doesn't have a name, currently, only function pointers that are |
2452 | // bitcast to i8* are parsed. |
2453 | if (auto *ce = dyn_cast<llvm::ConstantExpr>(Val: pf)) { |
2454 | if (ce->getOpcode() == llvm::Instruction::BitCast && |
2455 | ce->getType() == llvm::PointerType::getUnqual(C&: f->getContext())) { |
2456 | if (auto *func = dyn_cast<llvm::Function>(ce->getOperand(0))) |
2457 | return SymbolRefAttr::get(builder.getContext(), func->getName()); |
2458 | } |
2459 | } |
2460 | return FlatSymbolRefAttr(); |
2461 | } |
2462 | |
2463 | static void processMemoryEffects(llvm::Function *func, LLVMFuncOp funcOp) { |
2464 | llvm::MemoryEffects memEffects = func->getMemoryEffects(); |
2465 | |
2466 | auto othermem = convertModRefInfoFromLLVM( |
2467 | memEffects.getModRef(Loc: llvm::MemoryEffects::Location::Other)); |
2468 | auto argMem = convertModRefInfoFromLLVM( |
2469 | memEffects.getModRef(Loc: llvm::MemoryEffects::Location::ArgMem)); |
2470 | auto inaccessibleMem = convertModRefInfoFromLLVM( |
2471 | memEffects.getModRef(Loc: llvm::MemoryEffects::Location::InaccessibleMem)); |
2472 | auto memAttr = MemoryEffectsAttr::get(funcOp.getContext(), othermem, argMem, |
2473 | inaccessibleMem); |
2474 | // Only set the attr when it does not match the default value. |
2475 | if (memAttr.isReadWrite()) |
2476 | return; |
2477 | funcOp.setMemoryEffectsAttr(memAttr); |
2478 | } |
2479 | |
2480 | // List of LLVM IR attributes that map to an explicit attribute on the MLIR |
2481 | // LLVMFuncOp. |
2482 | static constexpr std::array kExplicitAttributes{ |
2483 | StringLiteral("aarch64_in_za"), |
2484 | StringLiteral("aarch64_inout_za"), |
2485 | StringLiteral("aarch64_new_za"), |
2486 | StringLiteral("aarch64_out_za"), |
2487 | StringLiteral("aarch64_preserves_za"), |
2488 | StringLiteral("aarch64_pstate_sm_body"), |
2489 | StringLiteral("aarch64_pstate_sm_compatible"), |
2490 | StringLiteral("aarch64_pstate_sm_enabled"), |
2491 | StringLiteral("alwaysinline"), |
2492 | StringLiteral("approx-func-fp-math"), |
2493 | StringLiteral("convergent"), |
2494 | StringLiteral("denormal-fp-math"), |
2495 | StringLiteral("denormal-fp-math-f32"), |
2496 | StringLiteral("fp-contract"), |
2497 | StringLiteral("frame-pointer"), |
2498 | StringLiteral("instrument-function-entry"), |
2499 | StringLiteral("instrument-function-exit"), |
2500 | StringLiteral("no-infs-fp-math"), |
2501 | StringLiteral("no-nans-fp-math"), |
2502 | StringLiteral("no-signed-zeros-fp-math"), |
2503 | StringLiteral("noinline"), |
2504 | StringLiteral("nounwind"), |
2505 | StringLiteral("optnone"), |
2506 | StringLiteral("target-features"), |
2507 | StringLiteral("tune-cpu"), |
2508 | StringLiteral("unsafe-fp-math"), |
2509 | StringLiteral("uwtable"), |
2510 | StringLiteral("vscale_range"), |
2511 | StringLiteral("willreturn"), |
2512 | }; |
2513 | |
2514 | static void processPassthroughAttrs(llvm::Function *func, LLVMFuncOp funcOp) { |
2515 | MLIRContext *context = funcOp.getContext(); |
2516 | SmallVector<Attribute> passthroughs; |
2517 | llvm::AttributeSet funcAttrs = func->getAttributes().getAttributes( |
2518 | Index: llvm::AttributeList::AttrIndex::FunctionIndex); |
2519 | for (llvm::Attribute attr : funcAttrs) { |
2520 | // Skip the memory attribute since the LLVMFuncOp has an explicit memory |
2521 | // attribute. |
2522 | if (attr.hasAttribute(llvm::Attribute::Memory)) |
2523 | continue; |
2524 | |
2525 | // Skip invalid type attributes. |
2526 | if (attr.isTypeAttribute()) { |
2527 | emitWarning(funcOp.getLoc(), |
2528 | "type attributes on a function are invalid, skipping it"); |
2529 | continue; |
2530 | } |
2531 | |
2532 | StringRef attrName; |
2533 | if (attr.isStringAttribute()) |
2534 | attrName = attr.getKindAsString(); |
2535 | else |
2536 | attrName = llvm::Attribute::getNameFromAttrKind(AttrKind: attr.getKindAsEnum()); |
2537 | auto keyAttr = StringAttr::get(context, attrName); |
2538 | |
2539 | // Skip attributes that map to an explicit attribute on the LLVMFuncOp. |
2540 | if (llvm::is_contained(Range: kExplicitAttributes, Element: attrName)) |
2541 | continue; |
2542 | |
2543 | if (attr.isStringAttribute()) { |
2544 | StringRef val = attr.getValueAsString(); |
2545 | if (val.empty()) { |
2546 | passthroughs.push_back(Elt: keyAttr); |
2547 | continue; |
2548 | } |
2549 | passthroughs.push_back( |
2550 | ArrayAttr::get(context, {keyAttr, StringAttr::get(context, val)})); |
2551 | continue; |
2552 | } |
2553 | if (attr.isIntAttribute()) { |
2554 | auto val = std::to_string(val: attr.getValueAsInt()); |
2555 | passthroughs.push_back( |
2556 | ArrayAttr::get(context, {keyAttr, StringAttr::get(context, val)})); |
2557 | continue; |
2558 | } |
2559 | if (attr.isEnumAttribute()) { |
2560 | passthroughs.push_back(Elt: keyAttr); |
2561 | continue; |
2562 | } |
2563 | |
2564 | llvm_unreachable("unexpected attribute kind"); |
2565 | } |
2566 | |
2567 | if (!passthroughs.empty()) |
2568 | funcOp.setPassthroughAttr(ArrayAttr::get(context, passthroughs)); |
2569 | } |
2570 | |
2571 | void ModuleImport::processFunctionAttributes(llvm::Function *func, |
2572 | LLVMFuncOp funcOp) { |
2573 | processMemoryEffects(func, funcOp); |
2574 | processPassthroughAttrs(func, funcOp); |
2575 | |
2576 | if (func->hasFnAttribute(llvm::Attribute::NoInline)) |
2577 | funcOp.setNoInline(true); |
2578 | if (func->hasFnAttribute(llvm::Attribute::AlwaysInline)) |
2579 | funcOp.setAlwaysInline(true); |
2580 | if (func->hasFnAttribute(llvm::Attribute::OptimizeNone)) |
2581 | funcOp.setOptimizeNone(true); |
2582 | if (func->hasFnAttribute(llvm::Attribute::Convergent)) |
2583 | funcOp.setConvergent(true); |
2584 | if (func->hasFnAttribute(llvm::Attribute::NoUnwind)) |
2585 | funcOp.setNoUnwind(true); |
2586 | if (func->hasFnAttribute(llvm::Attribute::WillReturn)) |
2587 | funcOp.setWillReturn(true); |
2588 | |
2589 | if (func->hasFnAttribute(Kind: "aarch64_pstate_sm_enabled")) |
2590 | funcOp.setArmStreaming(true); |
2591 | else if (func->hasFnAttribute(Kind: "aarch64_pstate_sm_body")) |
2592 | funcOp.setArmLocallyStreaming(true); |
2593 | else if (func->hasFnAttribute(Kind: "aarch64_pstate_sm_compatible")) |
2594 | funcOp.setArmStreamingCompatible(true); |
2595 | |
2596 | if (func->hasFnAttribute(Kind: "aarch64_new_za")) |
2597 | funcOp.setArmNewZa(true); |
2598 | else if (func->hasFnAttribute(Kind: "aarch64_in_za")) |
2599 | funcOp.setArmInZa(true); |
2600 | else if (func->hasFnAttribute(Kind: "aarch64_out_za")) |
2601 | funcOp.setArmOutZa(true); |
2602 | else if (func->hasFnAttribute(Kind: "aarch64_inout_za")) |
2603 | funcOp.setArmInoutZa(true); |
2604 | else if (func->hasFnAttribute(Kind: "aarch64_preserves_za")) |
2605 | funcOp.setArmPreservesZa(true); |
2606 | |
2607 | llvm::Attribute attr = func->getFnAttribute(llvm::Attribute::VScaleRange); |
2608 | if (attr.isValid()) { |
2609 | MLIRContext *context = funcOp.getContext(); |
2610 | auto intTy = IntegerType::get(context, 32); |
2611 | funcOp.setVscaleRangeAttr(LLVM::VScaleRangeAttr::get( |
2612 | context, IntegerAttr::get(intTy, attr.getVScaleRangeMin()), |
2613 | IntegerAttr::get(intTy, attr.getVScaleRangeMax().value_or(0)))); |
2614 | } |
2615 | |
2616 | // Process frame-pointer attribute. |
2617 | if (func->hasFnAttribute(Kind: "frame-pointer")) { |
2618 | StringRef stringRefFramePointerKind = |
2619 | func->getFnAttribute(Kind: "frame-pointer").getValueAsString(); |
2620 | funcOp.setFramePointerAttr(LLVM::FramePointerKindAttr::get( |
2621 | funcOp.getContext(), LLVM::framePointerKind::symbolizeFramePointerKind( |
2622 | stringRefFramePointerKind) |
2623 | .value())); |
2624 | } |
2625 | |
2626 | if (llvm::Attribute attr = func->getFnAttribute("target-cpu"); |
2627 | attr.isStringAttribute()) |
2628 | funcOp.setTargetCpuAttr(StringAttr::get(context, attr.getValueAsString())); |
2629 | |
2630 | if (llvm::Attribute attr = func->getFnAttribute("tune-cpu"); |
2631 | attr.isStringAttribute()) |
2632 | funcOp.setTuneCpuAttr(StringAttr::get(context, attr.getValueAsString())); |
2633 | |
2634 | if (llvm::Attribute attr = func->getFnAttribute("target-features"); |
2635 | attr.isStringAttribute()) |
2636 | funcOp.setTargetFeaturesAttr( |
2637 | LLVM::TargetFeaturesAttr::get(context, attr.getValueAsString())); |
2638 | |
2639 | if (llvm::Attribute attr = func->getFnAttribute("reciprocal-estimates"); |
2640 | attr.isStringAttribute()) |
2641 | funcOp.setReciprocalEstimatesAttr( |
2642 | StringAttr::get(context, attr.getValueAsString())); |
2643 | |
2644 | if (llvm::Attribute attr = func->getFnAttribute(Kind: "prefer-vector-width"); |
2645 | attr.isStringAttribute()) |
2646 | funcOp.setPreferVectorWidth(attr.getValueAsString()); |
2647 | |
2648 | if (llvm::Attribute attr = func->getFnAttribute(Kind: "unsafe-fp-math"); |
2649 | attr.isStringAttribute()) |
2650 | funcOp.setUnsafeFpMath(attr.getValueAsBool()); |
2651 | |
2652 | if (llvm::Attribute attr = func->getFnAttribute(Kind: "no-infs-fp-math"); |
2653 | attr.isStringAttribute()) |
2654 | funcOp.setNoInfsFpMath(attr.getValueAsBool()); |
2655 | |
2656 | if (llvm::Attribute attr = func->getFnAttribute(Kind: "no-nans-fp-math"); |
2657 | attr.isStringAttribute()) |
2658 | funcOp.setNoNansFpMath(attr.getValueAsBool()); |
2659 | |
2660 | if (llvm::Attribute attr = func->getFnAttribute(Kind: "approx-func-fp-math"); |
2661 | attr.isStringAttribute()) |
2662 | funcOp.setApproxFuncFpMath(attr.getValueAsBool()); |
2663 | |
2664 | if (llvm::Attribute attr = func->getFnAttribute("instrument-function-entry"); |
2665 | attr.isStringAttribute()) |
2666 | funcOp.setInstrumentFunctionEntry( |
2667 | StringAttr::get(context, attr.getValueAsString())); |
2668 | |
2669 | if (llvm::Attribute attr = func->getFnAttribute("instrument-function-exit"); |
2670 | attr.isStringAttribute()) |
2671 | funcOp.setInstrumentFunctionExit( |
2672 | StringAttr::get(context, attr.getValueAsString())); |
2673 | |
2674 | if (llvm::Attribute attr = func->getFnAttribute(Kind: "no-signed-zeros-fp-math"); |
2675 | attr.isStringAttribute()) |
2676 | funcOp.setNoSignedZerosFpMath(attr.getValueAsBool()); |
2677 | |
2678 | if (llvm::Attribute attr = func->getFnAttribute("denormal-fp-math"); |
2679 | attr.isStringAttribute()) |
2680 | funcOp.setDenormalFpMathAttr( |
2681 | StringAttr::get(context, attr.getValueAsString())); |
2682 | |
2683 | if (llvm::Attribute attr = func->getFnAttribute("denormal-fp-math-f32"); |
2684 | attr.isStringAttribute()) |
2685 | funcOp.setDenormalFpMathF32Attr( |
2686 | StringAttr::get(context, attr.getValueAsString())); |
2687 | |
2688 | if (llvm::Attribute attr = func->getFnAttribute("fp-contract"); |
2689 | attr.isStringAttribute()) |
2690 | funcOp.setFpContractAttr(StringAttr::get(context, attr.getValueAsString())); |
2691 | |
2692 | if (func->hasUWTable()) { |
2693 | ::llvm::UWTableKind uwtableKind = func->getUWTableKind(); |
2694 | funcOp.setUwtableKindAttr(LLVM::UWTableKindAttr::get( |
2695 | funcOp.getContext(), convertUWTableKindFromLLVM(uwtableKind))); |
2696 | } |
2697 | } |
2698 | |
2699 | DictionaryAttr |
2700 | ModuleImport::convertParameterAttribute(llvm::AttributeSet llvmParamAttrs, |
2701 | OpBuilder &builder) { |
2702 | SmallVector<NamedAttribute> paramAttrs; |
2703 | for (auto [llvmKind, mlirName] : getAttrKindToNameMapping()) { |
2704 | auto llvmAttr = llvmParamAttrs.getAttribute(Kind: llvmKind); |
2705 | // Skip attributes that are not attached. |
2706 | if (!llvmAttr.isValid()) |
2707 | continue; |
2708 | |
2709 | // TODO: Import captures(none) as a nocapture unit attribute until the |
2710 | // LLVM dialect switches to the captures representation. |
2711 | if (llvmAttr.hasKindAsEnum() && |
2712 | llvmAttr.getKindAsEnum() == llvm::Attribute::Captures) { |
2713 | if (llvm::capturesNothing(CC: llvmAttr.getCaptureInfo())) |
2714 | paramAttrs.push_back( |
2715 | Elt: builder.getNamedAttr(mlirName, builder.getUnitAttr())); |
2716 | continue; |
2717 | } |
2718 | |
2719 | Attribute mlirAttr; |
2720 | if (llvmAttr.isTypeAttribute()) |
2721 | mlirAttr = TypeAttr::get(convertType(llvmAttr.getValueAsType())); |
2722 | else if (llvmAttr.isIntAttribute()) |
2723 | mlirAttr = builder.getI64IntegerAttr(llvmAttr.getValueAsInt()); |
2724 | else if (llvmAttr.isEnumAttribute()) |
2725 | mlirAttr = builder.getUnitAttr(); |
2726 | else if (llvmAttr.isConstantRangeAttribute()) { |
2727 | const llvm::ConstantRange &value = llvmAttr.getValueAsConstantRange(); |
2728 | mlirAttr = builder.getAttr<LLVM::ConstantRangeAttr>(value.getLower(), |
2729 | value.getUpper()); |
2730 | } else { |
2731 | llvm_unreachable("unexpected parameter attribute kind"); |
2732 | } |
2733 | paramAttrs.push_back(Elt: builder.getNamedAttr(name: mlirName, val: mlirAttr)); |
2734 | } |
2735 | |
2736 | return builder.getDictionaryAttr(paramAttrs); |
2737 | } |
2738 | |
2739 | void ModuleImport::convertParameterAttributes(llvm::Function *func, |
2740 | LLVMFuncOp funcOp, |
2741 | OpBuilder &builder) { |
2742 | auto llvmAttrs = func->getAttributes(); |
2743 | for (size_t i = 0, e = funcOp.getNumArguments(); i < e; ++i) { |
2744 | llvm::AttributeSet llvmArgAttrs = llvmAttrs.getParamAttrs(ArgNo: i); |
2745 | funcOp.setArgAttrs(i, convertParameterAttribute(llvmArgAttrs, builder)); |
2746 | } |
2747 | // Convert the result attributes and attach them wrapped in an ArrayAttribute |
2748 | // to the funcOp. |
2749 | llvm::AttributeSet llvmResAttr = llvmAttrs.getRetAttrs(); |
2750 | if (!llvmResAttr.hasAttributes()) |
2751 | return; |
2752 | funcOp.setResAttrsAttr( |
2753 | builder.getArrayAttr(convertParameterAttribute(llvmResAttr, builder))); |
2754 | } |
2755 | |
2756 | void ModuleImport::convertParameterAttributes(llvm::CallBase *call, |
2757 | ArrayAttr &argsAttr, |
2758 | ArrayAttr &resAttr, |
2759 | OpBuilder &builder) { |
2760 | llvm::AttributeList llvmAttrs = call->getAttributes(); |
2761 | SmallVector<llvm::AttributeSet> llvmArgAttrsSet; |
2762 | bool anyArgAttrs = false; |
2763 | for (size_t i = 0, e = call->arg_size(); i < e; ++i) { |
2764 | llvmArgAttrsSet.emplace_back(Args: llvmAttrs.getParamAttrs(ArgNo: i)); |
2765 | if (llvmArgAttrsSet.back().hasAttributes()) |
2766 | anyArgAttrs = true; |
2767 | } |
2768 | auto getArrayAttr = [&](ArrayRef<DictionaryAttr> dictAttrs) { |
2769 | SmallVector<Attribute> attrs; |
2770 | for (auto &dict : dictAttrs) |
2771 | attrs.push_back(dict ? dict : builder.getDictionaryAttr({})); |
2772 | return builder.getArrayAttr(attrs); |
2773 | }; |
2774 | if (anyArgAttrs) { |
2775 | SmallVector<DictionaryAttr> argAttrs; |
2776 | for (auto &llvmArgAttrs : llvmArgAttrsSet) |
2777 | argAttrs.emplace_back(convertParameterAttribute(llvmArgAttrs, builder)); |
2778 | argsAttr = getArrayAttr(argAttrs); |
2779 | } |
2780 | |
2781 | llvm::AttributeSet llvmResAttr = llvmAttrs.getRetAttrs(); |
2782 | if (!llvmResAttr.hasAttributes()) |
2783 | return; |
2784 | DictionaryAttr resAttrs = convertParameterAttribute(llvmResAttr, builder); |
2785 | resAttr = getArrayAttr({resAttrs}); |
2786 | } |
2787 | |
2788 | void ModuleImport::convertParameterAttributes(llvm::CallBase *call, |
2789 | CallOpInterface callOp, |
2790 | OpBuilder &builder) { |
2791 | ArrayAttr argsAttr, resAttr; |
2792 | convertParameterAttributes(call, argsAttr, resAttr, builder); |
2793 | callOp.setArgAttrsAttr(argsAttr); |
2794 | callOp.setResAttrsAttr(resAttr); |
2795 | } |
2796 | |
2797 | template <typename Op> |
2798 | static LogicalResult convertCallBaseAttributes(llvm::CallBase *inst, Op op) { |
2799 | op.setCConv(convertCConvFromLLVM(inst->getCallingConv())); |
2800 | return success(); |
2801 | } |
2802 | |
2803 | LogicalResult ModuleImport::convertInvokeAttributes(llvm::InvokeInst *inst, |
2804 | InvokeOp op) { |
2805 | return convertCallBaseAttributes(inst, op); |
2806 | } |
2807 | |
2808 | LogicalResult ModuleImport::convertCallAttributes(llvm::CallInst *inst, |
2809 | CallOp op) { |
2810 | setFastmathFlagsAttr(inst, op: op.getOperation()); |
2811 | // Query the attributes directly instead of using `inst->getFnAttr(Kind)`, the |
2812 | // latter does additional lookup to the parent and inherits, changing the |
2813 | // semantics too early. |
2814 | llvm::AttributeList callAttrs = inst->getAttributes(); |
2815 | |
2816 | op.setTailCallKind(convertTailCallKindFromLLVM(inst->getTailCallKind())); |
2817 | op.setConvergent(callAttrs.getFnAttr(llvm::Attribute::Convergent).isValid()); |
2818 | op.setNoUnwind(callAttrs.getFnAttr(llvm::Attribute::NoUnwind).isValid()); |
2819 | op.setWillReturn(callAttrs.getFnAttr(llvm::Attribute::WillReturn).isValid()); |
2820 | op.setNoInline(callAttrs.getFnAttr(llvm::Attribute::NoInline).isValid()); |
2821 | op.setAlwaysInline( |
2822 | callAttrs.getFnAttr(llvm::Attribute::AlwaysInline).isValid()); |
2823 | op.setInlineHint(callAttrs.getFnAttr(llvm::Attribute::InlineHint).isValid()); |
2824 | |
2825 | llvm::MemoryEffects memEffects = inst->getMemoryEffects(); |
2826 | ModRefInfo othermem = convertModRefInfoFromLLVM( |
2827 | memEffects.getModRef(llvm::MemoryEffects::Location::Other)); |
2828 | ModRefInfo argMem = convertModRefInfoFromLLVM( |
2829 | memEffects.getModRef(llvm::MemoryEffects::Location::ArgMem)); |
2830 | ModRefInfo inaccessibleMem = convertModRefInfoFromLLVM( |
2831 | memEffects.getModRef(llvm::MemoryEffects::Location::InaccessibleMem)); |
2832 | auto memAttr = MemoryEffectsAttr::get(op.getContext(), othermem, argMem, |
2833 | inaccessibleMem); |
2834 | // Only set the attribute when it does not match the default value. |
2835 | if (!memAttr.isReadWrite()) |
2836 | op.setMemoryEffectsAttr(memAttr); |
2837 | |
2838 | return convertCallBaseAttributes(inst, op); |
2839 | } |
2840 | |
2841 | LogicalResult ModuleImport::processFunction(llvm::Function *func) { |
2842 | clearRegionState(); |
2843 | |
2844 | auto functionType = |
2845 | dyn_cast<LLVMFunctionType>(convertType(func->getFunctionType())); |
2846 | if (func->isIntrinsic() && |
2847 | iface.isConvertibleIntrinsic(id: func->getIntrinsicID())) |
2848 | return success(); |
2849 | |
2850 | bool dsoLocal = func->isDSOLocal(); |
2851 | CConv cconv = convertCConvFromLLVM(func->getCallingConv()); |
2852 | |
2853 | // Insert the function at the end of the module. |
2854 | OpBuilder::InsertionGuard guard(builder); |
2855 | builder.setInsertionPointToEnd(mlirModule.getBody()); |
2856 | |
2857 | Location loc = debugImporter->translateFuncLocation(func); |
2858 | LLVMFuncOp funcOp = builder.create<LLVMFuncOp>( |
2859 | loc, func->getName(), functionType, |
2860 | convertLinkageFromLLVM(func->getLinkage()), dsoLocal, cconv); |
2861 | |
2862 | convertParameterAttributes(func, funcOp, builder); |
2863 | |
2864 | if (FlatSymbolRefAttr personality = getPersonalityAsAttr(func)) |
2865 | funcOp.setPersonalityAttr(personality); |
2866 | else if (func->hasPersonalityFn()) |
2867 | emitWarning(funcOp.getLoc(), "could not deduce personality, skipping it"); |
2868 | |
2869 | if (func->hasGC()) |
2870 | funcOp.setGarbageCollector(StringRef(func->getGC())); |
2871 | |
2872 | if (func->hasAtLeastLocalUnnamedAddr()) |
2873 | funcOp.setUnnamedAddr(convertUnnamedAddrFromLLVM(func->getUnnamedAddr())); |
2874 | |
2875 | if (func->hasSection()) |
2876 | funcOp.setSection(StringRef(func->getSection())); |
2877 | |
2878 | funcOp.setVisibility_(convertVisibilityFromLLVM(func->getVisibility())); |
2879 | |
2880 | if (func->hasComdat()) |
2881 | funcOp.setComdatAttr(comdatMapping.lookup(func->getComdat())); |
2882 | |
2883 | if (llvm::MaybeAlign maybeAlign = func->getAlign()) |
2884 | funcOp.setAlignment(maybeAlign->value()); |
2885 | |
2886 | // Handle Function attributes. |
2887 | processFunctionAttributes(func, funcOp); |
2888 | |
2889 | // Convert non-debug metadata by using the dialect interface. |
2890 | SmallVector<std::pair<unsigned, llvm::MDNode *>> allMetadata; |
2891 | func->getAllMetadata(MDs&: allMetadata); |
2892 | for (auto &[kind, node] : allMetadata) { |
2893 | if (!iface.isConvertibleMetadata(kind)) |
2894 | continue; |
2895 | if (failed(iface.setMetadataAttrs(builder, kind, node, op: funcOp, moduleImport&: *this))) { |
2896 | emitWarning(funcOp.getLoc()) |
2897 | << "unhandled function metadata: "<< diagMD(node, module: llvmModule.get()) |
2898 | << " on "<< diag(value: *func); |
2899 | } |
2900 | } |
2901 | |
2902 | if (func->isDeclaration()) |
2903 | return success(); |
2904 | |
2905 | // Collect the set of basic blocks reachable from the function's entry block. |
2906 | // This step is crucial as LLVM IR can contain unreachable blocks that |
2907 | // self-dominate. As a result, an operation might utilize a variable it |
2908 | // defines, which the import does not support. Given that MLIR lacks block |
2909 | // label support, we can safely remove unreachable blocks, as there are no |
2910 | // indirect branch instructions that could potentially target these blocks. |
2911 | llvm::df_iterator_default_set<llvm::BasicBlock *> reachable; |
2912 | for (llvm::BasicBlock *basicBlock : llvm::depth_first_ext(G: func, S&: reachable)) |
2913 | (void)basicBlock; |
2914 | |
2915 | // Eagerly create all reachable blocks. |
2916 | SmallVector<llvm::BasicBlock *> reachableBasicBlocks; |
2917 | for (llvm::BasicBlock &basicBlock : *func) { |
2918 | // Skip unreachable blocks. |
2919 | if (!reachable.contains(Ptr: &basicBlock)) { |
2920 | if (basicBlock.hasAddressTaken()) |
2921 | return emitError(funcOp.getLoc()) |
2922 | << "unreachable block '"<< basicBlock.getName() |
2923 | << "' with address taken"; |
2924 | continue; |
2925 | } |
2926 | Region &body = funcOp.getBody(); |
2927 | Block *block = builder.createBlock(parent: &body, insertPt: body.end()); |
2928 | mapBlock(llvm: &basicBlock, mlir: block); |
2929 | reachableBasicBlocks.push_back(&basicBlock); |
2930 | } |
2931 | |
2932 | // Add function arguments to the entry block. |
2933 | for (const auto &it : llvm::enumerate(First: func->args())) { |
2934 | BlockArgument blockArg = funcOp.getFunctionBody().addArgument( |
2935 | functionType.getParamType(it.index()), funcOp.getLoc()); |
2936 | mapValue(llvm: &it.value(), mlir: blockArg); |
2937 | } |
2938 | |
2939 | // Process the blocks in topological order. The ordered traversal ensures |
2940 | // operands defined in a dominating block have a valid mapping to an MLIR |
2941 | // value once a block is translated. |
2942 | SetVector<llvm::BasicBlock *> blocks = |
2943 | getTopologicallySortedBlocks(reachableBasicBlocks); |
2944 | setConstantInsertionPointToStart(lookupBlock(blocks.front())); |
2945 | for (llvm::BasicBlock *basicBlock : blocks) |
2946 | if (failed(processBasicBlock(basicBlock, lookupBlock(basicBlock)))) |
2947 | return failure(); |
2948 | |
2949 | // Process the debug intrinsics that require a delayed conversion after |
2950 | // everything else was converted. |
2951 | if (failed(Result: processDebugIntrinsics())) |
2952 | return failure(); |
2953 | |
2954 | return success(); |
2955 | } |
2956 | |
2957 | /// Checks if `dbgIntr` is a kill location that holds metadata instead of an SSA |
2958 | /// value. |
2959 | static bool isMetadataKillLocation(llvm::DbgVariableIntrinsic *dbgIntr) { |
2960 | if (!dbgIntr->isKillLocation()) |
2961 | return false; |
2962 | llvm::Value *value = dbgIntr->getArgOperand(i: 0); |
2963 | auto *nodeAsVal = dyn_cast<llvm::MetadataAsValue>(Val: value); |
2964 | if (!nodeAsVal) |
2965 | return false; |
2966 | return !isa<llvm::ValueAsMetadata>(Val: nodeAsVal->getMetadata()); |
2967 | } |
2968 | |
2969 | LogicalResult |
2970 | ModuleImport::processDebugIntrinsic(llvm::DbgVariableIntrinsic *dbgIntr, |
2971 | DominanceInfo &domInfo) { |
2972 | Location loc = translateLoc(loc: dbgIntr->getDebugLoc()); |
2973 | auto emitUnsupportedWarning = [&]() { |
2974 | if (emitExpensiveWarnings) |
2975 | emitWarning(loc) << "dropped intrinsic: "<< diag(value: *dbgIntr); |
2976 | return success(); |
2977 | }; |
2978 | // Drop debug intrinsics with arg lists. |
2979 | // TODO: Support debug intrinsics that have arg lists. |
2980 | if (dbgIntr->hasArgList()) |
2981 | return emitUnsupportedWarning(); |
2982 | // Kill locations can have metadata nodes as location operand. This |
2983 | // cannot be converted to poison as the type cannot be reconstructed. |
2984 | // TODO: find a way to support this case. |
2985 | if (isMetadataKillLocation(dbgIntr)) |
2986 | return emitUnsupportedWarning(); |
2987 | // Drop debug intrinsics if the associated variable information cannot be |
2988 | // translated due to cyclic debug metadata. |
2989 | // TODO: Support cyclic debug metadata. |
2990 | DILocalVariableAttr localVariableAttr = |
2991 | matchLocalVariableAttr(dbgIntr->getArgOperand(1)); |
2992 | if (!localVariableAttr) |
2993 | return emitUnsupportedWarning(); |
2994 | FailureOr<Value> argOperand = convertMetadataValue(value: dbgIntr->getArgOperand(i: 0)); |
2995 | if (failed(Result: argOperand)) |
2996 | return emitError(loc) << "failed to convert a debug intrinsic operand: " |
2997 | << diag(value: *dbgIntr); |
2998 | |
2999 | // Ensure that the debug intrinsic is inserted right after its operand is |
3000 | // defined. Otherwise, the operand might not necessarily dominate the |
3001 | // intrinsic. If the defining operation is a terminator, insert the intrinsic |
3002 | // into a dominated block. |
3003 | OpBuilder::InsertionGuard guard(builder); |
3004 | if (Operation *op = argOperand->getDefiningOp(); |
3005 | op && op->hasTrait<OpTrait::IsTerminator>()) { |
3006 | // Find a dominated block that can hold the debug intrinsic. |
3007 | auto dominatedBlocks = domInfo.getNode(a: op->getBlock())->children(); |
3008 | // If no block is dominated by the terminator, this intrinisc cannot be |
3009 | // converted. |
3010 | if (dominatedBlocks.empty()) |
3011 | return emitUnsupportedWarning(); |
3012 | // Set insertion point before the terminator, to avoid inserting something |
3013 | // before landingpads. |
3014 | Block *dominatedBlock = (*dominatedBlocks.begin())->getBlock(); |
3015 | builder.setInsertionPoint(dominatedBlock->getTerminator()); |
3016 | } else { |
3017 | Value insertPt = *argOperand; |
3018 | if (auto blockArg = dyn_cast<BlockArgument>(Val&: *argOperand)) { |
3019 | // The value might be coming from a phi node and is now a block argument, |
3020 | // which means the insertion point is set to the start of the block. If |
3021 | // this block is a target destination of an invoke, the insertion point |
3022 | // must happen after the landing pad operation. |
3023 | Block *insertionBlock = argOperand->getParentBlock(); |
3024 | if (!insertionBlock->empty() && |
3025 | isa<LandingpadOp>(insertionBlock->front())) |
3026 | insertPt = cast<LandingpadOp>(insertionBlock->front()).getRes(); |
3027 | } |
3028 | |
3029 | builder.setInsertionPointAfterValue(insertPt); |
3030 | } |
3031 | auto locationExprAttr = |
3032 | debugImporter->translateExpression(dbgIntr->getExpression()); |
3033 | Operation *op = |
3034 | llvm::TypeSwitch<llvm::DbgVariableIntrinsic *, Operation *>(dbgIntr) |
3035 | .Case(caseFn: [&](llvm::DbgDeclareInst *) { |
3036 | return builder.create<LLVM::DbgDeclareOp>( |
3037 | loc, *argOperand, localVariableAttr, locationExprAttr); |
3038 | }) |
3039 | .Case(caseFn: [&](llvm::DbgValueInst *) { |
3040 | return builder.create<LLVM::DbgValueOp>( |
3041 | loc, *argOperand, localVariableAttr, locationExprAttr); |
3042 | }); |
3043 | mapNoResultOp(llvm: dbgIntr, mlir: op); |
3044 | setNonDebugMetadataAttrs(inst: dbgIntr, op); |
3045 | return success(); |
3046 | } |
3047 | |
3048 | LogicalResult ModuleImport::processDebugIntrinsics() { |
3049 | DominanceInfo domInfo; |
3050 | for (llvm::Instruction *inst : debugIntrinsics) { |
3051 | auto *intrCall = cast<llvm::DbgVariableIntrinsic>(Val: inst); |
3052 | if (failed(Result: processDebugIntrinsic(dbgIntr: intrCall, domInfo))) |
3053 | return failure(); |
3054 | } |
3055 | return success(); |
3056 | } |
3057 | |
3058 | LogicalResult ModuleImport::processBasicBlock(llvm::BasicBlock *bb, |
3059 | Block *block) { |
3060 | builder.setInsertionPointToStart(block); |
3061 | for (llvm::Instruction &inst : *bb) { |
3062 | if (failed(Result: processInstruction(inst: &inst))) |
3063 | return failure(); |
3064 | |
3065 | // Skip additional processing when the instructions is a debug intrinsics |
3066 | // that was not yet converted. |
3067 | if (debugIntrinsics.contains(key: &inst)) |
3068 | continue; |
3069 | |
3070 | // Set the non-debug metadata attributes on the imported operation and emit |
3071 | // a warning if an instruction other than a phi instruction is dropped |
3072 | // during the import. |
3073 | if (Operation *op = lookupOperation(inst: &inst)) { |
3074 | setNonDebugMetadataAttrs(inst: &inst, op); |
3075 | } else if (inst.getOpcode() != llvm::Instruction::PHI) { |
3076 | if (emitExpensiveWarnings) { |
3077 | Location loc = debugImporter->translateLoc(loc: inst.getDebugLoc()); |
3078 | emitWarning(loc) << "dropped instruction: "<< diag(value: inst); |
3079 | } |
3080 | } |
3081 | } |
3082 | |
3083 | if (bb->hasAddressTaken()) { |
3084 | OpBuilder::InsertionGuard guard(builder); |
3085 | builder.setInsertionPointToStart(block); |
3086 | builder.create<BlockTagOp>(block->getParentOp()->getLoc(), |
3087 | BlockTagAttr::get(context, bb->getNumber())); |
3088 | } |
3089 | return success(); |
3090 | } |
3091 | |
3092 | FailureOr<SmallVector<AccessGroupAttr>> |
3093 | ModuleImport::lookupAccessGroupAttrs(const llvm::MDNode *node) const { |
3094 | return loopAnnotationImporter->lookupAccessGroupAttrs(node); |
3095 | } |
3096 | |
3097 | LoopAnnotationAttr |
3098 | ModuleImport::translateLoopAnnotationAttr(const llvm::MDNode *node, |
3099 | Location loc) const { |
3100 | return loopAnnotationImporter->translateLoopAnnotation(node, loc); |
3101 | } |
3102 | |
3103 | FailureOr<DereferenceableAttr> |
3104 | ModuleImport::translateDereferenceableAttr(const llvm::MDNode *node, |
3105 | unsigned kindID) { |
3106 | Location loc = mlirModule.getLoc(); |
3107 | |
3108 | // The only operand should be a constant integer representing the number of |
3109 | // dereferenceable bytes. |
3110 | if (node->getNumOperands() != 1) |
3111 | return emitError(loc) << "dereferenceable metadata must have one operand: " |
3112 | << diagMD(node, module: llvmModule.get()); |
3113 | |
3114 | auto *numBytesMD = dyn_cast<llvm::ConstantAsMetadata>(Val: node->getOperand(I: 0)); |
3115 | auto *numBytesCst = dyn_cast<llvm::ConstantInt>(Val: numBytesMD->getValue()); |
3116 | if (!numBytesCst || !numBytesCst->getValue().isNonNegative()) |
3117 | return emitError(loc) << "dereferenceable metadata operand must be a " |
3118 | "non-negative constant integer: " |
3119 | << diagMD(node, module: llvmModule.get()); |
3120 | |
3121 | bool mayBeNull = kindID == llvm::LLVMContext::MD_dereferenceable_or_null; |
3122 | auto derefAttr = builder.getAttr<DereferenceableAttr>( |
3123 | numBytesCst->getZExtValue(), mayBeNull); |
3124 | |
3125 | return derefAttr; |
3126 | } |
3127 | |
3128 | OwningOpRef<ModuleOp> mlir::translateLLVMIRToModule( |
3129 | std::unique_ptr<llvm::Module> llvmModule, MLIRContext *context, |
3130 | bool emitExpensiveWarnings, bool dropDICompositeTypeElements, |
3131 | bool loadAllDialects, bool preferUnregisteredIntrinsics, |
3132 | bool importStructsAsLiterals) { |
3133 | // Preload all registered dialects to allow the import to iterate the |
3134 | // registered LLVMImportDialectInterface implementations and query the |
3135 | // supported LLVM IR constructs before starting the translation. Assumes the |
3136 | // LLVM and DLTI dialects that convert the core LLVM IR constructs have been |
3137 | // registered before. |
3138 | assert(llvm::is_contained(context->getAvailableDialects(), |
3139 | LLVMDialect::getDialectNamespace())); |
3140 | assert(llvm::is_contained(context->getAvailableDialects(), |
3141 | DLTIDialect::getDialectNamespace())); |
3142 | if (loadAllDialects) |
3143 | context->loadAllAvailableDialects(); |
3144 | OwningOpRef<ModuleOp> module(ModuleOp::create(FileLineColLoc::get( |
3145 | StringAttr::get(context, llvmModule->getSourceFileName()), /*line=*/0, |
3146 | /*column=*/0))); |
3147 | |
3148 | ModuleImport moduleImport(module.get(), std::move(llvmModule), |
3149 | emitExpensiveWarnings, dropDICompositeTypeElements, |
3150 | preferUnregisteredIntrinsics, |
3151 | importStructsAsLiterals); |
3152 | if (failed(Result: moduleImport.initializeImportInterface())) |
3153 | return {}; |
3154 | if (failed(Result: moduleImport.convertDataLayout())) |
3155 | return {}; |
3156 | if (failed(Result: moduleImport.convertComdats())) |
3157 | return {}; |
3158 | if (failed(Result: moduleImport.convertMetadata())) |
3159 | return {}; |
3160 | if (failed(Result: moduleImport.convertGlobals())) |
3161 | return {}; |
3162 | if (failed(Result: moduleImport.convertFunctions())) |
3163 | return {}; |
3164 | if (failed(Result: moduleImport.convertAliases())) |
3165 | return {}; |
3166 | moduleImport.convertTargetTriple(); |
3167 | return module; |
3168 | } |
3169 |
Definitions
- diag
- diagMD
- getGlobalCtorsVarName
- getNamelessGlobalPrefix
- getGlobalDtorsVarName
- getGlobalComdatOpName
- getLLVMSyncScope
- getPositionFromIndices
- convertInstructionImpl
- getTopologicallySortedBlocks
- ModuleImport
- getGlobalComdatOp
- processTBAAMetadata
- processAccessGroupMetadata
- processAliasScopeMetadata
- lookupAliasScopeAttrs
- addDebugIntrinsic
- convertCGProfileModuleFlagValue
- getTwoElementMDTuple
- getConstantMDFromKeyValueTuple
- convertInt64FromKeyValueTuple
- convertProfileSummaryFormat
- convertProfileSummaryDetailed
- convertProfileSummaryModuleFlagValue
- convertModuleFlagValueFromMDTuple
- convertModuleFlagsMetadata
- convertLinkerOptionsMetadata
- convertDependentLibrariesMetadata
- convertIdentMetadata
- convertCommandlineMetadata
- convertMetadata
- processComdat
- convertComdats
- convertGlobals
- convertAliases
- convertDataLayout
- convertTargetTriple
- convertFunctions
- setNonDebugMetadataAttrs
- setIntegerOverflowFlags
- setExactFlag
- setDisjointFlag
- setNonNegFlag
- setFastmathFlagsAttr
- getVectorTypeForAttr
- getBuiltinTypeForAttr
- getScalarConstantAsAttr
- getSequenceConstantAsAttrs
- getConstantAsAttr
- getOrCreateNamelessSymbolName
- setGlobalInsertionPoint
- convertAlias
- convertGlobal
- convertGlobalCtorsAndDtors
- getConstantsToConvert
- convertConstant
- convertConstantExpr
- convertValue
- convertMetadataValue
- convertValues
- convertIntrinsicArguments
- matchIntegerAttr
- matchFloatAttr
- matchLocalVariableAttr
- matchLabelAttr
- matchFPExceptionBehaviorAttr
- matchRoundingModeAttr
- matchAliasScopeAttrs
- translateLoc
- convertBranchArgs
- convertCallOperands
- checkFunctionTypeCompatibility
- convertFunctionType
- convertCalleeName
- convertIntrinsic
- convertAsmInlineOperandAttrs
- convertInstruction
- processInstruction
- getPersonalityAsAttr
- processMemoryEffects
- kExplicitAttributes
- processPassthroughAttrs
- processFunctionAttributes
- convertParameterAttribute
- convertParameterAttributes
- convertParameterAttributes
- convertParameterAttributes
- convertCallBaseAttributes
- convertInvokeAttributes
- convertCallAttributes
- processFunction
- isMetadataKillLocation
- processDebugIntrinsic
- processDebugIntrinsics
- processBasicBlock
- lookupAccessGroupAttrs
- translateLoopAnnotationAttr
- translateDereferenceableAttr
Learn to use CMake with our Intro Training
Find out more