1 | //===- ModuleTranslation.cpp - MLIR to LLVM conversion --------------------===// |
---|---|
2 | // |
3 | // Part of the LLVM Project, under the Apache License v2.0 with LLVM Exceptions. |
4 | // See https://llvm.org/LICENSE.txt for license information. |
5 | // SPDX-License-Identifier: Apache-2.0 WITH LLVM-exception |
6 | // |
7 | //===----------------------------------------------------------------------===// |
8 | // |
9 | // This file implements the translation between an MLIR LLVM dialect module and |
10 | // the corresponding LLVMIR module. It only handles core LLVM IR operations. |
11 | // |
12 | //===----------------------------------------------------------------------===// |
13 | |
14 | #include "mlir/Target/LLVMIR/ModuleTranslation.h" |
15 | |
16 | #include "AttrKindDetail.h" |
17 | #include "DebugTranslation.h" |
18 | #include "LoopAnnotationTranslation.h" |
19 | #include "mlir/Analysis/TopologicalSortUtils.h" |
20 | #include "mlir/Dialect/DLTI/DLTI.h" |
21 | #include "mlir/Dialect/LLVMIR/LLVMDialect.h" |
22 | #include "mlir/Dialect/LLVMIR/LLVMInterfaces.h" |
23 | #include "mlir/Dialect/LLVMIR/Transforms/DIExpressionLegalization.h" |
24 | #include "mlir/Dialect/LLVMIR/Transforms/LegalizeForExport.h" |
25 | #include "mlir/Dialect/OpenMP/OpenMPDialect.h" |
26 | #include "mlir/Dialect/OpenMP/OpenMPInterfaces.h" |
27 | #include "mlir/IR/AttrTypeSubElements.h" |
28 | #include "mlir/IR/Attributes.h" |
29 | #include "mlir/IR/BuiltinOps.h" |
30 | #include "mlir/IR/BuiltinTypes.h" |
31 | #include "mlir/IR/DialectResourceBlobManager.h" |
32 | #include "mlir/IR/RegionGraphTraits.h" |
33 | #include "mlir/Support/LLVM.h" |
34 | #include "mlir/Target/LLVMIR/LLVMTranslationInterface.h" |
35 | #include "mlir/Target/LLVMIR/TypeToLLVM.h" |
36 | |
37 | #include "llvm/ADT/PostOrderIterator.h" |
38 | #include "llvm/ADT/STLExtras.h" |
39 | #include "llvm/ADT/SetVector.h" |
40 | #include "llvm/ADT/StringExtras.h" |
41 | #include "llvm/ADT/TypeSwitch.h" |
42 | #include "llvm/Analysis/TargetFolder.h" |
43 | #include "llvm/Frontend/OpenMP/OMPIRBuilder.h" |
44 | #include "llvm/IR/BasicBlock.h" |
45 | #include "llvm/IR/CFG.h" |
46 | #include "llvm/IR/Constants.h" |
47 | #include "llvm/IR/DerivedTypes.h" |
48 | #include "llvm/IR/IRBuilder.h" |
49 | #include "llvm/IR/InlineAsm.h" |
50 | #include "llvm/IR/IntrinsicsNVPTX.h" |
51 | #include "llvm/IR/LLVMContext.h" |
52 | #include "llvm/IR/MDBuilder.h" |
53 | #include "llvm/IR/Module.h" |
54 | #include "llvm/IR/Verifier.h" |
55 | #include "llvm/Support/Debug.h" |
56 | #include "llvm/Support/ErrorHandling.h" |
57 | #include "llvm/Support/raw_ostream.h" |
58 | #include "llvm/Transforms/Utils/BasicBlockUtils.h" |
59 | #include "llvm/Transforms/Utils/Cloning.h" |
60 | #include "llvm/Transforms/Utils/ModuleUtils.h" |
61 | #include <numeric> |
62 | #include <optional> |
63 | |
64 | #define DEBUG_TYPE "llvm-dialect-to-llvm-ir" |
65 | |
66 | using namespace mlir; |
67 | using namespace mlir::LLVM; |
68 | using namespace mlir::LLVM::detail; |
69 | |
70 | #include "mlir/Dialect/LLVMIR/LLVMConversionEnumsToLLVM.inc" |
71 | |
72 | namespace { |
73 | /// A customized inserter for LLVM's IRBuilder that captures all LLVM IR |
74 | /// instructions that are created for future reference. |
75 | /// |
76 | /// This is intended to be used with the `CollectionScope` RAII object: |
77 | /// |
78 | /// llvm::IRBuilder<..., InstructionCapturingInserter> builder; |
79 | /// { |
80 | /// InstructionCapturingInserter::CollectionScope scope(builder); |
81 | /// // Call IRBuilder methods as usual. |
82 | /// |
83 | /// // This will return a list of all instructions created by the builder, |
84 | /// // in order of creation. |
85 | /// builder.getInserter().getCapturedInstructions(); |
86 | /// } |
87 | /// // This will return an empty list. |
88 | /// builder.getInserter().getCapturedInstructions(); |
89 | /// |
90 | /// The capturing functionality is _disabled_ by default for performance |
91 | /// consideration. It needs to be explicitly enabled, which is achieved by |
92 | /// creating a `CollectionScope`. |
93 | class InstructionCapturingInserter : public llvm::IRBuilderCallbackInserter { |
94 | public: |
95 | /// Constructs the inserter. |
96 | InstructionCapturingInserter() |
97 | : llvm::IRBuilderCallbackInserter([this](llvm::Instruction *instruction) { |
98 | if (LLVM_LIKELY(enabled)) |
99 | capturedInstructions.push_back(instruction); |
100 | }) {} |
101 | |
102 | /// Returns the list of LLVM IR instructions captured since the last cleanup. |
103 | ArrayRef<llvm::Instruction *> getCapturedInstructions() const { |
104 | return capturedInstructions; |
105 | } |
106 | |
107 | /// Clears the list of captured LLVM IR instructions. |
108 | void clearCapturedInstructions() { capturedInstructions.clear(); } |
109 | |
110 | /// RAII object enabling the capture of created LLVM IR instructions. |
111 | class CollectionScope { |
112 | public: |
113 | /// Creates the scope for the given inserter. |
114 | CollectionScope(llvm::IRBuilderBase &irBuilder, bool isBuilderCapturing); |
115 | |
116 | /// Ends the scope. |
117 | ~CollectionScope(); |
118 | |
119 | ArrayRef<llvm::Instruction *> getCapturedInstructions() { |
120 | if (!inserter) |
121 | return {}; |
122 | return inserter->getCapturedInstructions(); |
123 | } |
124 | |
125 | private: |
126 | /// Back reference to the inserter. |
127 | InstructionCapturingInserter *inserter = nullptr; |
128 | |
129 | /// List of instructions in the inserter prior to this scope. |
130 | SmallVector<llvm::Instruction *> previouslyCollectedInstructions; |
131 | |
132 | /// Whether the inserter was enabled prior to this scope. |
133 | bool wasEnabled; |
134 | }; |
135 | |
136 | /// Enable or disable the capturing mechanism. |
137 | void setEnabled(bool enabled = true) { this->enabled = enabled; } |
138 | |
139 | private: |
140 | /// List of captured instructions. |
141 | SmallVector<llvm::Instruction *> capturedInstructions; |
142 | |
143 | /// Whether the collection is enabled. |
144 | bool enabled = false; |
145 | }; |
146 | |
147 | using CapturingIRBuilder = |
148 | llvm::IRBuilder<llvm::TargetFolder, InstructionCapturingInserter>; |
149 | } // namespace |
150 | |
151 | InstructionCapturingInserter::CollectionScope::CollectionScope( |
152 | llvm::IRBuilderBase &irBuilder, bool isBuilderCapturing) { |
153 | |
154 | if (!isBuilderCapturing) |
155 | return; |
156 | |
157 | auto &capturingIRBuilder = static_cast<CapturingIRBuilder &>(irBuilder); |
158 | inserter = &capturingIRBuilder.getInserter(); |
159 | wasEnabled = inserter->enabled; |
160 | if (wasEnabled) |
161 | previouslyCollectedInstructions.swap(inserter->capturedInstructions); |
162 | inserter->setEnabled(true); |
163 | } |
164 | |
165 | InstructionCapturingInserter::CollectionScope::~CollectionScope() { |
166 | if (!inserter) |
167 | return; |
168 | |
169 | previouslyCollectedInstructions.swap(inserter->capturedInstructions); |
170 | // If collection was enabled (likely in another, surrounding scope), keep |
171 | // the instructions collected in this scope. |
172 | if (wasEnabled) { |
173 | llvm::append_range(inserter->capturedInstructions, |
174 | previouslyCollectedInstructions); |
175 | } |
176 | inserter->setEnabled(wasEnabled); |
177 | } |
178 | |
179 | /// Translates the given data layout spec attribute to the LLVM IR data layout. |
180 | /// Only integer, float, pointer and endianness entries are currently supported. |
181 | static FailureOr<llvm::DataLayout> |
182 | translateDataLayout(DataLayoutSpecInterface attribute, |
183 | const DataLayout &dataLayout, |
184 | std::optional<Location> loc = std::nullopt) { |
185 | if (!loc) |
186 | loc = UnknownLoc::get(attribute.getContext()); |
187 | |
188 | // Translate the endianness attribute. |
189 | std::string llvmDataLayout; |
190 | llvm::raw_string_ostream layoutStream(llvmDataLayout); |
191 | for (DataLayoutEntryInterface entry : attribute.getEntries()) { |
192 | auto key = llvm::dyn_cast_if_present<StringAttr>(entry.getKey()); |
193 | if (!key) |
194 | continue; |
195 | if (key.getValue() == DLTIDialect::kDataLayoutEndiannessKey) { |
196 | auto value = cast<StringAttr>(entry.getValue()); |
197 | bool isLittleEndian = |
198 | value.getValue() == DLTIDialect::kDataLayoutEndiannessLittle; |
199 | layoutStream << "-"<< (isLittleEndian ? "e": "E"); |
200 | continue; |
201 | } |
202 | if (key.getValue() == DLTIDialect::kDataLayoutManglingModeKey) { |
203 | auto value = cast<StringAttr>(entry.getValue()); |
204 | layoutStream << "-m:"<< value.getValue(); |
205 | continue; |
206 | } |
207 | if (key.getValue() == DLTIDialect::kDataLayoutProgramMemorySpaceKey) { |
208 | auto value = cast<IntegerAttr>(entry.getValue()); |
209 | uint64_t space = value.getValue().getZExtValue(); |
210 | // Skip the default address space. |
211 | if (space == 0) |
212 | continue; |
213 | layoutStream << "-P"<< space; |
214 | continue; |
215 | } |
216 | if (key.getValue() == DLTIDialect::kDataLayoutGlobalMemorySpaceKey) { |
217 | auto value = cast<IntegerAttr>(entry.getValue()); |
218 | uint64_t space = value.getValue().getZExtValue(); |
219 | // Skip the default address space. |
220 | if (space == 0) |
221 | continue; |
222 | layoutStream << "-G"<< space; |
223 | continue; |
224 | } |
225 | if (key.getValue() == DLTIDialect::kDataLayoutAllocaMemorySpaceKey) { |
226 | auto value = cast<IntegerAttr>(entry.getValue()); |
227 | uint64_t space = value.getValue().getZExtValue(); |
228 | // Skip the default address space. |
229 | if (space == 0) |
230 | continue; |
231 | layoutStream << "-A"<< space; |
232 | continue; |
233 | } |
234 | if (key.getValue() == DLTIDialect::kDataLayoutStackAlignmentKey) { |
235 | auto value = cast<IntegerAttr>(entry.getValue()); |
236 | uint64_t alignment = value.getValue().getZExtValue(); |
237 | // Skip the default stack alignment. |
238 | if (alignment == 0) |
239 | continue; |
240 | layoutStream << "-S"<< alignment; |
241 | continue; |
242 | } |
243 | if (key.getValue() == DLTIDialect::kDataLayoutFunctionPointerAlignmentKey) { |
244 | auto value = cast<FunctionPointerAlignmentAttr>(entry.getValue()); |
245 | uint64_t alignment = value.getAlignment(); |
246 | // Skip the default function pointer alignment. |
247 | if (alignment == 0) |
248 | continue; |
249 | layoutStream << "-F"<< (value.getFunctionDependent() ? "n": "i") |
250 | << alignment; |
251 | continue; |
252 | } |
253 | if (key.getValue() == DLTIDialect::kDataLayoutLegalIntWidthsKey) { |
254 | layoutStream << "-n"; |
255 | llvm::interleave( |
256 | cast<DenseI32ArrayAttr>(entry.getValue()).asArrayRef(), layoutStream, |
257 | [&](int32_t val) { layoutStream << val; }, ":"); |
258 | continue; |
259 | } |
260 | emitError(*loc) << "unsupported data layout key "<< key; |
261 | return failure(); |
262 | } |
263 | |
264 | // Go through the list of entries to check which types are explicitly |
265 | // specified in entries. Where possible, data layout queries are used instead |
266 | // of directly inspecting the entries. |
267 | for (DataLayoutEntryInterface entry : attribute.getEntries()) { |
268 | auto type = llvm::dyn_cast_if_present<Type>(entry.getKey()); |
269 | if (!type) |
270 | continue; |
271 | // Data layout for the index type is irrelevant at this point. |
272 | if (isa<IndexType>(type)) |
273 | continue; |
274 | layoutStream << "-"; |
275 | LogicalResult result = |
276 | llvm::TypeSwitch<Type, LogicalResult>(type) |
277 | .Case<IntegerType, Float16Type, Float32Type, Float64Type, |
278 | Float80Type, Float128Type>([&](Type type) -> LogicalResult { |
279 | if (auto intType = dyn_cast<IntegerType>(type)) { |
280 | if (intType.getSignedness() != IntegerType::Signless) |
281 | return emitError(*loc) |
282 | << "unsupported data layout for non-signless integer " |
283 | << intType; |
284 | layoutStream << "i"; |
285 | } else { |
286 | layoutStream << "f"; |
287 | } |
288 | uint64_t size = dataLayout.getTypeSizeInBits(type); |
289 | uint64_t abi = dataLayout.getTypeABIAlignment(type) * 8u; |
290 | uint64_t preferred = |
291 | dataLayout.getTypePreferredAlignment(type) * 8u; |
292 | layoutStream << size << ":"<< abi; |
293 | if (abi != preferred) |
294 | layoutStream << ":"<< preferred; |
295 | return success(); |
296 | }) |
297 | .Case([&](LLVMPointerType type) { |
298 | layoutStream << "p"<< type.getAddressSpace() << ":"; |
299 | uint64_t size = dataLayout.getTypeSizeInBits(type); |
300 | uint64_t abi = dataLayout.getTypeABIAlignment(type) * 8u; |
301 | uint64_t preferred = |
302 | dataLayout.getTypePreferredAlignment(type) * 8u; |
303 | uint64_t index = *dataLayout.getTypeIndexBitwidth(type); |
304 | layoutStream << size << ":"<< abi << ":"<< preferred << ":" |
305 | << index; |
306 | return success(); |
307 | }) |
308 | .Default([loc](Type type) { |
309 | return emitError(*loc) |
310 | << "unsupported type in data layout: "<< type; |
311 | }); |
312 | if (failed(result)) |
313 | return failure(); |
314 | } |
315 | StringRef layoutSpec(llvmDataLayout); |
316 | layoutSpec.consume_front(Prefix: "-"); |
317 | |
318 | return llvm::DataLayout(layoutSpec); |
319 | } |
320 | |
321 | /// Builds a constant of a sequential LLVM type `type`, potentially containing |
322 | /// other sequential types recursively, from the individual constant values |
323 | /// provided in `constants`. `shape` contains the number of elements in nested |
324 | /// sequential types. Reports errors at `loc` and returns nullptr on error. |
325 | static llvm::Constant * |
326 | buildSequentialConstant(ArrayRef<llvm::Constant *> &constants, |
327 | ArrayRef<int64_t> shape, llvm::Type *type, |
328 | Location loc) { |
329 | if (shape.empty()) { |
330 | llvm::Constant *result = constants.front(); |
331 | constants = constants.drop_front(); |
332 | return result; |
333 | } |
334 | |
335 | llvm::Type *elementType; |
336 | if (auto *arrayTy = dyn_cast<llvm::ArrayType>(Val: type)) { |
337 | elementType = arrayTy->getElementType(); |
338 | } else if (auto *vectorTy = dyn_cast<llvm::VectorType>(Val: type)) { |
339 | elementType = vectorTy->getElementType(); |
340 | } else { |
341 | emitError(loc) << "expected sequential LLVM types wrapping a scalar"; |
342 | return nullptr; |
343 | } |
344 | |
345 | SmallVector<llvm::Constant *, 8> nested; |
346 | nested.reserve(N: shape.front()); |
347 | for (int64_t i = 0; i < shape.front(); ++i) { |
348 | nested.push_back(Elt: buildSequentialConstant(constants, shape: shape.drop_front(), |
349 | type: elementType, loc)); |
350 | if (!nested.back()) |
351 | return nullptr; |
352 | } |
353 | |
354 | if (shape.size() == 1 && type->isVectorTy()) |
355 | return llvm::ConstantVector::get(V: nested); |
356 | return llvm::ConstantArray::get( |
357 | T: llvm::ArrayType::get(ElementType: elementType, NumElements: shape.front()), V: nested); |
358 | } |
359 | |
360 | /// Returns the first non-sequential type nested in sequential types. |
361 | static llvm::Type *getInnermostElementType(llvm::Type *type) { |
362 | do { |
363 | if (auto *arrayTy = dyn_cast<llvm::ArrayType>(Val: type)) { |
364 | type = arrayTy->getElementType(); |
365 | } else if (auto *vectorTy = dyn_cast<llvm::VectorType>(Val: type)) { |
366 | type = vectorTy->getElementType(); |
367 | } else { |
368 | return type; |
369 | } |
370 | } while (true); |
371 | } |
372 | |
373 | /// Convert a dense elements attribute to an LLVM IR constant using its raw data |
374 | /// storage if possible. This supports elements attributes of tensor or vector |
375 | /// type and avoids constructing separate objects for individual values of the |
376 | /// innermost dimension. Constants for other dimensions are still constructed |
377 | /// recursively. Returns null if constructing from raw data is not supported for |
378 | /// this type, e.g., element type is not a power-of-two-sized primitive. Reports |
379 | /// other errors at `loc`. |
380 | static llvm::Constant * |
381 | convertDenseElementsAttr(Location loc, DenseElementsAttr denseElementsAttr, |
382 | llvm::Type *llvmType, |
383 | const ModuleTranslation &moduleTranslation) { |
384 | if (!denseElementsAttr) |
385 | return nullptr; |
386 | |
387 | llvm::Type *innermostLLVMType = getInnermostElementType(type: llvmType); |
388 | if (!llvm::ConstantDataSequential::isElementTypeCompatible(Ty: innermostLLVMType)) |
389 | return nullptr; |
390 | |
391 | ShapedType type = denseElementsAttr.getType(); |
392 | if (type.getNumElements() == 0) |
393 | return nullptr; |
394 | |
395 | // Check that the raw data size matches what is expected for the scalar size. |
396 | // TODO: in theory, we could repack the data here to keep constructing from |
397 | // raw data. |
398 | // TODO: we may also need to consider endianness when cross-compiling to an |
399 | // architecture where it is different. |
400 | int64_t elementByteSize = denseElementsAttr.getRawData().size() / |
401 | denseElementsAttr.getNumElements(); |
402 | if (8 * elementByteSize != innermostLLVMType->getScalarSizeInBits()) |
403 | return nullptr; |
404 | |
405 | // Compute the shape of all dimensions but the innermost. Note that the |
406 | // innermost dimension may be that of the vector element type. |
407 | bool hasVectorElementType = isa<VectorType>(type.getElementType()); |
408 | int64_t numAggregates = |
409 | denseElementsAttr.getNumElements() / |
410 | (hasVectorElementType ? 1 |
411 | : denseElementsAttr.getType().getShape().back()); |
412 | ArrayRef<int64_t> outerShape = type.getShape(); |
413 | if (!hasVectorElementType) |
414 | outerShape = outerShape.drop_back(); |
415 | |
416 | // Handle the case of vector splat, LLVM has special support for it. |
417 | if (denseElementsAttr.isSplat() && |
418 | (isa<VectorType>(type) || hasVectorElementType)) { |
419 | llvm::Constant *splatValue = LLVM::detail::getLLVMConstant( |
420 | llvmType: innermostLLVMType, attr: denseElementsAttr.getSplatValue<Attribute>(), loc, |
421 | moduleTranslation); |
422 | llvm::Constant *splatVector = |
423 | llvm::ConstantDataVector::getSplat(NumElts: 0, Elt: splatValue); |
424 | SmallVector<llvm::Constant *> constants(numAggregates, splatVector); |
425 | ArrayRef<llvm::Constant *> constantsRef = constants; |
426 | return buildSequentialConstant(constants&: constantsRef, shape: outerShape, type: llvmType, loc); |
427 | } |
428 | if (denseElementsAttr.isSplat()) |
429 | return nullptr; |
430 | |
431 | // In case of non-splat, create a constructor for the innermost constant from |
432 | // a piece of raw data. |
433 | std::function<llvm::Constant *(StringRef)> buildCstData; |
434 | if (isa<TensorType>(type)) { |
435 | auto vectorElementType = dyn_cast<VectorType>(type.getElementType()); |
436 | if (vectorElementType && vectorElementType.getRank() == 1) { |
437 | buildCstData = [&](StringRef data) { |
438 | return llvm::ConstantDataVector::getRaw( |
439 | data, vectorElementType.getShape().back(), innermostLLVMType); |
440 | }; |
441 | } else if (!vectorElementType) { |
442 | buildCstData = [&](StringRef data) { |
443 | return llvm::ConstantDataArray::getRaw(data, type.getShape().back(), |
444 | innermostLLVMType); |
445 | }; |
446 | } |
447 | } else if (isa<VectorType>(type)) { |
448 | buildCstData = [&](StringRef data) { |
449 | return llvm::ConstantDataVector::getRaw(data, type.getShape().back(), |
450 | innermostLLVMType); |
451 | }; |
452 | } |
453 | if (!buildCstData) |
454 | return nullptr; |
455 | |
456 | // Create innermost constants and defer to the default constant creation |
457 | // mechanism for other dimensions. |
458 | SmallVector<llvm::Constant *> constants; |
459 | int64_t aggregateSize = denseElementsAttr.getType().getShape().back() * |
460 | (innermostLLVMType->getScalarSizeInBits() / 8); |
461 | constants.reserve(N: numAggregates); |
462 | for (unsigned i = 0; i < numAggregates; ++i) { |
463 | StringRef data(denseElementsAttr.getRawData().data() + i * aggregateSize, |
464 | aggregateSize); |
465 | constants.push_back(Elt: buildCstData(data)); |
466 | } |
467 | |
468 | ArrayRef<llvm::Constant *> constantsRef = constants; |
469 | return buildSequentialConstant(constants&: constantsRef, shape: outerShape, type: llvmType, loc); |
470 | } |
471 | |
472 | /// Convert a dense resource elements attribute to an LLVM IR constant using its |
473 | /// raw data storage if possible. This supports elements attributes of tensor or |
474 | /// vector type and avoids constructing separate objects for individual values |
475 | /// of the innermost dimension. Constants for other dimensions are still |
476 | /// constructed recursively. Returns nullptr on failure and emits errors at |
477 | /// `loc`. |
478 | static llvm::Constant *convertDenseResourceElementsAttr( |
479 | Location loc, DenseResourceElementsAttr denseResourceAttr, |
480 | llvm::Type *llvmType, const ModuleTranslation &moduleTranslation) { |
481 | assert(denseResourceAttr && "expected non-null attribute"); |
482 | |
483 | llvm::Type *innermostLLVMType = getInnermostElementType(type: llvmType); |
484 | if (!llvm::ConstantDataSequential::isElementTypeCompatible( |
485 | Ty: innermostLLVMType)) { |
486 | emitError(loc, message: "no known conversion for innermost element type"); |
487 | return nullptr; |
488 | } |
489 | |
490 | ShapedType type = denseResourceAttr.getType(); |
491 | assert(type.getNumElements() > 0 && "Expected non-empty elements attribute"); |
492 | |
493 | AsmResourceBlob *blob = denseResourceAttr.getRawHandle().getBlob(); |
494 | if (!blob) { |
495 | emitError(loc, message: "resource does not exist"); |
496 | return nullptr; |
497 | } |
498 | |
499 | ArrayRef<char> rawData = blob->getData(); |
500 | |
501 | // Check that the raw data size matches what is expected for the scalar size. |
502 | // TODO: in theory, we could repack the data here to keep constructing from |
503 | // raw data. |
504 | // TODO: we may also need to consider endianness when cross-compiling to an |
505 | // architecture where it is different. |
506 | int64_t numElements = denseResourceAttr.getType().getNumElements(); |
507 | int64_t elementByteSize = rawData.size() / numElements; |
508 | if (8 * elementByteSize != innermostLLVMType->getScalarSizeInBits()) { |
509 | emitError(loc, message: "raw data size does not match element type size"); |
510 | return nullptr; |
511 | } |
512 | |
513 | // Compute the shape of all dimensions but the innermost. Note that the |
514 | // innermost dimension may be that of the vector element type. |
515 | bool hasVectorElementType = isa<VectorType>(type.getElementType()); |
516 | int64_t numAggregates = |
517 | numElements / (hasVectorElementType |
518 | ? 1 |
519 | : denseResourceAttr.getType().getShape().back()); |
520 | ArrayRef<int64_t> outerShape = type.getShape(); |
521 | if (!hasVectorElementType) |
522 | outerShape = outerShape.drop_back(); |
523 | |
524 | // Create a constructor for the innermost constant from a piece of raw data. |
525 | std::function<llvm::Constant *(StringRef)> buildCstData; |
526 | if (isa<TensorType>(type)) { |
527 | auto vectorElementType = dyn_cast<VectorType>(type.getElementType()); |
528 | if (vectorElementType && vectorElementType.getRank() == 1) { |
529 | buildCstData = [&](StringRef data) { |
530 | return llvm::ConstantDataVector::getRaw( |
531 | data, vectorElementType.getShape().back(), innermostLLVMType); |
532 | }; |
533 | } else if (!vectorElementType) { |
534 | buildCstData = [&](StringRef data) { |
535 | return llvm::ConstantDataArray::getRaw(data, type.getShape().back(), |
536 | innermostLLVMType); |
537 | }; |
538 | } |
539 | } else if (isa<VectorType>(type)) { |
540 | buildCstData = [&](StringRef data) { |
541 | return llvm::ConstantDataVector::getRaw(data, type.getShape().back(), |
542 | innermostLLVMType); |
543 | }; |
544 | } |
545 | if (!buildCstData) { |
546 | emitError(loc, message: "unsupported dense_resource type"); |
547 | return nullptr; |
548 | } |
549 | |
550 | // Create innermost constants and defer to the default constant creation |
551 | // mechanism for other dimensions. |
552 | SmallVector<llvm::Constant *> constants; |
553 | int64_t aggregateSize = denseResourceAttr.getType().getShape().back() * |
554 | (innermostLLVMType->getScalarSizeInBits() / 8); |
555 | constants.reserve(N: numAggregates); |
556 | for (unsigned i = 0; i < numAggregates; ++i) { |
557 | StringRef data(rawData.data() + i * aggregateSize, aggregateSize); |
558 | constants.push_back(Elt: buildCstData(data)); |
559 | } |
560 | |
561 | ArrayRef<llvm::Constant *> constantsRef = constants; |
562 | return buildSequentialConstant(constants&: constantsRef, shape: outerShape, type: llvmType, loc); |
563 | } |
564 | |
565 | /// Create an LLVM IR constant of `llvmType` from the MLIR attribute `attr`. |
566 | /// This currently supports integer, floating point, splat and dense element |
567 | /// attributes and combinations thereof. Also, an array attribute with two |
568 | /// elements is supported to represent a complex constant. In case of error, |
569 | /// report it to `loc` and return nullptr. |
570 | llvm::Constant *mlir::LLVM::detail::getLLVMConstant( |
571 | llvm::Type *llvmType, Attribute attr, Location loc, |
572 | const ModuleTranslation &moduleTranslation) { |
573 | if (!attr || isa<UndefAttr>(attr)) |
574 | return llvm::UndefValue::get(T: llvmType); |
575 | if (isa<ZeroAttr>(attr)) |
576 | return llvm::Constant::getNullValue(Ty: llvmType); |
577 | if (auto *structType = dyn_cast<::llvm::StructType>(Val: llvmType)) { |
578 | auto arrayAttr = dyn_cast<ArrayAttr>(attr); |
579 | if (!arrayAttr) { |
580 | emitError(loc, message: "expected an array attribute for a struct constant"); |
581 | return nullptr; |
582 | } |
583 | SmallVector<llvm::Constant *> structElements; |
584 | structElements.reserve(N: structType->getNumElements()); |
585 | for (auto [elemType, elemAttr] : |
586 | zip_equal(structType->elements(), arrayAttr)) { |
587 | llvm::Constant *element = |
588 | getLLVMConstant(elemType, elemAttr, loc, moduleTranslation); |
589 | if (!element) |
590 | return nullptr; |
591 | structElements.push_back(element); |
592 | } |
593 | return llvm::ConstantStruct::get(T: structType, V: structElements); |
594 | } |
595 | // For integer types, we allow a mismatch in sizes as the index type in |
596 | // MLIR might have a different size than the index type in the LLVM module. |
597 | if (auto intAttr = dyn_cast<IntegerAttr>(attr)) |
598 | return llvm::ConstantInt::get( |
599 | llvmType, |
600 | intAttr.getValue().sextOrTrunc(llvmType->getIntegerBitWidth())); |
601 | if (auto floatAttr = dyn_cast<FloatAttr>(attr)) { |
602 | const llvm::fltSemantics &sem = floatAttr.getValue().getSemantics(); |
603 | // Special case for 8-bit floats, which are represented by integers due to |
604 | // the lack of native fp8 types in LLVM at the moment. Additionally, handle |
605 | // targets (like AMDGPU) that don't implement bfloat and convert all bfloats |
606 | // to i16. |
607 | unsigned floatWidth = APFloat::getSizeInBits(Sem: sem); |
608 | if (llvmType->isIntegerTy(Bitwidth: floatWidth)) |
609 | return llvm::ConstantInt::get(llvmType, |
610 | floatAttr.getValue().bitcastToAPInt()); |
611 | if (llvmType != |
612 | llvm::Type::getFloatingPointTy(C&: llvmType->getContext(), |
613 | S: floatAttr.getValue().getSemantics())) { |
614 | emitError(loc, message: "FloatAttr does not match expected type of the constant"); |
615 | return nullptr; |
616 | } |
617 | return llvm::ConstantFP::get(llvmType, floatAttr.getValue()); |
618 | } |
619 | if (auto funcAttr = dyn_cast<FlatSymbolRefAttr>(attr)) |
620 | return llvm::ConstantExpr::getBitCast( |
621 | C: moduleTranslation.lookupFunction(name: funcAttr.getValue()), Ty: llvmType); |
622 | if (auto splatAttr = dyn_cast<SplatElementsAttr>(Val&: attr)) { |
623 | llvm::Type *elementType; |
624 | uint64_t numElements; |
625 | bool isScalable = false; |
626 | if (auto *arrayTy = dyn_cast<llvm::ArrayType>(Val: llvmType)) { |
627 | elementType = arrayTy->getElementType(); |
628 | numElements = arrayTy->getNumElements(); |
629 | } else if (auto *fVectorTy = dyn_cast<llvm::FixedVectorType>(Val: llvmType)) { |
630 | elementType = fVectorTy->getElementType(); |
631 | numElements = fVectorTy->getNumElements(); |
632 | } else if (auto *sVectorTy = dyn_cast<llvm::ScalableVectorType>(Val: llvmType)) { |
633 | elementType = sVectorTy->getElementType(); |
634 | numElements = sVectorTy->getMinNumElements(); |
635 | isScalable = true; |
636 | } else { |
637 | llvm_unreachable("unrecognized constant vector type"); |
638 | } |
639 | // Splat value is a scalar. Extract it only if the element type is not |
640 | // another sequence type. The recursion terminates because each step removes |
641 | // one outer sequential type. |
642 | bool elementTypeSequential = |
643 | isa<llvm::ArrayType, llvm::VectorType>(Val: elementType); |
644 | llvm::Constant *child = getLLVMConstant( |
645 | llvmType: elementType, |
646 | attr: elementTypeSequential ? splatAttr |
647 | : splatAttr.getSplatValue<Attribute>(), |
648 | loc, moduleTranslation); |
649 | if (!child) |
650 | return nullptr; |
651 | if (llvmType->isVectorTy()) |
652 | return llvm::ConstantVector::getSplat( |
653 | EC: llvm::ElementCount::get(MinVal: numElements, /*Scalable=*/isScalable), Elt: child); |
654 | if (llvmType->isArrayTy()) { |
655 | auto *arrayType = llvm::ArrayType::get(ElementType: elementType, NumElements: numElements); |
656 | if (child->isZeroValue()) { |
657 | return llvm::ConstantAggregateZero::get(Ty: arrayType); |
658 | } else { |
659 | if (llvm::ConstantDataSequential::isElementTypeCompatible( |
660 | Ty: elementType)) { |
661 | // TODO: Handle all compatible types. This code only handles integer. |
662 | if (isa<llvm::IntegerType>(Val: elementType)) { |
663 | if (llvm::ConstantInt *ci = dyn_cast<llvm::ConstantInt>(Val: child)) { |
664 | if (ci->getBitWidth() == 8) { |
665 | SmallVector<int8_t> constants(numElements, ci->getZExtValue()); |
666 | return llvm::ConstantDataArray::get(Context&: elementType->getContext(), |
667 | Elts&: constants); |
668 | } |
669 | if (ci->getBitWidth() == 16) { |
670 | SmallVector<int16_t> constants(numElements, ci->getZExtValue()); |
671 | return llvm::ConstantDataArray::get(Context&: elementType->getContext(), |
672 | Elts&: constants); |
673 | } |
674 | if (ci->getBitWidth() == 32) { |
675 | SmallVector<int32_t> constants(numElements, ci->getZExtValue()); |
676 | return llvm::ConstantDataArray::get(Context&: elementType->getContext(), |
677 | Elts&: constants); |
678 | } |
679 | if (ci->getBitWidth() == 64) { |
680 | SmallVector<int64_t> constants(numElements, ci->getZExtValue()); |
681 | return llvm::ConstantDataArray::get(Context&: elementType->getContext(), |
682 | Elts&: constants); |
683 | } |
684 | } |
685 | } |
686 | } |
687 | // std::vector is used here to accomodate large number of elements that |
688 | // exceed SmallVector capacity. |
689 | std::vector<llvm::Constant *> constants(numElements, child); |
690 | return llvm::ConstantArray::get(T: arrayType, V: constants); |
691 | } |
692 | } |
693 | } |
694 | |
695 | // Try using raw elements data if possible. |
696 | if (llvm::Constant *result = |
697 | convertDenseElementsAttr(loc, denseElementsAttr: dyn_cast<DenseElementsAttr>(Val&: attr), |
698 | llvmType, moduleTranslation)) { |
699 | return result; |
700 | } |
701 | |
702 | if (auto denseResourceAttr = dyn_cast<DenseResourceElementsAttr>(attr)) { |
703 | return convertDenseResourceElementsAttr(loc, denseResourceAttr, llvmType, |
704 | moduleTranslation); |
705 | } |
706 | |
707 | // Fall back to element-by-element construction otherwise. |
708 | if (auto elementsAttr = dyn_cast<ElementsAttr>(attr)) { |
709 | assert(elementsAttr.getShapedType().hasStaticShape()); |
710 | assert(!elementsAttr.getShapedType().getShape().empty() && |
711 | "unexpected empty elements attribute shape"); |
712 | |
713 | SmallVector<llvm::Constant *, 8> constants; |
714 | constants.reserve(N: elementsAttr.getNumElements()); |
715 | llvm::Type *innermostType = getInnermostElementType(type: llvmType); |
716 | for (auto n : elementsAttr.getValues<Attribute>()) { |
717 | constants.push_back( |
718 | getLLVMConstant(innermostType, n, loc, moduleTranslation)); |
719 | if (!constants.back()) |
720 | return nullptr; |
721 | } |
722 | ArrayRef<llvm::Constant *> constantsRef = constants; |
723 | llvm::Constant *result = buildSequentialConstant( |
724 | constantsRef, elementsAttr.getShapedType().getShape(), llvmType, loc); |
725 | assert(constantsRef.empty() && "did not consume all elemental constants"); |
726 | return result; |
727 | } |
728 | |
729 | if (auto stringAttr = dyn_cast<StringAttr>(attr)) { |
730 | return llvm::ConstantDataArray::get( |
731 | Context&: moduleTranslation.getLLVMContext(), |
732 | Elts: ArrayRef<char>{stringAttr.getValue().data(), |
733 | stringAttr.getValue().size()}); |
734 | } |
735 | |
736 | // Handle arrays of structs that cannot be represented as DenseElementsAttr |
737 | // in MLIR. |
738 | if (auto arrayAttr = dyn_cast<ArrayAttr>(attr)) { |
739 | if (auto *arrayTy = dyn_cast<llvm::ArrayType>(Val: llvmType)) { |
740 | llvm::Type *elementType = arrayTy->getElementType(); |
741 | Attribute previousElementAttr; |
742 | llvm::Constant *elementCst = nullptr; |
743 | SmallVector<llvm::Constant *> constants; |
744 | constants.reserve(N: arrayTy->getNumElements()); |
745 | for (Attribute elementAttr : arrayAttr) { |
746 | // Arrays with a single value or with repeating values are quite common. |
747 | // Short-circuit the translation when the element value is the same as |
748 | // the previous one. |
749 | if (!previousElementAttr || previousElementAttr != elementAttr) { |
750 | previousElementAttr = elementAttr; |
751 | elementCst = |
752 | getLLVMConstant(elementType, elementAttr, loc, moduleTranslation); |
753 | if (!elementCst) |
754 | return nullptr; |
755 | } |
756 | constants.push_back(elementCst); |
757 | } |
758 | return llvm::ConstantArray::get(T: arrayTy, V: constants); |
759 | } |
760 | } |
761 | |
762 | emitError(loc, message: "unsupported constant value"); |
763 | return nullptr; |
764 | } |
765 | |
766 | ModuleTranslation::ModuleTranslation(Operation *module, |
767 | std::unique_ptr<llvm::Module> llvmModule) |
768 | : mlirModule(module), llvmModule(std::move(llvmModule)), |
769 | debugTranslation( |
770 | std::make_unique<DebugTranslation>(args&: module, args&: *this->llvmModule)), |
771 | loopAnnotationTranslation(std::make_unique<LoopAnnotationTranslation>( |
772 | args&: *this, args&: *this->llvmModule)), |
773 | typeTranslator(this->llvmModule->getContext()), |
774 | iface(module->getContext()) { |
775 | assert(satisfiesLLVMModule(mlirModule) && |
776 | "mlirModule should honor LLVM's module semantics."); |
777 | } |
778 | |
779 | ModuleTranslation::~ModuleTranslation() { |
780 | if (ompBuilder) |
781 | ompBuilder->finalize(); |
782 | } |
783 | |
784 | void ModuleTranslation::forgetMapping(Region ®ion) { |
785 | SmallVector<Region *> toProcess; |
786 | toProcess.push_back(Elt: ®ion); |
787 | while (!toProcess.empty()) { |
788 | Region *current = toProcess.pop_back_val(); |
789 | for (Block &block : *current) { |
790 | blockMapping.erase(Val: &block); |
791 | for (Value arg : block.getArguments()) |
792 | valueMapping.erase(Val: arg); |
793 | for (Operation &op : block) { |
794 | for (Value value : op.getResults()) |
795 | valueMapping.erase(Val: value); |
796 | if (op.hasSuccessors()) |
797 | branchMapping.erase(Val: &op); |
798 | if (isa<LLVM::GlobalOp>(op)) |
799 | globalsMapping.erase(Val: &op); |
800 | if (isa<LLVM::AliasOp>(op)) |
801 | aliasesMapping.erase(Val: &op); |
802 | if (isa<LLVM::CallOp>(op)) |
803 | callMapping.erase(Val: &op); |
804 | llvm::append_range( |
805 | C&: toProcess, |
806 | R: llvm::map_range(C: op.getRegions(), F: [](Region &r) { return &r; })); |
807 | } |
808 | } |
809 | } |
810 | } |
811 | |
812 | /// Get the SSA value passed to the current block from the terminator operation |
813 | /// of its predecessor. |
814 | static Value getPHISourceValue(Block *current, Block *pred, |
815 | unsigned numArguments, unsigned index) { |
816 | Operation &terminator = *pred->getTerminator(); |
817 | if (isa<LLVM::BrOp>(terminator)) |
818 | return terminator.getOperand(idx: index); |
819 | |
820 | #ifndef NDEBUG |
821 | llvm::SmallPtrSet<Block *, 4> seenSuccessors; |
822 | for (unsigned i = 0, e = terminator.getNumSuccessors(); i < e; ++i) { |
823 | Block *successor = terminator.getSuccessor(index: i); |
824 | auto branch = cast<BranchOpInterface>(terminator); |
825 | SuccessorOperands successorOperands = branch.getSuccessorOperands(i); |
826 | assert( |
827 | (!seenSuccessors.contains(successor) || successorOperands.empty()) && |
828 | "successors with arguments in LLVM branches must be different blocks"); |
829 | seenSuccessors.insert(Ptr: successor); |
830 | } |
831 | #endif |
832 | |
833 | // For instructions that branch based on a condition value, we need to take |
834 | // the operands for the branch that was taken. |
835 | if (auto condBranchOp = dyn_cast<LLVM::CondBrOp>(terminator)) { |
836 | // For conditional branches, we take the operands from either the "true" or |
837 | // the "false" branch. |
838 | return condBranchOp.getSuccessor(0) == current |
839 | ? condBranchOp.getTrueDestOperands()[index] |
840 | : condBranchOp.getFalseDestOperands()[index]; |
841 | } |
842 | |
843 | if (auto switchOp = dyn_cast<LLVM::SwitchOp>(terminator)) { |
844 | // For switches, we take the operands from either the default case, or from |
845 | // the case branch that was taken. |
846 | if (switchOp.getDefaultDestination() == current) |
847 | return switchOp.getDefaultOperands()[index]; |
848 | for (const auto &i : llvm::enumerate(switchOp.getCaseDestinations())) |
849 | if (i.value() == current) |
850 | return switchOp.getCaseOperands(i.index())[index]; |
851 | } |
852 | |
853 | if (auto indBrOp = dyn_cast<LLVM::IndirectBrOp>(terminator)) { |
854 | // For indirect branches we take operands for each successor. |
855 | for (const auto &i : llvm::enumerate(indBrOp->getSuccessors())) { |
856 | if (indBrOp->getSuccessor(i.index()) == current) |
857 | return indBrOp.getSuccessorOperands(i.index())[index]; |
858 | } |
859 | } |
860 | |
861 | if (auto invokeOp = dyn_cast<LLVM::InvokeOp>(terminator)) { |
862 | return invokeOp.getNormalDest() == current |
863 | ? invokeOp.getNormalDestOperands()[index] |
864 | : invokeOp.getUnwindDestOperands()[index]; |
865 | } |
866 | |
867 | llvm_unreachable( |
868 | "only branch, switch or invoke operations can be terminators " |
869 | "of a block that has successors"); |
870 | } |
871 | |
872 | /// Connect the PHI nodes to the results of preceding blocks. |
873 | void mlir::LLVM::detail::connectPHINodes(Region ®ion, |
874 | const ModuleTranslation &state) { |
875 | // Skip the first block, it cannot be branched to and its arguments correspond |
876 | // to the arguments of the LLVM function. |
877 | for (Block &bb : llvm::drop_begin(RangeOrContainer&: region)) { |
878 | llvm::BasicBlock *llvmBB = state.lookupBlock(block: &bb); |
879 | auto phis = llvmBB->phis(); |
880 | auto numArguments = bb.getNumArguments(); |
881 | assert(numArguments == std::distance(phis.begin(), phis.end())); |
882 | for (auto [index, phiNode] : llvm::enumerate(First&: phis)) { |
883 | for (auto *pred : bb.getPredecessors()) { |
884 | // Find the LLVM IR block that contains the converted terminator |
885 | // instruction and use it in the PHI node. Note that this block is not |
886 | // necessarily the same as state.lookupBlock(pred), some operations |
887 | // (in particular, OpenMP operations using OpenMPIRBuilder) may have |
888 | // split the blocks. |
889 | llvm::Instruction *terminator = |
890 | state.lookupBranch(op: pred->getTerminator()); |
891 | assert(terminator && "missing the mapping for a terminator"); |
892 | phiNode.addIncoming(V: state.lookupValue(value: getPHISourceValue( |
893 | current: &bb, pred, numArguments, index)), |
894 | BB: terminator->getParent()); |
895 | } |
896 | } |
897 | } |
898 | } |
899 | |
900 | llvm::CallInst *mlir::LLVM::detail::createIntrinsicCall( |
901 | llvm::IRBuilderBase &builder, llvm::Intrinsic::ID intrinsic, |
902 | ArrayRef<llvm::Value *> args, ArrayRef<llvm::Type *> tys) { |
903 | llvm::Module *module = builder.GetInsertBlock()->getModule(); |
904 | llvm::Function *fn = |
905 | llvm::Intrinsic::getOrInsertDeclaration(M: module, id: intrinsic, Tys: tys); |
906 | return builder.CreateCall(Callee: fn, Args: args); |
907 | } |
908 | |
909 | llvm::CallInst *mlir::LLVM::detail::createIntrinsicCall( |
910 | llvm::IRBuilderBase &builder, ModuleTranslation &moduleTranslation, |
911 | Operation *intrOp, llvm::Intrinsic::ID intrinsic, unsigned numResults, |
912 | ArrayRef<unsigned> overloadedResults, ArrayRef<unsigned> overloadedOperands, |
913 | ArrayRef<unsigned> immArgPositions, |
914 | ArrayRef<StringLiteral> immArgAttrNames) { |
915 | assert(immArgPositions.size() == immArgAttrNames.size() && |
916 | "LLVM `immArgPositions` and MLIR `immArgAttrNames` should have equal " |
917 | "length"); |
918 | |
919 | SmallVector<llvm::OperandBundleDef> opBundles; |
920 | size_t numOpBundleOperands = 0; |
921 | auto opBundleSizesAttr = cast_if_present<DenseI32ArrayAttr>( |
922 | intrOp->getAttr(LLVMDialect::getOpBundleSizesAttrName())); |
923 | auto opBundleTagsAttr = cast_if_present<ArrayAttr>( |
924 | intrOp->getAttr(LLVMDialect::getOpBundleTagsAttrName())); |
925 | |
926 | if (opBundleSizesAttr && opBundleTagsAttr) { |
927 | ArrayRef<int> opBundleSizes = opBundleSizesAttr.asArrayRef(); |
928 | assert(opBundleSizes.size() == opBundleTagsAttr.size() && |
929 | "operand bundles and tags do not match"); |
930 | |
931 | numOpBundleOperands = |
932 | std::accumulate(first: opBundleSizes.begin(), last: opBundleSizes.end(), init: size_t(0)); |
933 | assert(numOpBundleOperands <= intrOp->getNumOperands() && |
934 | "operand bundle operands is more than the number of operands"); |
935 | |
936 | ValueRange operands = intrOp->getOperands().take_back(n: numOpBundleOperands); |
937 | size_t nextOperandIdx = 0; |
938 | opBundles.reserve(N: opBundleSizesAttr.size()); |
939 | |
940 | for (auto [opBundleTagAttr, bundleSize] : |
941 | llvm::zip(opBundleTagsAttr, opBundleSizes)) { |
942 | auto bundleTag = cast<StringAttr>(opBundleTagAttr).str(); |
943 | auto bundleOperands = moduleTranslation.lookupValues( |
944 | operands.slice(nextOperandIdx, bundleSize)); |
945 | opBundles.emplace_back(std::move(bundleTag), std::move(bundleOperands)); |
946 | nextOperandIdx += bundleSize; |
947 | } |
948 | } |
949 | |
950 | // Map operands and attributes to LLVM values. |
951 | auto opOperands = intrOp->getOperands().drop_back(n: numOpBundleOperands); |
952 | auto operands = moduleTranslation.lookupValues(values: opOperands); |
953 | SmallVector<llvm::Value *> args(immArgPositions.size() + operands.size()); |
954 | for (auto [immArgPos, immArgName] : |
955 | llvm::zip(t&: immArgPositions, u&: immArgAttrNames)) { |
956 | auto attr = llvm::cast<TypedAttr>(intrOp->getAttr(immArgName)); |
957 | assert(attr.getType().isIntOrFloat() && "expected int or float immarg"); |
958 | auto *type = moduleTranslation.convertType(type: attr.getType()); |
959 | args[immArgPos] = LLVM::detail::getLLVMConstant( |
960 | llvmType: type, attr: attr, loc: intrOp->getLoc(), moduleTranslation); |
961 | } |
962 | unsigned opArg = 0; |
963 | for (auto &arg : args) { |
964 | if (!arg) |
965 | arg = operands[opArg++]; |
966 | } |
967 | |
968 | // Resolve overloaded intrinsic declaration. |
969 | SmallVector<llvm::Type *> overloadedTypes; |
970 | for (unsigned overloadedResultIdx : overloadedResults) { |
971 | if (numResults > 1) { |
972 | // More than one result is mapped to an LLVM struct. |
973 | overloadedTypes.push_back(moduleTranslation.convertType( |
974 | llvm::cast<LLVM::LLVMStructType>(intrOp->getResult(0).getType()) |
975 | .getBody()[overloadedResultIdx])); |
976 | } else { |
977 | overloadedTypes.push_back( |
978 | Elt: moduleTranslation.convertType(type: intrOp->getResult(idx: 0).getType())); |
979 | } |
980 | } |
981 | for (unsigned overloadedOperandIdx : overloadedOperands) |
982 | overloadedTypes.push_back(Elt: args[overloadedOperandIdx]->getType()); |
983 | llvm::Module *module = builder.GetInsertBlock()->getModule(); |
984 | llvm::Function *llvmIntr = llvm::Intrinsic::getOrInsertDeclaration( |
985 | M: module, id: intrinsic, Tys: overloadedTypes); |
986 | |
987 | return builder.CreateCall(Callee: llvmIntr, Args: args, OpBundles: opBundles); |
988 | } |
989 | |
990 | /// Given a single MLIR operation, create the corresponding LLVM IR operation |
991 | /// using the `builder`. |
992 | LogicalResult ModuleTranslation::convertOperation(Operation &op, |
993 | llvm::IRBuilderBase &builder, |
994 | bool recordInsertions) { |
995 | const LLVMTranslationDialectInterface *opIface = iface.getInterfaceFor(obj: &op); |
996 | if (!opIface) |
997 | return op.emitError(message: "cannot be converted to LLVM IR: missing " |
998 | "`LLVMTranslationDialectInterface` registration for " |
999 | "dialect for op: ") |
1000 | << op.getName(); |
1001 | |
1002 | InstructionCapturingInserter::CollectionScope scope(builder, |
1003 | recordInsertions); |
1004 | if (failed(Result: opIface->convertOperation(op: &op, builder, moduleTranslation&: *this))) |
1005 | return op.emitError(message: "LLVM Translation failed for operation: ") |
1006 | << op.getName(); |
1007 | |
1008 | return convertDialectAttributes(op: &op, instructions: scope.getCapturedInstructions()); |
1009 | } |
1010 | |
1011 | /// Convert block to LLVM IR. Unless `ignoreArguments` is set, emit PHI nodes |
1012 | /// to define values corresponding to the MLIR block arguments. These nodes |
1013 | /// are not connected to the source basic blocks, which may not exist yet. Uses |
1014 | /// `builder` to construct the LLVM IR. Expects the LLVM IR basic block to have |
1015 | /// been created for `bb` and included in the block mapping. Inserts new |
1016 | /// instructions at the end of the block and leaves `builder` in a state |
1017 | /// suitable for further insertion into the end of the block. |
1018 | LogicalResult ModuleTranslation::convertBlockImpl(Block &bb, |
1019 | bool ignoreArguments, |
1020 | llvm::IRBuilderBase &builder, |
1021 | bool recordInsertions) { |
1022 | builder.SetInsertPoint(lookupBlock(block: &bb)); |
1023 | auto *subprogram = builder.GetInsertBlock()->getParent()->getSubprogram(); |
1024 | |
1025 | // Before traversing operations, make block arguments available through |
1026 | // value remapping and PHI nodes, but do not add incoming edges for the PHI |
1027 | // nodes just yet: those values may be defined by this or following blocks. |
1028 | // This step is omitted if "ignoreArguments" is set. The arguments of the |
1029 | // first block have been already made available through the remapping of |
1030 | // LLVM function arguments. |
1031 | if (!ignoreArguments) { |
1032 | auto predecessors = bb.getPredecessors(); |
1033 | unsigned numPredecessors = |
1034 | std::distance(first: predecessors.begin(), last: predecessors.end()); |
1035 | for (auto arg : bb.getArguments()) { |
1036 | auto wrappedType = arg.getType(); |
1037 | if (!isCompatibleType(type: wrappedType)) |
1038 | return emitError(loc: bb.front().getLoc(), |
1039 | message: "block argument does not have an LLVM type"); |
1040 | builder.SetCurrentDebugLocation( |
1041 | debugTranslation->translateLoc(loc: arg.getLoc(), scope: subprogram)); |
1042 | llvm::Type *type = convertType(type: wrappedType); |
1043 | llvm::PHINode *phi = builder.CreatePHI(Ty: type, NumReservedValues: numPredecessors); |
1044 | mapValue(mlir: arg, llvm: phi); |
1045 | } |
1046 | } |
1047 | |
1048 | // Traverse operations. |
1049 | for (auto &op : bb) { |
1050 | // Set the current debug location within the builder. |
1051 | builder.SetCurrentDebugLocation( |
1052 | debugTranslation->translateLoc(loc: op.getLoc(), scope: subprogram)); |
1053 | |
1054 | if (failed(Result: convertOperation(op, builder, recordInsertions))) |
1055 | return failure(); |
1056 | |
1057 | // Set the branch weight metadata on the translated instruction. |
1058 | if (auto iface = dyn_cast<BranchWeightOpInterface>(op)) |
1059 | setBranchWeightsMetadata(iface); |
1060 | } |
1061 | |
1062 | return success(); |
1063 | } |
1064 | |
1065 | /// A helper method to get the single Block in an operation honoring LLVM's |
1066 | /// module requirements. |
1067 | static Block &getModuleBody(Operation *module) { |
1068 | return module->getRegion(index: 0).front(); |
1069 | } |
1070 | |
1071 | /// A helper method to decide if a constant must not be set as a global variable |
1072 | /// initializer. For an external linkage variable, the variable with an |
1073 | /// initializer is considered externally visible and defined in this module, the |
1074 | /// variable without an initializer is externally available and is defined |
1075 | /// elsewhere. |
1076 | static bool shouldDropGlobalInitializer(llvm::GlobalValue::LinkageTypes linkage, |
1077 | llvm::Constant *cst) { |
1078 | return (linkage == llvm::GlobalVariable::ExternalLinkage && !cst) || |
1079 | linkage == llvm::GlobalVariable::ExternalWeakLinkage; |
1080 | } |
1081 | |
1082 | /// Sets the runtime preemption specifier of `gv` to dso_local if |
1083 | /// `dsoLocalRequested` is true, otherwise it is left unchanged. |
1084 | static void addRuntimePreemptionSpecifier(bool dsoLocalRequested, |
1085 | llvm::GlobalValue *gv) { |
1086 | if (dsoLocalRequested) |
1087 | gv->setDSOLocal(true); |
1088 | } |
1089 | |
1090 | LogicalResult ModuleTranslation::convertGlobalsAndAliases() { |
1091 | // Mapping from compile unit to its respective set of global variables. |
1092 | DenseMap<llvm::DICompileUnit *, SmallVector<llvm::Metadata *>> allGVars; |
1093 | |
1094 | // First, create all global variables and global aliases in LLVM IR. A global |
1095 | // or alias body may refer to another global/alias or itself, so all the |
1096 | // mapping needs to happen prior to body conversion. |
1097 | |
1098 | // Create all llvm::GlobalVariable |
1099 | for (auto op : getModuleBody(mlirModule).getOps<LLVM::GlobalOp>()) { |
1100 | llvm::Type *type = convertType(op.getType()); |
1101 | llvm::Constant *cst = nullptr; |
1102 | if (op.getValueOrNull()) { |
1103 | // String attributes are treated separately because they cannot appear as |
1104 | // in-function constants and are thus not supported by getLLVMConstant. |
1105 | if (auto strAttr = dyn_cast_or_null<StringAttr>(op.getValueOrNull())) { |
1106 | cst = llvm::ConstantDataArray::getString( |
1107 | llvmModule->getContext(), strAttr.getValue(), /*AddNull=*/false); |
1108 | type = cst->getType(); |
1109 | } else if (!(cst = getLLVMConstant(type, op.getValueOrNull(), op.getLoc(), |
1110 | *this))) { |
1111 | return failure(); |
1112 | } |
1113 | } |
1114 | |
1115 | auto linkage = convertLinkageToLLVM(op.getLinkage()); |
1116 | |
1117 | // LLVM IR requires constant with linkage other than external or weak |
1118 | // external to have initializers. If MLIR does not provide an initializer, |
1119 | // default to undef. |
1120 | bool dropInitializer = shouldDropGlobalInitializer(linkage, cst); |
1121 | if (!dropInitializer && !cst) |
1122 | cst = llvm::UndefValue::get(type); |
1123 | else if (dropInitializer && cst) |
1124 | cst = nullptr; |
1125 | |
1126 | auto *var = new llvm::GlobalVariable( |
1127 | *llvmModule, type, op.getConstant(), linkage, cst, op.getSymName(), |
1128 | /*InsertBefore=*/nullptr, |
1129 | op.getThreadLocal_() ? llvm::GlobalValue::GeneralDynamicTLSModel |
1130 | : llvm::GlobalValue::NotThreadLocal, |
1131 | op.getAddrSpace(), op.getExternallyInitialized()); |
1132 | |
1133 | if (std::optional<mlir::SymbolRefAttr> comdat = op.getComdat()) { |
1134 | auto selectorOp = cast<ComdatSelectorOp>( |
1135 | SymbolTable::lookupNearestSymbolFrom(op, *comdat)); |
1136 | var->setComdat(comdatMapping.lookup(selectorOp)); |
1137 | } |
1138 | |
1139 | if (op.getUnnamedAddr().has_value()) |
1140 | var->setUnnamedAddr(convertUnnamedAddrToLLVM(*op.getUnnamedAddr())); |
1141 | |
1142 | if (op.getSection().has_value()) |
1143 | var->setSection(*op.getSection()); |
1144 | |
1145 | addRuntimePreemptionSpecifier(op.getDsoLocal(), var); |
1146 | |
1147 | std::optional<uint64_t> alignment = op.getAlignment(); |
1148 | if (alignment.has_value()) |
1149 | var->setAlignment(llvm::MaybeAlign(alignment.value())); |
1150 | |
1151 | var->setVisibility(convertVisibilityToLLVM(op.getVisibility_())); |
1152 | |
1153 | globalsMapping.try_emplace(op, var); |
1154 | |
1155 | // Add debug information if present. |
1156 | if (op.getDbgExprs()) { |
1157 | for (auto exprAttr : |
1158 | op.getDbgExprs()->getAsRange<DIGlobalVariableExpressionAttr>()) { |
1159 | llvm::DIGlobalVariableExpression *diGlobalExpr = |
1160 | debugTranslation->translateGlobalVariableExpression(exprAttr); |
1161 | llvm::DIGlobalVariable *diGlobalVar = diGlobalExpr->getVariable(); |
1162 | var->addDebugInfo(diGlobalExpr); |
1163 | |
1164 | // There is no `globals` field in DICompileUnitAttr which can be |
1165 | // directly assigned to DICompileUnit. We have to build the list by |
1166 | // looking at the dbgExpr of all the GlobalOps. The scope of the |
1167 | // variable is used to get the DICompileUnit in which to add it. But |
1168 | // there are cases where the scope of a global does not directly point |
1169 | // to the DICompileUnit and we have to do a bit more work to get to |
1170 | // it. Some of those cases are: |
1171 | // |
1172 | // 1. For the languages that support modules, the scope hierarchy can |
1173 | // be variable -> DIModule -> DICompileUnit |
1174 | // |
1175 | // 2. For the Fortran common block variable, the scope hierarchy can |
1176 | // be variable -> DICommonBlock -> DISubprogram -> DICompileUnit |
1177 | // |
1178 | // 3. For entities like static local variables in C or variable with |
1179 | // SAVE attribute in Fortran, the scope hierarchy can be |
1180 | // variable -> DISubprogram -> DICompileUnit |
1181 | llvm::DIScope *scope = diGlobalVar->getScope(); |
1182 | if (auto *mod = dyn_cast_if_present<llvm::DIModule>(scope)) |
1183 | scope = mod->getScope(); |
1184 | else if (auto *cb = dyn_cast_if_present<llvm::DICommonBlock>(scope)) { |
1185 | if (auto *sp = |
1186 | dyn_cast_if_present<llvm::DISubprogram>(cb->getScope())) |
1187 | scope = sp->getUnit(); |
1188 | } else if (auto *sp = dyn_cast_if_present<llvm::DISubprogram>(scope)) |
1189 | scope = sp->getUnit(); |
1190 | |
1191 | // Get the compile unit (scope) of the the global variable. |
1192 | if (llvm::DICompileUnit *compileUnit = |
1193 | dyn_cast_if_present<llvm::DICompileUnit>(scope)) { |
1194 | // Update the compile unit with this incoming global variable |
1195 | // expression during the finalizing step later. |
1196 | allGVars[compileUnit].push_back(diGlobalExpr); |
1197 | } |
1198 | } |
1199 | } |
1200 | } |
1201 | |
1202 | // Create all llvm::GlobalAlias |
1203 | for (auto op : getModuleBody(mlirModule).getOps<LLVM::AliasOp>()) { |
1204 | llvm::Type *type = convertType(op.getType()); |
1205 | llvm::Constant *cst = nullptr; |
1206 | llvm::GlobalValue::LinkageTypes linkage = |
1207 | convertLinkageToLLVM(op.getLinkage()); |
1208 | llvm::Module &llvmMod = *llvmModule; |
1209 | |
1210 | // Note address space and aliasee info isn't set just yet. |
1211 | llvm::GlobalAlias *var = llvm::GlobalAlias::create( |
1212 | type, op.getAddrSpace(), linkage, op.getSymName(), /*placeholder*/ cst, |
1213 | &llvmMod); |
1214 | |
1215 | var->setThreadLocalMode(op.getThreadLocal_() |
1216 | ? llvm::GlobalAlias::GeneralDynamicTLSModel |
1217 | : llvm::GlobalAlias::NotThreadLocal); |
1218 | |
1219 | // Note there is no need to setup the comdat because GlobalAlias calls into |
1220 | // the aliasee comdat information automatically. |
1221 | |
1222 | if (op.getUnnamedAddr().has_value()) |
1223 | var->setUnnamedAddr(convertUnnamedAddrToLLVM(*op.getUnnamedAddr())); |
1224 | |
1225 | var->setVisibility(convertVisibilityToLLVM(op.getVisibility_())); |
1226 | |
1227 | aliasesMapping.try_emplace(op, var); |
1228 | } |
1229 | |
1230 | // Convert global variable bodies. |
1231 | for (auto op : getModuleBody(mlirModule).getOps<LLVM::GlobalOp>()) { |
1232 | if (Block *initializer = op.getInitializerBlock()) { |
1233 | llvm::IRBuilder<llvm::TargetFolder> builder( |
1234 | llvmModule->getContext(), |
1235 | llvm::TargetFolder(llvmModule->getDataLayout())); |
1236 | |
1237 | [[maybe_unused]] int numConstantsHit = 0; |
1238 | [[maybe_unused]] int numConstantsErased = 0; |
1239 | DenseMap<llvm::ConstantAggregate *, int> constantAggregateUseMap; |
1240 | |
1241 | for (auto &op : initializer->without_terminator()) { |
1242 | if (failed(convertOperation(op, builder))) |
1243 | return emitError(op.getLoc(), "fail to convert global initializer"); |
1244 | auto *cst = dyn_cast<llvm::Constant>(lookupValue(op.getResult(0))); |
1245 | if (!cst) |
1246 | return emitError(op.getLoc(), "unemittable constant value"); |
1247 | |
1248 | // When emitting an LLVM constant, a new constant is created and the old |
1249 | // constant may become dangling and take space. We should remove the |
1250 | // dangling constants to avoid memory explosion especially for constant |
1251 | // arrays whose number of elements is large. |
1252 | // Because multiple operations may refer to the same constant, we need |
1253 | // to count the number of uses of each constant array and remove it only |
1254 | // when the count becomes zero. |
1255 | if (auto *agg = dyn_cast<llvm::ConstantAggregate>(cst)) { |
1256 | numConstantsHit++; |
1257 | Value result = op.getResult(0); |
1258 | int numUsers = std::distance(result.use_begin(), result.use_end()); |
1259 | auto [iterator, inserted] = |
1260 | constantAggregateUseMap.try_emplace(agg, numUsers); |
1261 | if (!inserted) { |
1262 | // Key already exists, update the value |
1263 | iterator->second += numUsers; |
1264 | } |
1265 | } |
1266 | // Scan the operands of the operation to decrement the use count of |
1267 | // constants. Erase the constant if the use count becomes zero. |
1268 | for (Value v : op.getOperands()) { |
1269 | auto cst = dyn_cast<llvm::ConstantAggregate>(lookupValue(v)); |
1270 | if (!cst) |
1271 | continue; |
1272 | auto iter = constantAggregateUseMap.find(cst); |
1273 | assert(iter != constantAggregateUseMap.end() && "constant not found"); |
1274 | iter->second--; |
1275 | if (iter->second == 0) { |
1276 | // NOTE: cannot call removeDeadConstantUsers() here because it |
1277 | // may remove the constant which has uses not be converted yet. |
1278 | if (cst->user_empty()) { |
1279 | cst->destroyConstant(); |
1280 | numConstantsErased++; |
1281 | } |
1282 | constantAggregateUseMap.erase(iter); |
1283 | } |
1284 | } |
1285 | } |
1286 | |
1287 | ReturnOp ret = cast<ReturnOp>(initializer->getTerminator()); |
1288 | llvm::Constant *cst = |
1289 | cast<llvm::Constant>(lookupValue(ret.getOperand(0))); |
1290 | auto *global = cast<llvm::GlobalVariable>(lookupGlobal(op)); |
1291 | if (!shouldDropGlobalInitializer(global->getLinkage(), cst)) |
1292 | global->setInitializer(cst); |
1293 | |
1294 | // Try to remove the dangling constants again after all operations are |
1295 | // converted. |
1296 | for (auto it : constantAggregateUseMap) { |
1297 | auto cst = it.first; |
1298 | cst->removeDeadConstantUsers(); |
1299 | if (cst->user_empty()) { |
1300 | cst->destroyConstant(); |
1301 | numConstantsErased++; |
1302 | } |
1303 | } |
1304 | |
1305 | LLVM_DEBUG(llvm::dbgs() |
1306 | << "Convert initializer for "<< op.getName() << "\n"; |
1307 | llvm::dbgs() << numConstantsHit << " new constants hit\n"; |
1308 | llvm::dbgs() |
1309 | << numConstantsErased << " dangling constants erased\n";); |
1310 | } |
1311 | } |
1312 | |
1313 | // Convert llvm.mlir.global_ctors and dtors. |
1314 | for (Operation &op : getModuleBody(module: mlirModule)) { |
1315 | auto ctorOp = dyn_cast<GlobalCtorsOp>(op); |
1316 | auto dtorOp = dyn_cast<GlobalDtorsOp>(op); |
1317 | if (!ctorOp && !dtorOp) |
1318 | continue; |
1319 | |
1320 | // The empty / zero initialized version of llvm.global_(c|d)tors cannot be |
1321 | // handled by appendGlobalFn logic below, which just ignores empty (c|d)tor |
1322 | // lists. Make sure it gets emitted. |
1323 | if ((ctorOp && ctorOp.getCtors().empty()) || |
1324 | (dtorOp && dtorOp.getDtors().empty())) { |
1325 | llvm::IRBuilder<llvm::TargetFolder> builder( |
1326 | llvmModule->getContext(), |
1327 | llvm::TargetFolder(llvmModule->getDataLayout())); |
1328 | llvm::Type *eltTy = llvm::StructType::get( |
1329 | elt1: builder.getInt32Ty(), elts: builder.getPtrTy(), elts: builder.getPtrTy()); |
1330 | llvm::ArrayType *at = llvm::ArrayType::get(ElementType: eltTy, NumElements: 0); |
1331 | llvm::Constant *zeroInit = llvm::Constant::getNullValue(Ty: at); |
1332 | (void)new llvm::GlobalVariable( |
1333 | *llvmModule, zeroInit->getType(), false, |
1334 | llvm::GlobalValue::AppendingLinkage, zeroInit, |
1335 | ctorOp ? "llvm.global_ctors": "llvm.global_dtors"); |
1336 | } else { |
1337 | auto range = ctorOp |
1338 | ? llvm::zip(ctorOp.getCtors(), ctorOp.getPriorities()) |
1339 | : llvm::zip(dtorOp.getDtors(), dtorOp.getPriorities()); |
1340 | auto appendGlobalFn = |
1341 | ctorOp ? llvm::appendToGlobalCtors : llvm::appendToGlobalDtors; |
1342 | for (const auto &[sym, prio] : range) { |
1343 | llvm::Function *f = |
1344 | lookupFunction(cast<FlatSymbolRefAttr>(sym).getValue()); |
1345 | appendGlobalFn(*llvmModule, f, cast<IntegerAttr>(prio).getInt(), |
1346 | /*Data=*/nullptr); |
1347 | } |
1348 | } |
1349 | } |
1350 | |
1351 | for (auto op : getModuleBody(mlirModule).getOps<LLVM::GlobalOp>()) |
1352 | if (failed(convertDialectAttributes(op, {}))) |
1353 | return failure(); |
1354 | |
1355 | // Finally, update the compile units their respective sets of global variables |
1356 | // created earlier. |
1357 | for (const auto &[compileUnit, globals] : allGVars) { |
1358 | compileUnit->replaceGlobalVariables( |
1359 | N: llvm::MDTuple::get(Context&: getLLVMContext(), MDs: globals)); |
1360 | } |
1361 | |
1362 | // Convert global alias bodies. |
1363 | for (auto op : getModuleBody(mlirModule).getOps<LLVM::AliasOp>()) { |
1364 | Block &initializer = op.getInitializerBlock(); |
1365 | llvm::IRBuilder<llvm::TargetFolder> builder( |
1366 | llvmModule->getContext(), |
1367 | llvm::TargetFolder(llvmModule->getDataLayout())); |
1368 | |
1369 | for (mlir::Operation &op : initializer.without_terminator()) { |
1370 | if (failed(convertOperation(op, builder))) |
1371 | return emitError(op.getLoc(), "fail to convert alias initializer"); |
1372 | if (!isa<llvm::Constant>(lookupValue(op.getResult(0)))) |
1373 | return emitError(op.getLoc(), "unemittable constant value"); |
1374 | } |
1375 | |
1376 | auto ret = cast<ReturnOp>(initializer.getTerminator()); |
1377 | auto *cst = cast<llvm::Constant>(lookupValue(ret.getOperand(0))); |
1378 | assert(aliasesMapping.count(op)); |
1379 | auto *alias = cast<llvm::GlobalAlias>(aliasesMapping[op]); |
1380 | alias->setAliasee(cst); |
1381 | } |
1382 | |
1383 | for (auto op : getModuleBody(mlirModule).getOps<LLVM::AliasOp>()) |
1384 | if (failed(convertDialectAttributes(op, {}))) |
1385 | return failure(); |
1386 | |
1387 | return success(); |
1388 | } |
1389 | |
1390 | /// Attempts to add an attribute identified by `key`, optionally with the given |
1391 | /// `value` to LLVM function `llvmFunc`. Reports errors at `loc` if any. If the |
1392 | /// attribute has a kind known to LLVM IR, create the attribute of this kind, |
1393 | /// otherwise keep it as a string attribute. Performs additional checks for |
1394 | /// attributes known to have or not have a value in order to avoid assertions |
1395 | /// inside LLVM upon construction. |
1396 | static LogicalResult checkedAddLLVMFnAttribute(Location loc, |
1397 | llvm::Function *llvmFunc, |
1398 | StringRef key, |
1399 | StringRef value = StringRef()) { |
1400 | auto kind = llvm::Attribute::getAttrKindFromName(AttrName: key); |
1401 | if (kind == llvm::Attribute::None) { |
1402 | llvmFunc->addFnAttr(Kind: key, Val: value); |
1403 | return success(); |
1404 | } |
1405 | |
1406 | if (llvm::Attribute::isIntAttrKind(Kind: kind)) { |
1407 | if (value.empty()) |
1408 | return emitError(loc) << "LLVM attribute '"<< key << "' expects a value"; |
1409 | |
1410 | int64_t result; |
1411 | if (!value.getAsInteger(/*Radix=*/0, Result&: result)) |
1412 | llvmFunc->addFnAttr( |
1413 | Attr: llvm::Attribute::get(Context&: llvmFunc->getContext(), Kind: kind, Val: result)); |
1414 | else |
1415 | llvmFunc->addFnAttr(Kind: key, Val: value); |
1416 | return success(); |
1417 | } |
1418 | |
1419 | if (!value.empty()) |
1420 | return emitError(loc) << "LLVM attribute '"<< key |
1421 | << "' does not expect a value, found '"<< value |
1422 | << "'"; |
1423 | |
1424 | llvmFunc->addFnAttr(Kind: kind); |
1425 | return success(); |
1426 | } |
1427 | |
1428 | /// Return a representation of `value` as metadata. |
1429 | static llvm::Metadata *convertIntegerToMetadata(llvm::LLVMContext &context, |
1430 | const llvm::APInt &value) { |
1431 | llvm::Constant *constant = llvm::ConstantInt::get(Context&: context, V: value); |
1432 | return llvm::ConstantAsMetadata::get(C: constant); |
1433 | } |
1434 | |
1435 | /// Return a representation of `value` as an MDNode. |
1436 | static llvm::MDNode *convertIntegerToMDNode(llvm::LLVMContext &context, |
1437 | const llvm::APInt &value) { |
1438 | return llvm::MDNode::get(Context&: context, MDs: convertIntegerToMetadata(context, value)); |
1439 | } |
1440 | |
1441 | /// Return an MDNode encoding `vec_type_hint` metadata. |
1442 | static llvm::MDNode *convertVecTypeHintToMDNode(llvm::LLVMContext &context, |
1443 | llvm::Type *type, |
1444 | bool isSigned) { |
1445 | llvm::Metadata *typeMD = |
1446 | llvm::ConstantAsMetadata::get(C: llvm::UndefValue::get(T: type)); |
1447 | llvm::Metadata *isSignedMD = |
1448 | convertIntegerToMetadata(context, value: llvm::APInt(32, isSigned ? 1 : 0)); |
1449 | return llvm::MDNode::get(Context&: context, MDs: {typeMD, isSignedMD}); |
1450 | } |
1451 | |
1452 | /// Return an MDNode with a tuple given by the values in `values`. |
1453 | static llvm::MDNode *convertIntegerArrayToMDNode(llvm::LLVMContext &context, |
1454 | ArrayRef<int32_t> values) { |
1455 | SmallVector<llvm::Metadata *> mdValues; |
1456 | llvm::transform( |
1457 | Range&: values, d_first: std::back_inserter(x&: mdValues), F: [&context](int32_t value) { |
1458 | return convertIntegerToMetadata(context, value: llvm::APInt(32, value)); |
1459 | }); |
1460 | return llvm::MDNode::get(Context&: context, MDs: mdValues); |
1461 | } |
1462 | |
1463 | /// Attaches the attributes listed in the given array attribute to `llvmFunc`. |
1464 | /// Reports error to `loc` if any and returns immediately. Expects `attributes` |
1465 | /// to be an array attribute containing either string attributes, treated as |
1466 | /// value-less LLVM attributes, or array attributes containing two string |
1467 | /// attributes, with the first string being the name of the corresponding LLVM |
1468 | /// attribute and the second string beings its value. Note that even integer |
1469 | /// attributes are expected to have their values expressed as strings. |
1470 | static LogicalResult |
1471 | forwardPassthroughAttributes(Location loc, std::optional<ArrayAttr> attributes, |
1472 | llvm::Function *llvmFunc) { |
1473 | if (!attributes) |
1474 | return success(); |
1475 | |
1476 | for (Attribute attr : *attributes) { |
1477 | if (auto stringAttr = dyn_cast<StringAttr>(attr)) { |
1478 | if (failed( |
1479 | checkedAddLLVMFnAttribute(loc, llvmFunc, stringAttr.getValue()))) |
1480 | return failure(); |
1481 | continue; |
1482 | } |
1483 | |
1484 | auto arrayAttr = dyn_cast<ArrayAttr>(attr); |
1485 | if (!arrayAttr || arrayAttr.size() != 2) |
1486 | return emitError(loc) |
1487 | << "expected 'passthrough' to contain string or array attributes"; |
1488 | |
1489 | auto keyAttr = dyn_cast<StringAttr>(arrayAttr[0]); |
1490 | auto valueAttr = dyn_cast<StringAttr>(arrayAttr[1]); |
1491 | if (!keyAttr || !valueAttr) |
1492 | return emitError(loc) |
1493 | << "expected arrays within 'passthrough' to contain two strings"; |
1494 | |
1495 | if (failed(checkedAddLLVMFnAttribute(loc, llvmFunc, keyAttr.getValue(), |
1496 | valueAttr.getValue()))) |
1497 | return failure(); |
1498 | } |
1499 | return success(); |
1500 | } |
1501 | |
1502 | LogicalResult ModuleTranslation::convertOneFunction(LLVMFuncOp func) { |
1503 | // Clear the block, branch value mappings, they are only relevant within one |
1504 | // function. |
1505 | blockMapping.clear(); |
1506 | valueMapping.clear(); |
1507 | branchMapping.clear(); |
1508 | llvm::Function *llvmFunc = lookupFunction(name: func.getName()); |
1509 | |
1510 | // Add function arguments to the value remapping table. |
1511 | for (auto [mlirArg, llvmArg] : |
1512 | llvm::zip(func.getArguments(), llvmFunc->args())) |
1513 | mapValue(mlirArg, &llvmArg); |
1514 | |
1515 | // Check the personality and set it. |
1516 | if (func.getPersonality()) { |
1517 | llvm::Type *ty = llvm::PointerType::getUnqual(C&: llvmFunc->getContext()); |
1518 | if (llvm::Constant *pfunc = getLLVMConstant(ty, func.getPersonalityAttr(), |
1519 | func.getLoc(), *this)) |
1520 | llvmFunc->setPersonalityFn(pfunc); |
1521 | } |
1522 | |
1523 | if (std::optional<StringRef> section = func.getSection()) |
1524 | llvmFunc->setSection(*section); |
1525 | |
1526 | if (func.getArmStreaming()) |
1527 | llvmFunc->addFnAttr(Kind: "aarch64_pstate_sm_enabled"); |
1528 | else if (func.getArmLocallyStreaming()) |
1529 | llvmFunc->addFnAttr(Kind: "aarch64_pstate_sm_body"); |
1530 | else if (func.getArmStreamingCompatible()) |
1531 | llvmFunc->addFnAttr(Kind: "aarch64_pstate_sm_compatible"); |
1532 | |
1533 | if (func.getArmNewZa()) |
1534 | llvmFunc->addFnAttr(Kind: "aarch64_new_za"); |
1535 | else if (func.getArmInZa()) |
1536 | llvmFunc->addFnAttr(Kind: "aarch64_in_za"); |
1537 | else if (func.getArmOutZa()) |
1538 | llvmFunc->addFnAttr(Kind: "aarch64_out_za"); |
1539 | else if (func.getArmInoutZa()) |
1540 | llvmFunc->addFnAttr(Kind: "aarch64_inout_za"); |
1541 | else if (func.getArmPreservesZa()) |
1542 | llvmFunc->addFnAttr(Kind: "aarch64_preserves_za"); |
1543 | |
1544 | if (auto targetCpu = func.getTargetCpu()) |
1545 | llvmFunc->addFnAttr("target-cpu", *targetCpu); |
1546 | |
1547 | if (auto tuneCpu = func.getTuneCpu()) |
1548 | llvmFunc->addFnAttr("tune-cpu", *tuneCpu); |
1549 | |
1550 | if (auto reciprocalEstimates = func.getReciprocalEstimates()) |
1551 | llvmFunc->addFnAttr("reciprocal-estimates", *reciprocalEstimates); |
1552 | |
1553 | if (auto preferVectorWidth = func.getPreferVectorWidth()) |
1554 | llvmFunc->addFnAttr("prefer-vector-width", *preferVectorWidth); |
1555 | |
1556 | if (auto attr = func.getVscaleRange()) |
1557 | llvmFunc->addFnAttr(llvm::Attribute::getWithVScaleRangeArgs( |
1558 | Context&: getLLVMContext(), MinValue: attr->getMinRange().getInt(), |
1559 | MaxValue: attr->getMaxRange().getInt())); |
1560 | |
1561 | if (auto unsafeFpMath = func.getUnsafeFpMath()) |
1562 | llvmFunc->addFnAttr("unsafe-fp-math", llvm::toStringRef(*unsafeFpMath)); |
1563 | |
1564 | if (auto noInfsFpMath = func.getNoInfsFpMath()) |
1565 | llvmFunc->addFnAttr("no-infs-fp-math", llvm::toStringRef(*noInfsFpMath)); |
1566 | |
1567 | if (auto noNansFpMath = func.getNoNansFpMath()) |
1568 | llvmFunc->addFnAttr("no-nans-fp-math", llvm::toStringRef(*noNansFpMath)); |
1569 | |
1570 | if (auto approxFuncFpMath = func.getApproxFuncFpMath()) |
1571 | llvmFunc->addFnAttr("approx-func-fp-math", |
1572 | llvm::toStringRef(*approxFuncFpMath)); |
1573 | |
1574 | if (auto noSignedZerosFpMath = func.getNoSignedZerosFpMath()) |
1575 | llvmFunc->addFnAttr("no-signed-zeros-fp-math", |
1576 | llvm::toStringRef(*noSignedZerosFpMath)); |
1577 | |
1578 | if (auto denormalFpMath = func.getDenormalFpMath()) |
1579 | llvmFunc->addFnAttr("denormal-fp-math", *denormalFpMath); |
1580 | |
1581 | if (auto denormalFpMathF32 = func.getDenormalFpMathF32()) |
1582 | llvmFunc->addFnAttr("denormal-fp-math-f32", *denormalFpMathF32); |
1583 | |
1584 | if (auto fpContract = func.getFpContract()) |
1585 | llvmFunc->addFnAttr("fp-contract", *fpContract); |
1586 | |
1587 | if (auto instrumentFunctionEntry = func.getInstrumentFunctionEntry()) |
1588 | llvmFunc->addFnAttr("instrument-function-entry", *instrumentFunctionEntry); |
1589 | |
1590 | if (auto instrumentFunctionExit = func.getInstrumentFunctionExit()) |
1591 | llvmFunc->addFnAttr("instrument-function-exit", *instrumentFunctionExit); |
1592 | |
1593 | // First, create all blocks so we can jump to them. |
1594 | llvm::LLVMContext &llvmContext = llvmFunc->getContext(); |
1595 | for (auto &bb : func) { |
1596 | auto *llvmBB = llvm::BasicBlock::Create(llvmContext); |
1597 | llvmBB->insertInto(llvmFunc); |
1598 | mapBlock(&bb, llvmBB); |
1599 | } |
1600 | |
1601 | // Then, convert blocks one by one in topological order to ensure defs are |
1602 | // converted before uses. |
1603 | auto blocks = getBlocksSortedByDominance(func.getBody()); |
1604 | for (Block *bb : blocks) { |
1605 | CapturingIRBuilder builder(llvmContext, |
1606 | llvm::TargetFolder(llvmModule->getDataLayout())); |
1607 | if (failed(convertBlockImpl(*bb, bb->isEntryBlock(), builder, |
1608 | /*recordInsertions=*/true))) |
1609 | return failure(); |
1610 | } |
1611 | |
1612 | // After all blocks have been traversed and values mapped, connect the PHI |
1613 | // nodes to the results of preceding blocks. |
1614 | detail::connectPHINodes(region&: func.getBody(), state: *this); |
1615 | |
1616 | // Finally, convert dialect attributes attached to the function. |
1617 | return convertDialectAttributes(op: func, instructions: {}); |
1618 | } |
1619 | |
1620 | LogicalResult ModuleTranslation::convertDialectAttributes( |
1621 | Operation *op, ArrayRef<llvm::Instruction *> instructions) { |
1622 | for (NamedAttribute attribute : op->getDialectAttrs()) |
1623 | if (failed(Result: iface.amendOperation(op, instructions, attribute, moduleTranslation&: *this))) |
1624 | return failure(); |
1625 | return success(); |
1626 | } |
1627 | |
1628 | /// Converts memory effect attributes from `func` and attaches them to |
1629 | /// `llvmFunc`. |
1630 | static void convertFunctionMemoryAttributes(LLVMFuncOp func, |
1631 | llvm::Function *llvmFunc) { |
1632 | if (!func.getMemoryEffects()) |
1633 | return; |
1634 | |
1635 | MemoryEffectsAttr memEffects = func.getMemoryEffectsAttr(); |
1636 | |
1637 | // Add memory effects incrementally. |
1638 | llvm::MemoryEffects newMemEffects = |
1639 | llvm::MemoryEffects(llvm::MemoryEffects::Location::ArgMem, |
1640 | convertModRefInfoToLLVM(memEffects.getArgMem())); |
1641 | newMemEffects |= llvm::MemoryEffects( |
1642 | llvm::MemoryEffects::Location::InaccessibleMem, |
1643 | convertModRefInfoToLLVM(memEffects.getInaccessibleMem())); |
1644 | newMemEffects |= |
1645 | llvm::MemoryEffects(llvm::MemoryEffects::Location::Other, |
1646 | convertModRefInfoToLLVM(memEffects.getOther())); |
1647 | llvmFunc->setMemoryEffects(newMemEffects); |
1648 | } |
1649 | |
1650 | /// Converts function attributes from `func` and attaches them to `llvmFunc`. |
1651 | static void convertFunctionAttributes(LLVMFuncOp func, |
1652 | llvm::Function *llvmFunc) { |
1653 | if (func.getNoInlineAttr()) |
1654 | llvmFunc->addFnAttr(llvm::Attribute::NoInline); |
1655 | if (func.getAlwaysInlineAttr()) |
1656 | llvmFunc->addFnAttr(llvm::Attribute::AlwaysInline); |
1657 | if (func.getOptimizeNoneAttr()) |
1658 | llvmFunc->addFnAttr(llvm::Attribute::OptimizeNone); |
1659 | if (func.getConvergentAttr()) |
1660 | llvmFunc->addFnAttr(llvm::Attribute::Convergent); |
1661 | if (func.getNoUnwindAttr()) |
1662 | llvmFunc->addFnAttr(llvm::Attribute::NoUnwind); |
1663 | if (func.getWillReturnAttr()) |
1664 | llvmFunc->addFnAttr(llvm::Attribute::WillReturn); |
1665 | if (TargetFeaturesAttr targetFeatAttr = func.getTargetFeaturesAttr()) |
1666 | llvmFunc->addFnAttr("target-features", targetFeatAttr.getFeaturesString()); |
1667 | if (FramePointerKindAttr fpAttr = func.getFramePointerAttr()) |
1668 | llvmFunc->addFnAttr("frame-pointer", stringifyFramePointerKind( |
1669 | fpAttr.getFramePointerKind())); |
1670 | if (UWTableKindAttr uwTableKindAttr = func.getUwtableKindAttr()) |
1671 | llvmFunc->setUWTableKind( |
1672 | convertUWTableKindToLLVM(uwTableKindAttr.getUwtableKind())); |
1673 | convertFunctionMemoryAttributes(func, llvmFunc); |
1674 | } |
1675 | |
1676 | /// Converts function attributes from `func` and attaches them to `llvmFunc`. |
1677 | static void convertFunctionKernelAttributes(LLVMFuncOp func, |
1678 | llvm::Function *llvmFunc, |
1679 | ModuleTranslation &translation) { |
1680 | llvm::LLVMContext &llvmContext = llvmFunc->getContext(); |
1681 | |
1682 | if (VecTypeHintAttr vecTypeHint = func.getVecTypeHintAttr()) { |
1683 | Type type = vecTypeHint.getHint().getValue(); |
1684 | llvm::Type *llvmType = translation.convertType(type); |
1685 | bool isSigned = vecTypeHint.getIsSigned(); |
1686 | llvmFunc->setMetadata( |
1687 | func.getVecTypeHintAttrName(), |
1688 | convertVecTypeHintToMDNode(context&: llvmContext, type: llvmType, isSigned)); |
1689 | } |
1690 | |
1691 | if (std::optional<ArrayRef<int32_t>> workGroupSizeHint = |
1692 | func.getWorkGroupSizeHint()) { |
1693 | llvmFunc->setMetadata( |
1694 | func.getWorkGroupSizeHintAttrName(), |
1695 | convertIntegerArrayToMDNode(context&: llvmContext, values: *workGroupSizeHint)); |
1696 | } |
1697 | |
1698 | if (std::optional<ArrayRef<int32_t>> reqdWorkGroupSize = |
1699 | func.getReqdWorkGroupSize()) { |
1700 | llvmFunc->setMetadata( |
1701 | func.getReqdWorkGroupSizeAttrName(), |
1702 | convertIntegerArrayToMDNode(context&: llvmContext, values: *reqdWorkGroupSize)); |
1703 | } |
1704 | |
1705 | if (std::optional<uint32_t> intelReqdSubGroupSize = |
1706 | func.getIntelReqdSubGroupSize()) { |
1707 | llvmFunc->setMetadata( |
1708 | func.getIntelReqdSubGroupSizeAttrName(), |
1709 | convertIntegerToMDNode(context&: llvmContext, |
1710 | value: llvm::APInt(32, *intelReqdSubGroupSize))); |
1711 | } |
1712 | } |
1713 | |
1714 | static LogicalResult convertParameterAttr(llvm::AttrBuilder &attrBuilder, |
1715 | llvm::Attribute::AttrKind llvmKind, |
1716 | NamedAttribute namedAttr, |
1717 | ModuleTranslation &moduleTranslation, |
1718 | Location loc) { |
1719 | return llvm::TypeSwitch<Attribute, LogicalResult>(namedAttr.getValue()) |
1720 | .Case<TypeAttr>([&](auto typeAttr) { |
1721 | attrBuilder.addTypeAttr( |
1722 | llvmKind, moduleTranslation.convertType(typeAttr.getValue())); |
1723 | return success(); |
1724 | }) |
1725 | .Case<IntegerAttr>([&](auto intAttr) { |
1726 | attrBuilder.addRawIntAttr(llvmKind, intAttr.getInt()); |
1727 | return success(); |
1728 | }) |
1729 | .Case<UnitAttr>([&](auto) { |
1730 | attrBuilder.addAttribute(llvmKind); |
1731 | return success(); |
1732 | }) |
1733 | .Case<LLVM::ConstantRangeAttr>([&](auto rangeAttr) { |
1734 | attrBuilder.addConstantRangeAttr( |
1735 | llvmKind, |
1736 | llvm::ConstantRange(rangeAttr.getLower(), rangeAttr.getUpper())); |
1737 | return success(); |
1738 | }) |
1739 | .Default([loc](auto) { |
1740 | return emitError(loc, "unsupported parameter attribute type"); |
1741 | }); |
1742 | } |
1743 | |
1744 | FailureOr<llvm::AttrBuilder> |
1745 | ModuleTranslation::convertParameterAttrs(LLVMFuncOp func, int argIdx, |
1746 | DictionaryAttr paramAttrs) { |
1747 | llvm::AttrBuilder attrBuilder(llvmModule->getContext()); |
1748 | auto attrNameToKindMapping = getAttrNameToKindMapping(); |
1749 | Location loc = func.getLoc(); |
1750 | |
1751 | for (auto namedAttr : paramAttrs) { |
1752 | auto it = attrNameToKindMapping.find(namedAttr.getName()); |
1753 | if (it != attrNameToKindMapping.end()) { |
1754 | llvm::Attribute::AttrKind llvmKind = it->second; |
1755 | if (failed(convertParameterAttr(attrBuilder, llvmKind, namedAttr, *this, |
1756 | loc))) |
1757 | return failure(); |
1758 | } else if (namedAttr.getNameDialect()) { |
1759 | if (failed(iface.convertParameterAttr(func, argIdx, namedAttr, *this))) |
1760 | return failure(); |
1761 | } |
1762 | } |
1763 | |
1764 | return attrBuilder; |
1765 | } |
1766 | |
1767 | FailureOr<llvm::AttrBuilder> |
1768 | ModuleTranslation::convertParameterAttrs(Location loc, |
1769 | DictionaryAttr paramAttrs) { |
1770 | llvm::AttrBuilder attrBuilder(llvmModule->getContext()); |
1771 | auto attrNameToKindMapping = getAttrNameToKindMapping(); |
1772 | |
1773 | for (auto namedAttr : paramAttrs) { |
1774 | auto it = attrNameToKindMapping.find(namedAttr.getName()); |
1775 | if (it != attrNameToKindMapping.end()) { |
1776 | llvm::Attribute::AttrKind llvmKind = it->second; |
1777 | if (failed(convertParameterAttr(attrBuilder, llvmKind, namedAttr, *this, |
1778 | loc))) |
1779 | return failure(); |
1780 | } |
1781 | } |
1782 | |
1783 | return attrBuilder; |
1784 | } |
1785 | |
1786 | LogicalResult ModuleTranslation::convertFunctionSignatures() { |
1787 | // Declare all functions first because there may be function calls that form a |
1788 | // call graph with cycles, or global initializers that reference functions. |
1789 | for (auto function : getModuleBody(mlirModule).getOps<LLVMFuncOp>()) { |
1790 | llvm::FunctionCallee llvmFuncCst = llvmModule->getOrInsertFunction( |
1791 | function.getName(), |
1792 | cast<llvm::FunctionType>(convertType(function.getFunctionType()))); |
1793 | llvm::Function *llvmFunc = cast<llvm::Function>(llvmFuncCst.getCallee()); |
1794 | llvmFunc->setLinkage(convertLinkageToLLVM(function.getLinkage())); |
1795 | llvmFunc->setCallingConv(convertCConvToLLVM(function.getCConv())); |
1796 | mapFunction(function.getName(), llvmFunc); |
1797 | addRuntimePreemptionSpecifier(function.getDsoLocal(), llvmFunc); |
1798 | |
1799 | // Convert function attributes. |
1800 | convertFunctionAttributes(function, llvmFunc); |
1801 | |
1802 | // Convert function kernel attributes to metadata. |
1803 | convertFunctionKernelAttributes(function, llvmFunc, *this); |
1804 | |
1805 | // Convert function_entry_count attribute to metadata. |
1806 | if (std::optional<uint64_t> entryCount = function.getFunctionEntryCount()) |
1807 | llvmFunc->setEntryCount(entryCount.value()); |
1808 | |
1809 | // Convert result attributes. |
1810 | if (ArrayAttr allResultAttrs = function.getAllResultAttrs()) { |
1811 | DictionaryAttr resultAttrs = cast<DictionaryAttr>(allResultAttrs[0]); |
1812 | FailureOr<llvm::AttrBuilder> attrBuilder = |
1813 | convertParameterAttrs(function, -1, resultAttrs); |
1814 | if (failed(attrBuilder)) |
1815 | return failure(); |
1816 | llvmFunc->addRetAttrs(*attrBuilder); |
1817 | } |
1818 | |
1819 | // Convert argument attributes. |
1820 | for (auto [argIdx, llvmArg] : llvm::enumerate(llvmFunc->args())) { |
1821 | if (DictionaryAttr argAttrs = function.getArgAttrDict(argIdx)) { |
1822 | FailureOr<llvm::AttrBuilder> attrBuilder = |
1823 | convertParameterAttrs(function, argIdx, argAttrs); |
1824 | if (failed(attrBuilder)) |
1825 | return failure(); |
1826 | llvmArg.addAttrs(*attrBuilder); |
1827 | } |
1828 | } |
1829 | |
1830 | // Forward the pass-through attributes to LLVM. |
1831 | if (failed(forwardPassthroughAttributes( |
1832 | function.getLoc(), function.getPassthrough(), llvmFunc))) |
1833 | return failure(); |
1834 | |
1835 | // Convert visibility attribute. |
1836 | llvmFunc->setVisibility(convertVisibilityToLLVM(function.getVisibility_())); |
1837 | |
1838 | // Convert the comdat attribute. |
1839 | if (std::optional<mlir::SymbolRefAttr> comdat = function.getComdat()) { |
1840 | auto selectorOp = cast<ComdatSelectorOp>( |
1841 | SymbolTable::lookupNearestSymbolFrom(function, *comdat)); |
1842 | llvmFunc->setComdat(comdatMapping.lookup(selectorOp)); |
1843 | } |
1844 | |
1845 | if (auto gc = function.getGarbageCollector()) |
1846 | llvmFunc->setGC(gc->str()); |
1847 | |
1848 | if (auto unnamedAddr = function.getUnnamedAddr()) |
1849 | llvmFunc->setUnnamedAddr(convertUnnamedAddrToLLVM(*unnamedAddr)); |
1850 | |
1851 | if (auto alignment = function.getAlignment()) |
1852 | llvmFunc->setAlignment(llvm::MaybeAlign(*alignment)); |
1853 | |
1854 | // Translate the debug information for this function. |
1855 | debugTranslation->translate(function, *llvmFunc); |
1856 | } |
1857 | |
1858 | return success(); |
1859 | } |
1860 | |
1861 | LogicalResult ModuleTranslation::convertFunctions() { |
1862 | // Convert functions. |
1863 | for (auto function : getModuleBody(mlirModule).getOps<LLVMFuncOp>()) { |
1864 | // Do not convert external functions, but do process dialect attributes |
1865 | // attached to them. |
1866 | if (function.isExternal()) { |
1867 | if (failed(convertDialectAttributes(function, {}))) |
1868 | return failure(); |
1869 | continue; |
1870 | } |
1871 | |
1872 | if (failed(convertOneFunction(function))) |
1873 | return failure(); |
1874 | } |
1875 | |
1876 | return success(); |
1877 | } |
1878 | |
1879 | LogicalResult ModuleTranslation::convertComdats() { |
1880 | for (auto comdatOp : getModuleBody(mlirModule).getOps<ComdatOp>()) { |
1881 | for (auto selectorOp : comdatOp.getOps<ComdatSelectorOp>()) { |
1882 | llvm::Module *module = getLLVMModule(); |
1883 | if (module->getComdatSymbolTable().contains(selectorOp.getSymName())) |
1884 | return emitError(selectorOp.getLoc()) |
1885 | << "comdat selection symbols must be unique even in different " |
1886 | "comdat regions"; |
1887 | llvm::Comdat *comdat = module->getOrInsertComdat(selectorOp.getSymName()); |
1888 | comdat->setSelectionKind(convertComdatToLLVM(selectorOp.getComdat())); |
1889 | comdatMapping.try_emplace(selectorOp, comdat); |
1890 | } |
1891 | } |
1892 | return success(); |
1893 | } |
1894 | |
1895 | LogicalResult ModuleTranslation::convertUnresolvedBlockAddress() { |
1896 | for (auto &[blockAddressOp, llvmCst] : unresolvedBlockAddressMapping) { |
1897 | BlockAddressAttr blockAddressAttr = blockAddressOp.getBlockAddr(); |
1898 | llvm::BasicBlock *llvmBlock = lookupBlockAddress(blockAddressAttr); |
1899 | assert(llvmBlock && "expected LLVM blocks to be already translated"); |
1900 | |
1901 | // Update mapping with new block address constant. |
1902 | auto *llvmBlockAddr = llvm::BlockAddress::get( |
1903 | lookupFunction(blockAddressAttr.getFunction().getValue()), llvmBlock); |
1904 | llvmCst->replaceAllUsesWith(llvmBlockAddr); |
1905 | assert(llvmCst->use_empty() && "expected all uses to be replaced"); |
1906 | cast<llvm::GlobalVariable>(llvmCst)->eraseFromParent(); |
1907 | } |
1908 | unresolvedBlockAddressMapping.clear(); |
1909 | return success(); |
1910 | } |
1911 | |
1912 | void ModuleTranslation::setAccessGroupsMetadata(AccessGroupOpInterface op, |
1913 | llvm::Instruction *inst) { |
1914 | if (llvm::MDNode *node = loopAnnotationTranslation->getAccessGroups(op)) |
1915 | inst->setMetadata(KindID: llvm::LLVMContext::MD_access_group, Node: node); |
1916 | } |
1917 | |
1918 | llvm::MDNode * |
1919 | ModuleTranslation::getOrCreateAliasScope(AliasScopeAttr aliasScopeAttr) { |
1920 | auto [scopeIt, scopeInserted] = |
1921 | aliasScopeMetadataMapping.try_emplace(aliasScopeAttr, nullptr); |
1922 | if (!scopeInserted) |
1923 | return scopeIt->second; |
1924 | llvm::LLVMContext &ctx = llvmModule->getContext(); |
1925 | auto dummy = llvm::MDNode::getTemporary(Context&: ctx, MDs: std::nullopt); |
1926 | // Convert the domain metadata node if necessary. |
1927 | auto [domainIt, insertedDomain] = aliasDomainMetadataMapping.try_emplace( |
1928 | aliasScopeAttr.getDomain(), nullptr); |
1929 | if (insertedDomain) { |
1930 | llvm::SmallVector<llvm::Metadata *, 2> operands; |
1931 | // Placeholder for potential self-reference. |
1932 | operands.push_back(Elt: dummy.get()); |
1933 | if (StringAttr description = aliasScopeAttr.getDomain().getDescription()) |
1934 | operands.push_back(Elt: llvm::MDString::get(ctx, description)); |
1935 | domainIt->second = llvm::MDNode::get(ctx, operands); |
1936 | // Self-reference for uniqueness. |
1937 | llvm::Metadata *replacement; |
1938 | if (auto stringAttr = |
1939 | dyn_cast<StringAttr>(aliasScopeAttr.getDomain().getId())) |
1940 | replacement = llvm::MDString::get(ctx, stringAttr.getValue()); |
1941 | else |
1942 | replacement = domainIt->second; |
1943 | domainIt->second->replaceOperandWith(0, replacement); |
1944 | } |
1945 | // Convert the scope metadata node. |
1946 | assert(domainIt->second && "Scope's domain should already be valid"); |
1947 | llvm::SmallVector<llvm::Metadata *, 3> operands; |
1948 | // Placeholder for potential self-reference. |
1949 | operands.push_back(Elt: dummy.get()); |
1950 | operands.push_back(Elt: domainIt->second); |
1951 | if (StringAttr description = aliasScopeAttr.getDescription()) |
1952 | operands.push_back(Elt: llvm::MDString::get(ctx, description)); |
1953 | scopeIt->second = llvm::MDNode::get(ctx, operands); |
1954 | // Self-reference for uniqueness. |
1955 | llvm::Metadata *replacement; |
1956 | if (auto stringAttr = dyn_cast<StringAttr>(aliasScopeAttr.getId())) |
1957 | replacement = llvm::MDString::get(ctx, stringAttr.getValue()); |
1958 | else |
1959 | replacement = scopeIt->second; |
1960 | scopeIt->second->replaceOperandWith(0, replacement); |
1961 | return scopeIt->second; |
1962 | } |
1963 | |
1964 | llvm::MDNode *ModuleTranslation::getOrCreateAliasScopes( |
1965 | ArrayRef<AliasScopeAttr> aliasScopeAttrs) { |
1966 | SmallVector<llvm::Metadata *> nodes; |
1967 | nodes.reserve(N: aliasScopeAttrs.size()); |
1968 | for (AliasScopeAttr aliasScopeAttr : aliasScopeAttrs) |
1969 | nodes.push_back(getOrCreateAliasScope(aliasScopeAttr)); |
1970 | return llvm::MDNode::get(Context&: getLLVMContext(), MDs: nodes); |
1971 | } |
1972 | |
1973 | void ModuleTranslation::setAliasScopeMetadata(AliasAnalysisOpInterface op, |
1974 | llvm::Instruction *inst) { |
1975 | auto populateScopeMetadata = [&](ArrayAttr aliasScopeAttrs, unsigned kind) { |
1976 | if (!aliasScopeAttrs || aliasScopeAttrs.empty()) |
1977 | return; |
1978 | llvm::MDNode *node = getOrCreateAliasScopes( |
1979 | aliasScopeAttrs: llvm::to_vector(aliasScopeAttrs.getAsRange<AliasScopeAttr>())); |
1980 | inst->setMetadata(KindID: kind, Node: node); |
1981 | }; |
1982 | |
1983 | populateScopeMetadata(op.getAliasScopesOrNull(), |
1984 | llvm::LLVMContext::MD_alias_scope); |
1985 | populateScopeMetadata(op.getNoAliasScopesOrNull(), |
1986 | llvm::LLVMContext::MD_noalias); |
1987 | } |
1988 | |
1989 | llvm::MDNode *ModuleTranslation::getTBAANode(TBAATagAttr tbaaAttr) const { |
1990 | return tbaaMetadataMapping.lookup(Val: tbaaAttr); |
1991 | } |
1992 | |
1993 | void ModuleTranslation::setTBAAMetadata(AliasAnalysisOpInterface op, |
1994 | llvm::Instruction *inst) { |
1995 | ArrayAttr tagRefs = op.getTBAATagsOrNull(); |
1996 | if (!tagRefs || tagRefs.empty()) |
1997 | return; |
1998 | |
1999 | // LLVM IR currently does not support attaching more than one TBAA access tag |
2000 | // to a memory accessing instruction. It may be useful to support this in |
2001 | // future, but for the time being just ignore the metadata if MLIR operation |
2002 | // has multiple access tags. |
2003 | if (tagRefs.size() > 1) { |
2004 | op.emitWarning() << "TBAA access tags were not translated, because LLVM " |
2005 | "IR only supports a single tag per instruction"; |
2006 | return; |
2007 | } |
2008 | |
2009 | llvm::MDNode *node = getTBAANode(cast<TBAATagAttr>(tagRefs[0])); |
2010 | inst->setMetadata(KindID: llvm::LLVMContext::MD_tbaa, Node: node); |
2011 | } |
2012 | |
2013 | void ModuleTranslation::setDereferenceableMetadata( |
2014 | DereferenceableOpInterface op, llvm::Instruction *inst) { |
2015 | DereferenceableAttr derefAttr = op.getDereferenceableOrNull(); |
2016 | if (!derefAttr) |
2017 | return; |
2018 | |
2019 | llvm::MDNode *derefSizeNode = llvm::MDNode::get( |
2020 | Context&: getLLVMContext(), |
2021 | MDs: llvm::ConstantAsMetadata::get(C: llvm::ConstantInt::get( |
2022 | llvm::IntegerType::get(C&: getLLVMContext(), NumBits: 64), derefAttr.getBytes()))); |
2023 | unsigned kindId = derefAttr.getMayBeNull() |
2024 | ? llvm::LLVMContext::MD_dereferenceable_or_null |
2025 | : llvm::LLVMContext::MD_dereferenceable; |
2026 | inst->setMetadata(KindID: kindId, Node: derefSizeNode); |
2027 | } |
2028 | |
2029 | void ModuleTranslation::setBranchWeightsMetadata(BranchWeightOpInterface op) { |
2030 | DenseI32ArrayAttr weightsAttr = op.getBranchWeightsOrNull(); |
2031 | if (!weightsAttr) |
2032 | return; |
2033 | |
2034 | llvm::Instruction *inst = isa<CallOp>(op) ? lookupCall(op) : lookupBranch(op); |
2035 | assert(inst && "expected the operation to have a mapping to an instruction"); |
2036 | SmallVector<uint32_t> weights(weightsAttr.asArrayRef()); |
2037 | inst->setMetadata( |
2038 | KindID: llvm::LLVMContext::MD_prof, |
2039 | Node: llvm::MDBuilder(getLLVMContext()).createBranchWeights(Weights: weights)); |
2040 | } |
2041 | |
2042 | LogicalResult ModuleTranslation::createTBAAMetadata() { |
2043 | llvm::LLVMContext &ctx = llvmModule->getContext(); |
2044 | llvm::IntegerType *offsetTy = llvm::IntegerType::get(C&: ctx, NumBits: 64); |
2045 | |
2046 | // Walk the entire module and create all metadata nodes for the TBAA |
2047 | // attributes. The code below relies on two invariants of the |
2048 | // `AttrTypeWalker`: |
2049 | // 1. Attributes are visited in post-order: Since the attributes create a DAG, |
2050 | // this ensures that any lookups into `tbaaMetadataMapping` for child |
2051 | // attributes succeed. |
2052 | // 2. Attributes are only ever visited once: This way we don't leak any |
2053 | // LLVM metadata instances. |
2054 | AttrTypeWalker walker; |
2055 | walker.addWalk(callback: [&](TBAARootAttr root) { |
2056 | tbaaMetadataMapping.insert( |
2057 | {root, llvm::MDNode::get(Context&: ctx, MDs: llvm::MDString::get(ctx, root.getId()))}); |
2058 | }); |
2059 | |
2060 | walker.addWalk(callback: [&](TBAATypeDescriptorAttr descriptor) { |
2061 | SmallVector<llvm::Metadata *> operands; |
2062 | operands.push_back(Elt: llvm::MDString::get(ctx, descriptor.getId())); |
2063 | for (TBAAMemberAttr member : descriptor.getMembers()) { |
2064 | operands.push_back(tbaaMetadataMapping.lookup(member.getTypeDesc())); |
2065 | operands.push_back(llvm::ConstantAsMetadata::get( |
2066 | llvm::ConstantInt::get(offsetTy, member.getOffset()))); |
2067 | } |
2068 | |
2069 | tbaaMetadataMapping.insert({descriptor, llvm::MDNode::get(Context&: ctx, MDs: operands)}); |
2070 | }); |
2071 | |
2072 | walker.addWalk(callback: [&](TBAATagAttr tag) { |
2073 | SmallVector<llvm::Metadata *> operands; |
2074 | |
2075 | operands.push_back(Elt: tbaaMetadataMapping.lookup(Val: tag.getBaseType())); |
2076 | operands.push_back(Elt: tbaaMetadataMapping.lookup(Val: tag.getAccessType())); |
2077 | |
2078 | operands.push_back(Elt: llvm::ConstantAsMetadata::get( |
2079 | C: llvm::ConstantInt::get(offsetTy, tag.getOffset()))); |
2080 | if (tag.getConstant()) |
2081 | operands.push_back( |
2082 | Elt: llvm::ConstantAsMetadata::get(C: llvm::ConstantInt::get(Ty: offsetTy, V: 1))); |
2083 | |
2084 | tbaaMetadataMapping.insert({tag, llvm::MDNode::get(Context&: ctx, MDs: operands)}); |
2085 | }); |
2086 | |
2087 | mlirModule->walk(callback: [&](AliasAnalysisOpInterface analysisOpInterface) { |
2088 | if (auto attr = analysisOpInterface.getTBAATagsOrNull()) |
2089 | walker.walk(attr); |
2090 | }); |
2091 | |
2092 | return success(); |
2093 | } |
2094 | |
2095 | LogicalResult ModuleTranslation::createIdentMetadata() { |
2096 | if (auto attr = mlirModule->getAttrOfType<StringAttr>( |
2097 | LLVMDialect::getIdentAttrName())) { |
2098 | StringRef ident = attr; |
2099 | llvm::LLVMContext &ctx = llvmModule->getContext(); |
2100 | llvm::NamedMDNode *namedMd = |
2101 | llvmModule->getOrInsertNamedMetadata(LLVMDialect::getIdentAttrName()); |
2102 | llvm::MDNode *md = llvm::MDNode::get(Context&: ctx, MDs: llvm::MDString::get(Context&: ctx, Str: ident)); |
2103 | namedMd->addOperand(M: md); |
2104 | } |
2105 | |
2106 | return success(); |
2107 | } |
2108 | |
2109 | LogicalResult ModuleTranslation::createCommandlineMetadata() { |
2110 | if (auto attr = mlirModule->getAttrOfType<StringAttr>( |
2111 | LLVMDialect::getCommandlineAttrName())) { |
2112 | StringRef cmdLine = attr; |
2113 | llvm::LLVMContext &ctx = llvmModule->getContext(); |
2114 | llvm::NamedMDNode *nmd = llvmModule->getOrInsertNamedMetadata( |
2115 | LLVMDialect::getCommandlineAttrName()); |
2116 | llvm::MDNode *md = |
2117 | llvm::MDNode::get(Context&: ctx, MDs: llvm::MDString::get(Context&: ctx, Str: cmdLine)); |
2118 | nmd->addOperand(M: md); |
2119 | } |
2120 | |
2121 | return success(); |
2122 | } |
2123 | |
2124 | LogicalResult ModuleTranslation::createDependentLibrariesMetadata() { |
2125 | if (auto dependentLibrariesAttr = mlirModule->getDiscardableAttr( |
2126 | LLVM::LLVMDialect::getDependentLibrariesAttrName())) { |
2127 | auto *nmd = |
2128 | llvmModule->getOrInsertNamedMetadata(Name: "llvm.dependent-libraries"); |
2129 | llvm::LLVMContext &ctx = llvmModule->getContext(); |
2130 | for (auto libAttr : |
2131 | cast<ArrayAttr>(dependentLibrariesAttr).getAsRange<StringAttr>()) { |
2132 | auto *md = |
2133 | llvm::MDNode::get(ctx, llvm::MDString::get(ctx, libAttr.getValue())); |
2134 | nmd->addOperand(md); |
2135 | } |
2136 | } |
2137 | return success(); |
2138 | } |
2139 | |
2140 | void ModuleTranslation::setLoopMetadata(Operation *op, |
2141 | llvm::Instruction *inst) { |
2142 | LoopAnnotationAttr attr = |
2143 | TypeSwitch<Operation *, LoopAnnotationAttr>(op) |
2144 | .Case<LLVM::BrOp, LLVM::CondBrOp>( |
2145 | [](auto branchOp) { return branchOp.getLoopAnnotationAttr(); }); |
2146 | if (!attr) |
2147 | return; |
2148 | llvm::MDNode *loopMD = |
2149 | loopAnnotationTranslation->translateLoopAnnotation(attr, op); |
2150 | inst->setMetadata(KindID: llvm::LLVMContext::MD_loop, Node: loopMD); |
2151 | } |
2152 | |
2153 | void ModuleTranslation::setDisjointFlag(Operation *op, llvm::Value *value) { |
2154 | auto iface = cast<DisjointFlagInterface>(op); |
2155 | // We do a dyn_cast here in case the value got folded into a constant. |
2156 | if (auto disjointInst = dyn_cast<llvm::PossiblyDisjointInst>(Val: value)) |
2157 | disjointInst->setIsDisjoint(iface.getIsDisjoint()); |
2158 | } |
2159 | |
2160 | llvm::Type *ModuleTranslation::convertType(Type type) { |
2161 | return typeTranslator.translateType(type); |
2162 | } |
2163 | |
2164 | /// A helper to look up remapped operands in the value remapping table. |
2165 | SmallVector<llvm::Value *> ModuleTranslation::lookupValues(ValueRange values) { |
2166 | SmallVector<llvm::Value *> remapped; |
2167 | remapped.reserve(N: values.size()); |
2168 | for (Value v : values) |
2169 | remapped.push_back(Elt: lookupValue(value: v)); |
2170 | return remapped; |
2171 | } |
2172 | |
2173 | llvm::OpenMPIRBuilder *ModuleTranslation::getOpenMPBuilder() { |
2174 | if (!ompBuilder) { |
2175 | ompBuilder = std::make_unique<llvm::OpenMPIRBuilder>(args&: *llvmModule); |
2176 | ompBuilder->initialize(); |
2177 | |
2178 | // Flags represented as top-level OpenMP dialect attributes are set in |
2179 | // `OpenMPDialectLLVMIRTranslationInterface::amendOperation()`. Here we set |
2180 | // the default configuration. |
2181 | ompBuilder->setConfig(llvm::OpenMPIRBuilderConfig( |
2182 | /* IsTargetDevice = */ false, /* IsGPU = */ false, |
2183 | /* OpenMPOffloadMandatory = */ false, |
2184 | /* HasRequiresReverseOffload = */ false, |
2185 | /* HasRequiresUnifiedAddress = */ false, |
2186 | /* HasRequiresUnifiedSharedMemory = */ false, |
2187 | /* HasRequiresDynamicAllocators = */ false)); |
2188 | } |
2189 | return ompBuilder.get(); |
2190 | } |
2191 | |
2192 | llvm::DILocation *ModuleTranslation::translateLoc(Location loc, |
2193 | llvm::DILocalScope *scope) { |
2194 | return debugTranslation->translateLoc(loc, scope); |
2195 | } |
2196 | |
2197 | llvm::DIExpression * |
2198 | ModuleTranslation::translateExpression(LLVM::DIExpressionAttr attr) { |
2199 | return debugTranslation->translateExpression(attr); |
2200 | } |
2201 | |
2202 | llvm::DIGlobalVariableExpression * |
2203 | ModuleTranslation::translateGlobalVariableExpression( |
2204 | LLVM::DIGlobalVariableExpressionAttr attr) { |
2205 | return debugTranslation->translateGlobalVariableExpression(attr); |
2206 | } |
2207 | |
2208 | llvm::Metadata *ModuleTranslation::translateDebugInfo(LLVM::DINodeAttr attr) { |
2209 | return debugTranslation->translate(attr); |
2210 | } |
2211 | |
2212 | llvm::RoundingMode |
2213 | ModuleTranslation::translateRoundingMode(LLVM::RoundingMode rounding) { |
2214 | return convertRoundingModeToLLVM(rounding); |
2215 | } |
2216 | |
2217 | llvm::fp::ExceptionBehavior ModuleTranslation::translateFPExceptionBehavior( |
2218 | LLVM::FPExceptionBehavior exceptionBehavior) { |
2219 | return convertFPExceptionBehaviorToLLVM(exceptionBehavior); |
2220 | } |
2221 | |
2222 | llvm::NamedMDNode * |
2223 | ModuleTranslation::getOrInsertNamedModuleMetadata(StringRef name) { |
2224 | return llvmModule->getOrInsertNamedMetadata(Name: name); |
2225 | } |
2226 | |
2227 | void ModuleTranslation::StackFrame::anchor() {} |
2228 | |
2229 | static std::unique_ptr<llvm::Module> |
2230 | prepareLLVMModule(Operation *m, llvm::LLVMContext &llvmContext, |
2231 | StringRef name) { |
2232 | m->getContext()->getOrLoadDialect<LLVM::LLVMDialect>(); |
2233 | auto llvmModule = std::make_unique<llvm::Module>(args&: name, args&: llvmContext); |
2234 | // ModuleTranslation can currently only construct modules in the old debug |
2235 | // info format, so set the flag accordingly. |
2236 | llvmModule->setNewDbgInfoFormatFlag(false); |
2237 | if (auto dataLayoutAttr = |
2238 | m->getDiscardableAttr(LLVM::LLVMDialect::getDataLayoutAttrName())) { |
2239 | llvmModule->setDataLayout(cast<StringAttr>(dataLayoutAttr).getValue()); |
2240 | } else { |
2241 | FailureOr<llvm::DataLayout> llvmDataLayout(llvm::DataLayout("")); |
2242 | if (auto iface = dyn_cast<DataLayoutOpInterface>(m)) { |
2243 | if (DataLayoutSpecInterface spec = iface.getDataLayoutSpec()) { |
2244 | llvmDataLayout = |
2245 | translateDataLayout(spec, DataLayout(iface), m->getLoc()); |
2246 | } |
2247 | } else if (auto mod = dyn_cast<ModuleOp>(m)) { |
2248 | if (DataLayoutSpecInterface spec = mod.getDataLayoutSpec()) { |
2249 | llvmDataLayout = |
2250 | translateDataLayout(spec, DataLayout(mod), m->getLoc()); |
2251 | } |
2252 | } |
2253 | if (failed(Result: llvmDataLayout)) |
2254 | return nullptr; |
2255 | llvmModule->setDataLayout(*llvmDataLayout); |
2256 | } |
2257 | if (auto targetTripleAttr = |
2258 | m->getDiscardableAttr(LLVM::LLVMDialect::getTargetTripleAttrName())) |
2259 | llvmModule->setTargetTriple( |
2260 | llvm::Triple(cast<StringAttr>(targetTripleAttr).getValue())); |
2261 | |
2262 | return llvmModule; |
2263 | } |
2264 | |
2265 | std::unique_ptr<llvm::Module> |
2266 | mlir::translateModuleToLLVMIR(Operation *module, llvm::LLVMContext &llvmContext, |
2267 | StringRef name, bool disableVerification) { |
2268 | if (!satisfiesLLVMModule(op: module)) { |
2269 | module->emitOpError(message: "can not be translated to an LLVMIR module"); |
2270 | return nullptr; |
2271 | } |
2272 | |
2273 | std::unique_ptr<llvm::Module> llvmModule = |
2274 | prepareLLVMModule(m: module, llvmContext, name); |
2275 | if (!llvmModule) |
2276 | return nullptr; |
2277 | |
2278 | LLVM::ensureDistinctSuccessors(op: module); |
2279 | LLVM::legalizeDIExpressionsRecursively(op: module); |
2280 | |
2281 | ModuleTranslation translator(module, std::move(llvmModule)); |
2282 | llvm::IRBuilder<llvm::TargetFolder> llvmBuilder( |
2283 | llvmContext, |
2284 | llvm::TargetFolder(translator.getLLVMModule()->getDataLayout())); |
2285 | |
2286 | // Convert module before functions and operations inside, so dialect |
2287 | // attributes can be used to change dialect-specific global configurations via |
2288 | // `amendOperation()`. These configurations can then influence the translation |
2289 | // of operations afterwards. |
2290 | if (failed(Result: translator.convertOperation(op&: *module, builder&: llvmBuilder))) |
2291 | return nullptr; |
2292 | |
2293 | if (failed(Result: translator.convertComdats())) |
2294 | return nullptr; |
2295 | if (failed(Result: translator.convertFunctionSignatures())) |
2296 | return nullptr; |
2297 | if (failed(Result: translator.convertGlobalsAndAliases())) |
2298 | return nullptr; |
2299 | if (failed(Result: translator.createTBAAMetadata())) |
2300 | return nullptr; |
2301 | if (failed(Result: translator.createIdentMetadata())) |
2302 | return nullptr; |
2303 | if (failed(Result: translator.createCommandlineMetadata())) |
2304 | return nullptr; |
2305 | if (failed(Result: translator.createDependentLibrariesMetadata())) |
2306 | return nullptr; |
2307 | |
2308 | // Convert other top-level operations if possible. |
2309 | for (Operation &o : getModuleBody(module).getOperations()) { |
2310 | if (!isa<LLVM::LLVMFuncOp, LLVM::AliasOp, LLVM::GlobalOp, |
2311 | LLVM::GlobalCtorsOp, LLVM::GlobalDtorsOp, LLVM::ComdatOp>(&o) && |
2312 | !o.hasTrait<OpTrait::IsTerminator>() && |
2313 | failed(translator.convertOperation(o, llvmBuilder))) { |
2314 | return nullptr; |
2315 | } |
2316 | } |
2317 | |
2318 | // Operations in function bodies with symbolic references must be converted |
2319 | // after the top-level operations they refer to are declared, so we do it |
2320 | // last. |
2321 | if (failed(Result: translator.convertFunctions())) |
2322 | return nullptr; |
2323 | |
2324 | // Now that all MLIR blocks are resolved into LLVM ones, patch block address |
2325 | // constants to point to the correct blocks. |
2326 | if (failed(Result: translator.convertUnresolvedBlockAddress())) |
2327 | return nullptr; |
2328 | |
2329 | // Once we've finished constructing elements in the module, we should convert |
2330 | // it to use the debug info format desired by LLVM. |
2331 | // See https://llvm.org/docs/RemoveDIsDebugInfo.html |
2332 | translator.llvmModule->setIsNewDbgInfoFormat(true); |
2333 | |
2334 | // Add the necessary debug info module flags, if they were not encoded in MLIR |
2335 | // beforehand. |
2336 | translator.debugTranslation->addModuleFlagsIfNotPresent(); |
2337 | |
2338 | if (!disableVerification && |
2339 | llvm::verifyModule(M: *translator.llvmModule, OS: &llvm::errs())) |
2340 | return nullptr; |
2341 | |
2342 | return std::move(translator.llvmModule); |
2343 | } |
2344 |
Definitions
- InstructionCapturingInserter
- InstructionCapturingInserter
- getCapturedInstructions
- clearCapturedInstructions
- CollectionScope
- getCapturedInstructions
- setEnabled
- CollectionScope
- ~CollectionScope
- translateDataLayout
- buildSequentialConstant
- getInnermostElementType
- convertDenseElementsAttr
- convertDenseResourceElementsAttr
- getLLVMConstant
- ModuleTranslation
- ~ModuleTranslation
- forgetMapping
- getPHISourceValue
- connectPHINodes
- createIntrinsicCall
- createIntrinsicCall
- convertOperation
- convertBlockImpl
- getModuleBody
- shouldDropGlobalInitializer
- addRuntimePreemptionSpecifier
- convertGlobalsAndAliases
- checkedAddLLVMFnAttribute
- convertIntegerToMetadata
- convertIntegerToMDNode
- convertVecTypeHintToMDNode
- convertIntegerArrayToMDNode
- forwardPassthroughAttributes
- convertOneFunction
- convertDialectAttributes
- convertFunctionMemoryAttributes
- convertFunctionAttributes
- convertFunctionKernelAttributes
- convertParameterAttr
- convertParameterAttrs
- convertParameterAttrs
- convertFunctionSignatures
- convertFunctions
- convertComdats
- convertUnresolvedBlockAddress
- setAccessGroupsMetadata
- getOrCreateAliasScope
- getOrCreateAliasScopes
- setAliasScopeMetadata
- getTBAANode
- setTBAAMetadata
- setDereferenceableMetadata
- setBranchWeightsMetadata
- createTBAAMetadata
- createIdentMetadata
- createCommandlineMetadata
- createDependentLibrariesMetadata
- setLoopMetadata
- setDisjointFlag
- convertType
- lookupValues
- getOpenMPBuilder
- translateLoc
- translateExpression
- translateGlobalVariableExpression
- translateDebugInfo
- translateRoundingMode
- translateFPExceptionBehavior
- getOrInsertNamedModuleMetadata
- anchor
- prepareLLVMModule
Improve your Profiling and Debugging skills
Find out more