1//===- DataLayoutInterfaces.cpp - Data Layout Interface Implementation ----===//
2//
3// Part of the LLVM Project, under the Apache License v2.0 with LLVM Exceptions.
4// See https://llvm.org/LICENSE.txt for license information.
5// SPDX-License-Identifier: Apache-2.0 WITH LLVM-exception
6//
7//===----------------------------------------------------------------------===//
8
9#include "mlir/Interfaces/DataLayoutInterfaces.h"
10#include "mlir/IR/BuiltinDialect.h"
11#include "mlir/IR/BuiltinOps.h"
12#include "mlir/IR/BuiltinTypes.h"
13#include "mlir/IR/Operation.h"
14
15#include "llvm/ADT/TypeSwitch.h"
16#include "llvm/Support/MathExtras.h"
17
18using namespace mlir;
19
20//===----------------------------------------------------------------------===//
21// Default implementations
22//===----------------------------------------------------------------------===//
23
24/// Reports that the given type is missing the data layout information and
25/// exits.
26[[noreturn]] static void reportMissingDataLayout(Type type) {
27 std::string message;
28 llvm::raw_string_ostream os(message);
29 os << "neither the scoping op nor the type class provide data layout "
30 "information for "
31 << type;
32 llvm::report_fatal_error(reason: Twine(message));
33}
34
35/// Returns the bitwidth of the index type if specified in the param list.
36/// Assumes 64-bit index otherwise.
37static uint64_t getIndexBitwidth(DataLayoutEntryListRef params) {
38 if (params.empty())
39 return 64;
40 auto attr = cast<IntegerAttr>(params.front().getValue());
41 return attr.getValue().getZExtValue();
42}
43
44llvm::TypeSize
45mlir::detail::getDefaultTypeSize(Type type, const DataLayout &dataLayout,
46 ArrayRef<DataLayoutEntryInterface> params) {
47 llvm::TypeSize bits = getDefaultTypeSizeInBits(type, dataLayout, params);
48 return divideCeil(numerator: bits, denominator: 8);
49}
50
51llvm::TypeSize
52mlir::detail::getDefaultTypeSizeInBits(Type type, const DataLayout &dataLayout,
53 DataLayoutEntryListRef params) {
54 if (type.isIntOrFloat())
55 return llvm::TypeSize::getFixed(ExactSize: type.getIntOrFloatBitWidth());
56
57 if (auto ctype = dyn_cast<ComplexType>(type)) {
58 Type et = ctype.getElementType();
59 uint64_t innerAlignment =
60 getDefaultPreferredAlignment(type: et, dataLayout, params) * 8;
61 llvm::TypeSize innerSize = getDefaultTypeSizeInBits(type: et, dataLayout, params);
62
63 // Include padding required to align the imaginary value in the complex
64 // type.
65 return llvm::alignTo(Size: innerSize, Align: innerAlignment) + innerSize;
66 }
67
68 // Index is an integer of some bitwidth.
69 if (isa<IndexType>(type))
70 return dataLayout.getTypeSizeInBits(
71 IntegerType::get(type.getContext(), getIndexBitwidth(params)));
72
73 // Sizes of vector types are rounded up to those of types with closest
74 // power-of-two number of elements in the innermost dimension. We also assume
75 // there is no bit-packing at the moment element sizes are taken in bytes and
76 // multiplied with 8 bits.
77 // TODO: make this extensible.
78 if (auto vecType = dyn_cast<VectorType>(type)) {
79 uint64_t baseSize = vecType.getNumElements() / vecType.getShape().back() *
80 llvm::PowerOf2Ceil(A: vecType.getShape().back()) *
81 dataLayout.getTypeSize(t: vecType.getElementType()) * 8;
82 return llvm::TypeSize::get(Quantity: baseSize, Scalable: vecType.isScalable());
83 }
84
85 if (auto typeInterface = dyn_cast<DataLayoutTypeInterface>(type))
86 return typeInterface.getTypeSizeInBits(dataLayout, params);
87
88 reportMissingDataLayout(type);
89}
90
91static DataLayoutEntryInterface
92findEntryForIntegerType(IntegerType intType,
93 ArrayRef<DataLayoutEntryInterface> params) {
94 assert(!params.empty() && "expected non-empty parameter list");
95 std::map<unsigned, DataLayoutEntryInterface> sortedParams;
96 for (DataLayoutEntryInterface entry : params) {
97 sortedParams.insert(std::make_pair(
98 cast<Type>(entry.getKey()).getIntOrFloatBitWidth(), entry));
99 }
100 auto iter = sortedParams.lower_bound(intType.getWidth());
101 if (iter == sortedParams.end())
102 iter = std::prev(iter);
103
104 return iter->second;
105}
106
107constexpr const static uint64_t kDefaultBitsInByte = 8u;
108
109static uint64_t extractABIAlignment(DataLayoutEntryInterface entry) {
110 auto values =
111 cast<DenseIntElementsAttr>(entry.getValue()).getValues<uint64_t>();
112 return static_cast<uint64_t>(*values.begin()) / kDefaultBitsInByte;
113}
114
115static uint64_t
116getIntegerTypeABIAlignment(IntegerType intType,
117 ArrayRef<DataLayoutEntryInterface> params) {
118 constexpr uint64_t kDefaultSmallIntAlignment = 4u;
119 constexpr unsigned kSmallIntSize = 64;
120 if (params.empty()) {
121 return intType.getWidth() < kSmallIntSize
122 ? llvm::PowerOf2Ceil(
123 A: llvm::divideCeil(intType.getWidth(), kDefaultBitsInByte))
124 : kDefaultSmallIntAlignment;
125 }
126
127 return extractABIAlignment(findEntryForIntegerType(intType, params));
128}
129
130static uint64_t
131getFloatTypeABIAlignment(FloatType fltType, const DataLayout &dataLayout,
132 ArrayRef<DataLayoutEntryInterface> params) {
133 assert(params.size() <= 1 && "at most one data layout entry is expected for "
134 "the singleton floating-point type");
135 if (params.empty())
136 return llvm::PowerOf2Ceil(A: dataLayout.getTypeSize(t: fltType).getFixedValue());
137 return extractABIAlignment(params[0]);
138}
139
140uint64_t mlir::detail::getDefaultABIAlignment(
141 Type type, const DataLayout &dataLayout,
142 ArrayRef<DataLayoutEntryInterface> params) {
143 // Natural alignment is the closest power-of-two number above. For scalable
144 // vectors, aligning them to the same as the base vector is sufficient.
145 if (isa<VectorType>(type))
146 return llvm::PowerOf2Ceil(A: dataLayout.getTypeSize(t: type).getKnownMinValue());
147
148 if (auto fltType = dyn_cast<FloatType>(type))
149 return getFloatTypeABIAlignment(fltType, dataLayout, params);
150
151 // Index is an integer of some bitwidth.
152 if (isa<IndexType>(type))
153 return dataLayout.getTypeABIAlignment(
154 IntegerType::get(type.getContext(), getIndexBitwidth(params)));
155
156 if (auto intType = dyn_cast<IntegerType>(type))
157 return getIntegerTypeABIAlignment(intType, params);
158
159 if (auto ctype = dyn_cast<ComplexType>(type))
160 return getDefaultABIAlignment(ctype.getElementType(), dataLayout, params);
161
162 if (auto typeInterface = dyn_cast<DataLayoutTypeInterface>(type))
163 return typeInterface.getABIAlignment(dataLayout, params);
164
165 reportMissingDataLayout(type);
166}
167
168static uint64_t extractPreferredAlignment(DataLayoutEntryInterface entry) {
169 auto values =
170 cast<DenseIntElementsAttr>(entry.getValue()).getValues<uint64_t>();
171 return *std::next(values.begin(), values.size() - 1) / kDefaultBitsInByte;
172}
173
174static uint64_t
175getIntegerTypePreferredAlignment(IntegerType intType,
176 const DataLayout &dataLayout,
177 ArrayRef<DataLayoutEntryInterface> params) {
178 if (params.empty())
179 return llvm::PowerOf2Ceil(A: dataLayout.getTypeSize(t: intType).getFixedValue());
180
181 return extractPreferredAlignment(findEntryForIntegerType(intType, params));
182}
183
184static uint64_t
185getFloatTypePreferredAlignment(FloatType fltType, const DataLayout &dataLayout,
186 ArrayRef<DataLayoutEntryInterface> params) {
187 assert(params.size() <= 1 && "at most one data layout entry is expected for "
188 "the singleton floating-point type");
189 if (params.empty())
190 return dataLayout.getTypeABIAlignment(t: fltType);
191 return extractPreferredAlignment(params[0]);
192}
193
194uint64_t mlir::detail::getDefaultPreferredAlignment(
195 Type type, const DataLayout &dataLayout,
196 ArrayRef<DataLayoutEntryInterface> params) {
197 // Preferred alignment is same as natural for floats and vectors.
198 if (isa<VectorType>(type))
199 return dataLayout.getTypeABIAlignment(t: type);
200
201 if (auto fltType = dyn_cast<FloatType>(type))
202 return getFloatTypePreferredAlignment(fltType, dataLayout, params);
203
204 // Preferred alignment is the closest power-of-two number above for integers
205 // (ABI alignment may be smaller).
206 if (auto intType = dyn_cast<IntegerType>(type))
207 return getIntegerTypePreferredAlignment(intType, dataLayout, params);
208
209 if (isa<IndexType>(Val: type)) {
210 return dataLayout.getTypePreferredAlignment(
211 IntegerType::get(type.getContext(), getIndexBitwidth(params)));
212 }
213
214 if (auto ctype = dyn_cast<ComplexType>(type))
215 return getDefaultPreferredAlignment(ctype.getElementType(), dataLayout,
216 params);
217
218 if (auto typeInterface = dyn_cast<DataLayoutTypeInterface>(type))
219 return typeInterface.getPreferredAlignment(dataLayout, params);
220
221 reportMissingDataLayout(type);
222}
223
224std::optional<uint64_t> mlir::detail::getDefaultIndexBitwidth(
225 Type type, const DataLayout &dataLayout,
226 ArrayRef<DataLayoutEntryInterface> params) {
227 if (isa<IndexType>(Val: type))
228 return getIndexBitwidth(params);
229
230 if (auto typeInterface = dyn_cast<DataLayoutTypeInterface>(type))
231 if (std::optional<uint64_t> indexBitwidth =
232 typeInterface.getIndexBitwidth(dataLayout, params))
233 return *indexBitwidth;
234
235 // Return std::nullopt for all other types, which are assumed to be non
236 // pointer-like types.
237 return std::nullopt;
238}
239
240// Returns the endianness if specified in the given entry. If the entry is empty
241// the default endianness represented by an empty attribute is returned.
242Attribute mlir::detail::getDefaultEndianness(DataLayoutEntryInterface entry) {
243 if (entry == DataLayoutEntryInterface())
244 return Attribute();
245
246 return entry.getValue();
247}
248
249// Returns the default memory space if specified in the given entry. If the
250// entry is empty the default memory space represented by an empty attribute is
251// returned.
252Attribute mlir::detail::getDefaultMemorySpace(DataLayoutEntryInterface entry) {
253 if (!entry)
254 return Attribute();
255
256 return entry.getValue();
257}
258
259// Returns the memory space used for alloca operations if specified in the
260// given entry. If the entry is empty the default memory space represented by
261// an empty attribute is returned.
262Attribute
263mlir::detail::getDefaultAllocaMemorySpace(DataLayoutEntryInterface entry) {
264 if (entry == DataLayoutEntryInterface()) {
265 return Attribute();
266 }
267
268 return entry.getValue();
269}
270
271// Returns the mangling mode if specified in the given entry.
272// If the entry is empty, an empty attribute is returned.
273Attribute mlir::detail::getDefaultManglingMode(DataLayoutEntryInterface entry) {
274 if (entry == DataLayoutEntryInterface())
275 return Attribute();
276
277 return entry.getValue();
278}
279
280// Returns the memory space used for the program memory space. if
281// specified in the given entry. If the entry is empty the default
282// memory space represented by an empty attribute is returned.
283Attribute
284mlir::detail::getDefaultProgramMemorySpace(DataLayoutEntryInterface entry) {
285 if (entry == DataLayoutEntryInterface()) {
286 return Attribute();
287 }
288
289 return entry.getValue();
290}
291
292// Returns the memory space used for global the global memory space. if
293// specified in the given entry. If the entry is empty the default memory
294// space represented by an empty attribute is returned.
295Attribute
296mlir::detail::getDefaultGlobalMemorySpace(DataLayoutEntryInterface entry) {
297 if (entry == DataLayoutEntryInterface()) {
298 return Attribute();
299 }
300
301 return entry.getValue();
302}
303
304// Returns the stack alignment if specified in the given entry. If the entry is
305// empty the default alignment zero is returned.
306uint64_t
307mlir::detail::getDefaultStackAlignment(DataLayoutEntryInterface entry) {
308 if (entry == DataLayoutEntryInterface())
309 return 0;
310
311 auto value = cast<IntegerAttr>(entry.getValue());
312 return value.getValue().getZExtValue();
313}
314
315// Returns the function pointer alignment if specified in the given entry. If
316// the entry is empty the default alignment zero is returned.
317Attribute mlir::detail::getDefaultFunctionPointerAlignment(
318 DataLayoutEntryInterface entry) {
319 if (entry == DataLayoutEntryInterface())
320 return Attribute();
321 return entry.getValue();
322}
323
324// Returns the legal int widths if specified in the given entry. If the entry is
325// empty the default legal int widths represented by an empty attribute is
326// returned.
327Attribute
328mlir::detail::getDefaultLegalIntWidths(DataLayoutEntryInterface entry) {
329 if (entry == DataLayoutEntryInterface())
330 return Attribute();
331 return entry.getValue();
332}
333
334std::optional<Attribute>
335mlir::detail::getDevicePropertyValue(DataLayoutEntryInterface entry) {
336 if (entry == DataLayoutEntryInterface())
337 return std::nullopt;
338
339 return entry.getValue();
340}
341
342DataLayoutEntryList
343mlir::detail::filterEntriesForType(DataLayoutEntryListRef entries,
344 TypeID typeID) {
345 return llvm::filter_to_vector<4>(
346 C&: entries, Pred: [typeID](DataLayoutEntryInterface entry) {
347 auto type = llvm::dyn_cast_if_present<Type>(entry.getKey());
348 return type && type.getTypeID() == typeID;
349 });
350}
351
352DataLayoutEntryInterface
353mlir::detail::filterEntryForIdentifier(DataLayoutEntryListRef entries,
354 StringAttr id) {
355 const auto *it = llvm::find_if(Range&: entries, P: [id](DataLayoutEntryInterface entry) {
356 if (auto attr = dyn_cast<StringAttr>(entry.getKey()))
357 return attr == id;
358 return false;
359 });
360 return it == entries.end() ? DataLayoutEntryInterface() : *it;
361}
362
363static DataLayoutSpecInterface getSpec(Operation *operation) {
364 return llvm::TypeSwitch<Operation *, DataLayoutSpecInterface>(operation)
365 .Case<ModuleOp, DataLayoutOpInterface>(
366 [&](auto op) { return op.getDataLayoutSpec(); })
367 .Default([](Operation *) {
368 llvm_unreachable("expected an op with data layout spec");
369 return DataLayoutSpecInterface();
370 });
371}
372
373static TargetSystemSpecInterface getTargetSystemSpec(Operation *operation) {
374 if (operation) {
375 ModuleOp moduleOp = dyn_cast<ModuleOp>(operation);
376 if (!moduleOp)
377 moduleOp = operation->getParentOfType<ModuleOp>();
378 return moduleOp.getTargetSystemSpec();
379 }
380 return TargetSystemSpecInterface();
381}
382
383/// Populates `opsWithLayout` with the list of proper ancestors of `leaf` that
384/// are either modules or implement the `DataLayoutOpInterface`.
385static void
386collectParentLayouts(Operation *leaf,
387 SmallVectorImpl<DataLayoutSpecInterface> &specs,
388 SmallVectorImpl<Location> *opLocations = nullptr) {
389 if (!leaf)
390 return;
391
392 for (Operation *parent = leaf->getParentOp(); parent != nullptr;
393 parent = parent->getParentOp()) {
394 llvm::TypeSwitch<Operation *>(parent)
395 .Case<ModuleOp>(caseFn: [&](ModuleOp op) {
396 // Skip top-level module op unless it has a layout. Top-level module
397 // without layout is most likely the one implicitly added by the
398 // parser and it doesn't have location. Top-level null specification
399 // would have had the same effect as not having a specification at all
400 // (using type defaults).
401 if (!op->getParentOp() && !op.getDataLayoutSpec())
402 return;
403 specs.push_back(op.getDataLayoutSpec());
404 if (opLocations)
405 opLocations->push_back(Elt: op.getLoc());
406 })
407 .Case<DataLayoutOpInterface>(caseFn: [&](DataLayoutOpInterface op) {
408 specs.push_back(op.getDataLayoutSpec());
409 if (opLocations)
410 opLocations->push_back(Elt: op.getLoc());
411 });
412 }
413}
414
415/// Returns a layout spec that is a combination of the layout specs attached
416/// to the given operation and all its ancestors.
417static DataLayoutSpecInterface getCombinedDataLayout(Operation *leaf) {
418 if (!leaf)
419 return {};
420
421 assert((isa<ModuleOp, DataLayoutOpInterface>(leaf)) &&
422 "expected an op with data layout spec");
423
424 SmallVector<DataLayoutSpecInterface> specs;
425 collectParentLayouts(leaf, specs);
426
427 // Fast track if there are no ancestors.
428 if (specs.empty())
429 return getSpec(leaf);
430
431 // Create the list of non-null specs (null/missing specs can be safely
432 // ignored) from the outermost to the innermost.
433 auto nonNullSpecs = llvm::filter_to_vector<2>(
434 llvm::reverse(specs),
435 [](DataLayoutSpecInterface iface) { return iface != nullptr; });
436
437 // Combine the specs using the innermost as anchor.
438 if (DataLayoutSpecInterface current = getSpec(leaf))
439 return current.combineWith(nonNullSpecs);
440 if (nonNullSpecs.empty())
441 return {};
442 return nonNullSpecs.back().combineWith(
443 llvm::ArrayRef(nonNullSpecs).drop_back());
444}
445
446LogicalResult mlir::detail::verifyDataLayoutOp(Operation *op) {
447 DataLayoutSpecInterface spec = getSpec(op);
448 // The layout specification may be missing and it's fine.
449 if (!spec)
450 return success();
451
452 if (failed(spec.verifySpec(op->getLoc())))
453 return failure();
454 if (!getCombinedDataLayout(op)) {
455 InFlightDiagnostic diag =
456 op->emitError()
457 << "data layout does not combine with layouts of enclosing ops";
458 SmallVector<DataLayoutSpecInterface> specs;
459 SmallVector<Location> opLocations;
460 collectParentLayouts(op, specs, &opLocations);
461 for (Location loc : opLocations)
462 diag.attachNote(noteLoc: loc) << "enclosing op with data layout";
463 return diag;
464 }
465 return success();
466}
467
468llvm::TypeSize mlir::detail::divideCeil(llvm::TypeSize numerator,
469 uint64_t denominator) {
470 uint64_t divided =
471 llvm::divideCeil(Numerator: numerator.getKnownMinValue(), Denominator: denominator);
472 return llvm::TypeSize::get(Quantity: divided, Scalable: numerator.isScalable());
473}
474
475//===----------------------------------------------------------------------===//
476// DataLayout
477//===----------------------------------------------------------------------===//
478
479template <typename OpTy>
480void checkMissingLayout(DataLayoutSpecInterface originalLayout, OpTy op) {
481 if (!originalLayout) {
482 assert((!op || !op.getDataLayoutSpec()) &&
483 "could not compute layout information for an op (failed to "
484 "combine attributes?)");
485 }
486}
487
488mlir::DataLayout::DataLayout() : DataLayout(ModuleOp()) {}
489
490mlir::DataLayout::DataLayout(DataLayoutOpInterface op)
491 : originalLayout(getCombinedDataLayout(op)),
492 originalTargetSystemDesc(getTargetSystemSpec(op)), scope(op),
493 allocaMemorySpace(std::nullopt), programMemorySpace(std::nullopt),
494 globalMemorySpace(std::nullopt), stackAlignment(std::nullopt) {
495#if LLVM_ENABLE_ABI_BREAKING_CHECKS
496 checkMissingLayout(originalLayout, op);
497 collectParentLayouts(op, layoutStack);
498#endif
499}
500
501mlir::DataLayout::DataLayout(ModuleOp op)
502 : originalLayout(getCombinedDataLayout(op)),
503 originalTargetSystemDesc(getTargetSystemSpec(op)), scope(op),
504 allocaMemorySpace(std::nullopt), programMemorySpace(std::nullopt),
505 globalMemorySpace(std::nullopt), stackAlignment(std::nullopt) {
506#if LLVM_ENABLE_ABI_BREAKING_CHECKS
507 checkMissingLayout(originalLayout, op);
508 collectParentLayouts(op, layoutStack);
509#endif
510}
511
512mlir::DataLayout mlir::DataLayout::closest(Operation *op) {
513 // Search the closest parent either being a module operation or implementing
514 // the data layout interface.
515 while (op) {
516 if (auto module = dyn_cast<ModuleOp>(op))
517 return DataLayout(module);
518 if (auto iface = dyn_cast<DataLayoutOpInterface>(op))
519 return DataLayout(iface);
520 op = op->getParentOp();
521 }
522 return DataLayout();
523}
524
525void mlir::DataLayout::checkValid() const {
526#if LLVM_ENABLE_ABI_BREAKING_CHECKS
527 SmallVector<DataLayoutSpecInterface> specs;
528 collectParentLayouts(scope, specs);
529 assert(specs.size() == layoutStack.size() &&
530 "data layout object used, but no longer valid due to the change in "
531 "number of nested layouts");
532 for (auto pair : llvm::zip(specs, layoutStack)) {
533 Attribute newLayout = std::get<0>(pair);
534 Attribute origLayout = std::get<1>(pair);
535 assert(newLayout == origLayout &&
536 "data layout object used, but no longer valid "
537 "due to the change in layout attributes");
538 }
539#endif
540 assert(((!scope && !this->originalLayout) ||
541 (scope && this->originalLayout == getCombinedDataLayout(scope))) &&
542 "data layout object used, but no longer valid due to the change in "
543 "layout spec");
544}
545
546/// Looks up the value for the given type key in the given cache. If there is no
547/// such value in the cache, compute it using the given callback and put it in
548/// the cache before returning.
549template <typename T>
550static T cachedLookup(Type t, DenseMap<Type, T> &cache,
551 function_ref<T(Type)> compute) {
552 auto it = cache.find(t);
553 if (it != cache.end())
554 return it->second;
555
556 auto result = cache.try_emplace(t, compute(t));
557 return result.first->second;
558}
559
560llvm::TypeSize mlir::DataLayout::getTypeSize(Type t) const {
561 checkValid();
562 return cachedLookup<llvm::TypeSize>(t, sizes, [&](Type ty) {
563 DataLayoutEntryList list;
564 if (originalLayout)
565 list = originalLayout.getSpecForType(ty.getTypeID());
566 if (auto iface = dyn_cast_or_null<DataLayoutOpInterface>(scope))
567 return iface.getTypeSize(ty, *this, list);
568 return detail::getDefaultTypeSize(ty, *this, list);
569 });
570}
571
572llvm::TypeSize mlir::DataLayout::getTypeSizeInBits(Type t) const {
573 checkValid();
574 return cachedLookup<llvm::TypeSize>(t, bitsizes, [&](Type ty) {
575 DataLayoutEntryList list;
576 if (originalLayout)
577 list = originalLayout.getSpecForType(ty.getTypeID());
578 if (auto iface = dyn_cast_or_null<DataLayoutOpInterface>(scope))
579 return iface.getTypeSizeInBits(ty, *this, list);
580 return detail::getDefaultTypeSizeInBits(ty, *this, list);
581 });
582}
583
584uint64_t mlir::DataLayout::getTypeABIAlignment(Type t) const {
585 checkValid();
586 return cachedLookup<uint64_t>(t, abiAlignments, [&](Type ty) {
587 DataLayoutEntryList list;
588 if (originalLayout)
589 list = originalLayout.getSpecForType(ty.getTypeID());
590 if (auto iface = dyn_cast_or_null<DataLayoutOpInterface>(scope))
591 return iface.getTypeABIAlignment(ty, *this, list);
592 return detail::getDefaultABIAlignment(ty, *this, list);
593 });
594}
595
596uint64_t mlir::DataLayout::getTypePreferredAlignment(Type t) const {
597 checkValid();
598 return cachedLookup<uint64_t>(t, preferredAlignments, [&](Type ty) {
599 DataLayoutEntryList list;
600 if (originalLayout)
601 list = originalLayout.getSpecForType(ty.getTypeID());
602 if (auto iface = dyn_cast_or_null<DataLayoutOpInterface>(scope))
603 return iface.getTypePreferredAlignment(ty, *this, list);
604 return detail::getDefaultPreferredAlignment(ty, *this, list);
605 });
606}
607
608std::optional<uint64_t> mlir::DataLayout::getTypeIndexBitwidth(Type t) const {
609 checkValid();
610 return cachedLookup<std::optional<uint64_t>>(t, indexBitwidths, [&](Type ty) {
611 DataLayoutEntryList list;
612 if (originalLayout)
613 list = originalLayout.getSpecForType(ty.getTypeID());
614 if (auto iface = dyn_cast_or_null<DataLayoutOpInterface>(scope))
615 return iface.getIndexBitwidth(ty, *this, list);
616 return detail::getDefaultIndexBitwidth(ty, *this, list);
617 });
618}
619
620mlir::Attribute mlir::DataLayout::getEndianness() const {
621 checkValid();
622 if (endianness)
623 return *endianness;
624 DataLayoutEntryInterface entry;
625 if (originalLayout)
626 entry = originalLayout.getSpecForIdentifier(
627 originalLayout.getEndiannessIdentifier(originalLayout.getContext()));
628
629 if (auto iface = dyn_cast_or_null<DataLayoutOpInterface>(scope))
630 endianness = iface.getEndianness(entry);
631 else
632 endianness = detail::getDefaultEndianness(entry: entry);
633 return *endianness;
634}
635
636mlir::Attribute mlir::DataLayout::getDefaultMemorySpace() const {
637 checkValid();
638 if (defaultMemorySpace)
639 return *defaultMemorySpace;
640 DataLayoutEntryInterface entry;
641 if (originalLayout)
642 entry = originalLayout.getSpecForIdentifier(
643 originalLayout.getDefaultMemorySpaceIdentifier(
644 originalLayout.getContext()));
645 if (auto iface = dyn_cast_or_null<DataLayoutOpInterface>(scope))
646 defaultMemorySpace = iface.getDefaultMemorySpace(entry);
647 else
648 defaultMemorySpace = detail::getDefaultMemorySpace(entry: entry);
649 return *defaultMemorySpace;
650}
651
652mlir::Attribute mlir::DataLayout::getAllocaMemorySpace() const {
653 checkValid();
654 if (allocaMemorySpace)
655 return *allocaMemorySpace;
656 DataLayoutEntryInterface entry;
657 if (originalLayout)
658 entry = originalLayout.getSpecForIdentifier(
659 originalLayout.getAllocaMemorySpaceIdentifier(
660 originalLayout.getContext()));
661 if (auto iface = dyn_cast_or_null<DataLayoutOpInterface>(scope))
662 allocaMemorySpace = iface.getAllocaMemorySpace(entry);
663 else
664 allocaMemorySpace = detail::getDefaultAllocaMemorySpace(entry: entry);
665 return *allocaMemorySpace;
666}
667
668mlir::Attribute mlir::DataLayout::getManglingMode() const {
669 checkValid();
670 if (manglingMode)
671 return *manglingMode;
672 DataLayoutEntryInterface entry;
673 if (originalLayout)
674 entry = originalLayout.getSpecForIdentifier(
675 originalLayout.getManglingModeIdentifier(originalLayout.getContext()));
676
677 if (auto iface = dyn_cast_or_null<DataLayoutOpInterface>(scope))
678 manglingMode = iface.getManglingMode(entry);
679 else
680 manglingMode = detail::getDefaultManglingMode(entry: entry);
681 return *manglingMode;
682}
683
684mlir::Attribute mlir::DataLayout::getProgramMemorySpace() const {
685 checkValid();
686 if (programMemorySpace)
687 return *programMemorySpace;
688 DataLayoutEntryInterface entry;
689 if (originalLayout)
690 entry = originalLayout.getSpecForIdentifier(
691 originalLayout.getProgramMemorySpaceIdentifier(
692 originalLayout.getContext()));
693 if (auto iface = dyn_cast_or_null<DataLayoutOpInterface>(scope))
694 programMemorySpace = iface.getProgramMemorySpace(entry);
695 else
696 programMemorySpace = detail::getDefaultProgramMemorySpace(entry: entry);
697 return *programMemorySpace;
698}
699
700mlir::Attribute mlir::DataLayout::getGlobalMemorySpace() const {
701 checkValid();
702 if (globalMemorySpace)
703 return *globalMemorySpace;
704 DataLayoutEntryInterface entry;
705 if (originalLayout)
706 entry = originalLayout.getSpecForIdentifier(
707 originalLayout.getGlobalMemorySpaceIdentifier(
708 originalLayout.getContext()));
709 if (auto iface = dyn_cast_or_null<DataLayoutOpInterface>(scope))
710 globalMemorySpace = iface.getGlobalMemorySpace(entry);
711 else
712 globalMemorySpace = detail::getDefaultGlobalMemorySpace(entry: entry);
713 return *globalMemorySpace;
714}
715
716uint64_t mlir::DataLayout::getStackAlignment() const {
717 checkValid();
718 if (stackAlignment)
719 return *stackAlignment;
720 DataLayoutEntryInterface entry;
721 if (originalLayout)
722 entry = originalLayout.getSpecForIdentifier(
723 originalLayout.getStackAlignmentIdentifier(
724 originalLayout.getContext()));
725 if (auto iface = dyn_cast_or_null<DataLayoutOpInterface>(scope))
726 stackAlignment = iface.getStackAlignment(entry);
727 else
728 stackAlignment = detail::getDefaultStackAlignment(entry: entry);
729 return *stackAlignment;
730}
731
732Attribute mlir::DataLayout::getFunctionPointerAlignment() const {
733 checkValid();
734 if (functionPointerAlignment)
735 return *functionPointerAlignment;
736 DataLayoutEntryInterface entry;
737 if (originalLayout)
738 entry = originalLayout.getSpecForIdentifier(
739 originalLayout.getFunctionPointerAlignmentIdentifier(
740 originalLayout.getContext()));
741 if (auto iface = dyn_cast_or_null<DataLayoutOpInterface>(scope))
742 functionPointerAlignment = iface.getFunctionPointerAlignment(entry);
743 else
744 functionPointerAlignment =
745 detail::getDefaultFunctionPointerAlignment(entry: entry);
746 return *functionPointerAlignment;
747}
748
749Attribute mlir::DataLayout::getLegalIntWidths() const {
750 checkValid();
751 if (legalIntWidths)
752 return *legalIntWidths;
753 DataLayoutEntryInterface entry;
754 if (originalLayout)
755 entry = originalLayout.getSpecForIdentifier(
756 originalLayout.getLegalIntWidthsIdentifier(
757 originalLayout.getContext()));
758 if (auto iface = dyn_cast_or_null<DataLayoutOpInterface>(scope))
759 legalIntWidths = iface.getLegalIntWidths(entry);
760 else
761 legalIntWidths = detail::getDefaultLegalIntWidths(entry: entry);
762 return *legalIntWidths;
763}
764
765std::optional<Attribute> mlir::DataLayout::getDevicePropertyValue(
766 TargetSystemSpecInterface::DeviceID deviceID,
767 StringAttr propertyName) const {
768 checkValid();
769 DataLayoutEntryInterface entry;
770 if (originalTargetSystemDesc) {
771 if (std::optional<TargetDeviceSpecInterface> device =
772 originalTargetSystemDesc.getDeviceSpecForDeviceID(deviceID))
773 entry = device->getSpecForIdentifier(propertyName);
774 }
775 // Currently I am not caching the results because we do not return
776 // default values of these properties. Instead if the property is
777 // missing, we return std::nullopt so that the users can resort to
778 // the default value however they want.
779 if (auto iface = dyn_cast_or_null<DataLayoutOpInterface>(scope))
780 return iface.getDevicePropertyValue(entry);
781 else
782 return detail::getDevicePropertyValue(entry: entry);
783}
784
785//===----------------------------------------------------------------------===//
786// DataLayoutSpecInterface
787//===----------------------------------------------------------------------===//
788
789void DataLayoutSpecInterface::bucketEntriesByType(
790 llvm::MapVector<TypeID, DataLayoutEntryList> &types,
791 llvm::MapVector<StringAttr, DataLayoutEntryInterface> &ids) {
792 for (DataLayoutEntryInterface entry : getEntries()) {
793 if (auto type = llvm::dyn_cast_if_present<Type>(entry.getKey()))
794 types[type.getTypeID()].push_back(entry);
795 else
796 ids[llvm::cast<StringAttr>(entry.getKey())] = entry;
797 }
798}
799
800LogicalResult mlir::detail::verifyDataLayoutSpec(DataLayoutSpecInterface spec,
801 Location loc) {
802 // First, verify individual entries.
803 for (DataLayoutEntryInterface entry : spec.getEntries())
804 if (failed(entry.verifyEntry(loc)))
805 return failure();
806
807 // Second, dispatch verifications of entry groups to types or dialects they
808 // are associated with.
809 llvm::MapVector<TypeID, DataLayoutEntryList> types;
810 llvm::MapVector<StringAttr, DataLayoutEntryInterface> ids;
811 spec.bucketEntriesByType(types, ids);
812
813 for (const auto &kvp : types) {
814 auto sampleType = cast<Type>(kvp.second.front().getKey());
815 if (isa<IndexType>(sampleType)) {
816 assert(kvp.second.size() == 1 &&
817 "expected one data layout entry for non-parametric 'index' type");
818 if (!isa<IntegerAttr>(kvp.second.front().getValue()))
819 return emitError(loc)
820 << "expected integer attribute in the data layout entry for "
821 << sampleType;
822 continue;
823 }
824
825 if (sampleType.isIntOrFloat()) {
826 for (DataLayoutEntryInterface entry : kvp.second) {
827 auto value = dyn_cast<DenseIntElementsAttr>(entry.getValue());
828 if (!value || !value.getElementType().isSignlessInteger(64)) {
829 emitError(loc) << "expected a dense i64 elements attribute in the "
830 "data layout entry "
831 << entry;
832 return failure();
833 }
834
835 auto elements = llvm::to_vector<2>(value.getValues<uint64_t>());
836 unsigned numElements = elements.size();
837 if (numElements < 1 || numElements > 2) {
838 emitError(loc) << "expected 1 or 2 elements in the data layout entry "
839 << entry;
840 return failure();
841 }
842
843 uint64_t abi = elements[0];
844 uint64_t preferred = numElements == 2 ? elements[1] : abi;
845 if (preferred < abi) {
846 emitError(loc)
847 << "preferred alignment is expected to be greater than or equal "
848 "to the abi alignment in data layout entry "
849 << entry;
850 return failure();
851 }
852 }
853 continue;
854 }
855
856 if (isa<BuiltinDialect>(&sampleType.getDialect()))
857 return emitError(loc) << "unexpected data layout for a built-in type";
858
859 auto dlType = dyn_cast<DataLayoutTypeInterface>(sampleType);
860 if (!dlType)
861 return emitError(loc)
862 << "data layout specified for a type that does not support it";
863 if (failed(dlType.verifyEntries(kvp.second, loc)))
864 return failure();
865 }
866
867 for (const auto &kvp : ids) {
868 StringAttr identifier = cast<StringAttr>(kvp.second.getKey());
869 Dialect *dialect = identifier.getReferencedDialect();
870
871 // Ignore attributes that belong to an unknown dialect, the dialect may
872 // actually implement the relevant interface but we don't know about that.
873 if (!dialect)
874 continue;
875
876 const auto *iface = dyn_cast<DataLayoutDialectInterface>(Val: dialect);
877 if (!iface) {
878 return emitError(loc)
879 << "the '" << dialect->getNamespace()
880 << "' dialect does not support identifier data layout entries";
881 }
882 if (failed(iface->verifyEntry(kvp.second, loc)))
883 return failure();
884 }
885
886 return success();
887}
888
889LogicalResult
890mlir::detail::verifyTargetSystemSpec(TargetSystemSpecInterface spec,
891 Location loc) {
892 DenseMap<StringAttr, DataLayoutEntryInterface> deviceDescKeys;
893 DenseSet<TargetSystemSpecInterface::DeviceID> deviceIDs;
894 for (const auto &entry : spec.getEntries()) {
895 auto targetDeviceSpec =
896 dyn_cast<TargetDeviceSpecInterface>(entry.getValue());
897
898 if (!targetDeviceSpec)
899 return failure();
900
901 // First, verify individual target device desc specs.
902 if (failed(targetDeviceSpec.verifyEntry(loc)))
903 return failure();
904
905 // Check that device IDs are unique across all entries.
906 auto deviceID =
907 llvm::dyn_cast<TargetSystemSpecInterface::DeviceID>(entry.getKey());
908 if (!deviceID)
909 return failure();
910
911 if (!deviceIDs.insert(deviceID).second) {
912 return failure();
913 }
914
915 // collect all the keys used by all the target device specs.
916 for (DataLayoutEntryInterface entry : targetDeviceSpec.getEntries()) {
917 if (auto type = llvm::dyn_cast_if_present<Type>(entry.getKey())) {
918 // targetDeviceSpec does not support Type as a key.
919 return failure();
920 } else {
921 deviceDescKeys[cast<StringAttr>(entry.getKey())] = entry;
922 }
923 }
924 }
925
926 for (const auto &[keyName, keyVal] : deviceDescKeys) {
927 Dialect *dialect = keyName.getReferencedDialect();
928
929 // Ignore attributes that belong to an unknown dialect, the dialect may
930 // actually implement the relevant interface but we don't know about that.
931 if (!dialect)
932 return failure();
933
934 const auto *iface = dyn_cast<DataLayoutDialectInterface>(Val: dialect);
935 if (!iface) {
936 return emitError(loc)
937 << "the '" << dialect->getNamespace()
938 << "' dialect does not support identifier data layout entries";
939 }
940 if (failed(iface->verifyEntry(keyVal, loc)))
941 return failure();
942 }
943
944 return success();
945}
946
947#include "mlir/Interfaces/DataLayoutAttrInterface.cpp.inc"
948#include "mlir/Interfaces/DataLayoutOpInterface.cpp.inc"
949#include "mlir/Interfaces/DataLayoutTypeInterface.cpp.inc"
950

source code of mlir/lib/Interfaces/DataLayoutInterfaces.cpp