1//===- DataLayoutInterfaces.cpp - Data Layout Interface Implementation ----===//
2//
3// Part of the LLVM Project, under the Apache License v2.0 with LLVM Exceptions.
4// See https://llvm.org/LICENSE.txt for license information.
5// SPDX-License-Identifier: Apache-2.0 WITH LLVM-exception
6//
7//===----------------------------------------------------------------------===//
8
9#include "mlir/Interfaces/DataLayoutInterfaces.h"
10#include "mlir/IR/BuiltinDialect.h"
11#include "mlir/IR/BuiltinOps.h"
12#include "mlir/IR/BuiltinTypes.h"
13#include "mlir/IR/Operation.h"
14
15#include "llvm/ADT/TypeSwitch.h"
16#include "llvm/Support/MathExtras.h"
17
18using namespace mlir;
19
20//===----------------------------------------------------------------------===//
21// Default implementations
22//===----------------------------------------------------------------------===//
23
24/// Reports that the given type is missing the data layout information and
25/// exits.
26[[noreturn]] static void reportMissingDataLayout(Type type) {
27 std::string message;
28 llvm::raw_string_ostream os(message);
29 os << "neither the scoping op nor the type class provide data layout "
30 "information for "
31 << type;
32 llvm::report_fatal_error(reason: Twine(os.str()));
33}
34
35/// Returns the bitwidth of the index type if specified in the param list.
36/// Assumes 64-bit index otherwise.
37static uint64_t getIndexBitwidth(DataLayoutEntryListRef params) {
38 if (params.empty())
39 return 64;
40 auto attr = cast<IntegerAttr>(params.front().getValue());
41 return attr.getValue().getZExtValue();
42}
43
44llvm::TypeSize
45mlir::detail::getDefaultTypeSize(Type type, const DataLayout &dataLayout,
46 ArrayRef<DataLayoutEntryInterface> params) {
47 llvm::TypeSize bits = getDefaultTypeSizeInBits(type, dataLayout, params);
48 return divideCeil(numerator: bits, denominator: 8);
49}
50
51llvm::TypeSize
52mlir::detail::getDefaultTypeSizeInBits(Type type, const DataLayout &dataLayout,
53 DataLayoutEntryListRef params) {
54 if (isa<IntegerType, FloatType>(Val: type))
55 return llvm::TypeSize::getFixed(ExactSize: type.getIntOrFloatBitWidth());
56
57 if (auto ctype = dyn_cast<ComplexType>(type)) {
58 Type et = ctype.getElementType();
59 uint64_t innerAlignment =
60 getDefaultPreferredAlignment(type: et, dataLayout, params) * 8;
61 llvm::TypeSize innerSize = getDefaultTypeSizeInBits(type: et, dataLayout, params);
62
63 // Include padding required to align the imaginary value in the complex
64 // type.
65 return llvm::alignTo(Size: innerSize, Align: innerAlignment) + innerSize;
66 }
67
68 // Index is an integer of some bitwidth.
69 if (isa<IndexType>(type))
70 return dataLayout.getTypeSizeInBits(
71 IntegerType::get(type.getContext(), getIndexBitwidth(params)));
72
73 // Sizes of vector types are rounded up to those of types with closest
74 // power-of-two number of elements in the innermost dimension. We also assume
75 // there is no bit-packing at the moment element sizes are taken in bytes and
76 // multiplied with 8 bits.
77 // TODO: make this extensible.
78 if (auto vecType = dyn_cast<VectorType>(type)) {
79 uint64_t baseSize = vecType.getNumElements() / vecType.getShape().back() *
80 llvm::PowerOf2Ceil(A: vecType.getShape().back()) *
81 dataLayout.getTypeSize(t: vecType.getElementType()) * 8;
82 return llvm::TypeSize::get(Quantity: baseSize, Scalable: vecType.isScalable());
83 }
84
85 if (auto typeInterface = dyn_cast<DataLayoutTypeInterface>(type))
86 return typeInterface.getTypeSizeInBits(dataLayout, params);
87
88 reportMissingDataLayout(type);
89}
90
91static DataLayoutEntryInterface
92findEntryForIntegerType(IntegerType intType,
93 ArrayRef<DataLayoutEntryInterface> params) {
94 assert(!params.empty() && "expected non-empty parameter list");
95 std::map<unsigned, DataLayoutEntryInterface> sortedParams;
96 for (DataLayoutEntryInterface entry : params) {
97 sortedParams.insert(std::make_pair(
98 entry.getKey().get<Type>().getIntOrFloatBitWidth(), entry));
99 }
100 auto iter = sortedParams.lower_bound(intType.getWidth());
101 if (iter == sortedParams.end())
102 iter = std::prev(iter);
103
104 return iter->second;
105}
106
107constexpr const static uint64_t kDefaultBitsInByte = 8u;
108
109static uint64_t extractABIAlignment(DataLayoutEntryInterface entry) {
110 auto values =
111 cast<DenseIntElementsAttr>(entry.getValue()).getValues<uint64_t>();
112 return static_cast<uint64_t>(*values.begin()) / kDefaultBitsInByte;
113}
114
115static uint64_t
116getIntegerTypeABIAlignment(IntegerType intType,
117 ArrayRef<DataLayoutEntryInterface> params) {
118 constexpr uint64_t kDefaultSmallIntAlignment = 4u;
119 constexpr unsigned kSmallIntSize = 64;
120 if (params.empty()) {
121 return intType.getWidth() < kSmallIntSize
122 ? llvm::PowerOf2Ceil(
123 A: llvm::divideCeil(Numerator: intType.getWidth(), Denominator: kDefaultBitsInByte))
124 : kDefaultSmallIntAlignment;
125 }
126
127 return extractABIAlignment(findEntryForIntegerType(intType, params));
128}
129
130static uint64_t
131getFloatTypeABIAlignment(FloatType fltType, const DataLayout &dataLayout,
132 ArrayRef<DataLayoutEntryInterface> params) {
133 assert(params.size() <= 1 && "at most one data layout entry is expected for "
134 "the singleton floating-point type");
135 if (params.empty())
136 return llvm::PowerOf2Ceil(A: dataLayout.getTypeSize(t: fltType).getFixedValue());
137 return extractABIAlignment(params[0]);
138}
139
140uint64_t mlir::detail::getDefaultABIAlignment(
141 Type type, const DataLayout &dataLayout,
142 ArrayRef<DataLayoutEntryInterface> params) {
143 // Natural alignment is the closest power-of-two number above. For scalable
144 // vectors, aligning them to the same as the base vector is sufficient.
145 if (isa<VectorType>(type))
146 return llvm::PowerOf2Ceil(A: dataLayout.getTypeSize(t: type).getKnownMinValue());
147
148 if (auto fltType = dyn_cast<FloatType>(Val&: type))
149 return getFloatTypeABIAlignment(fltType, dataLayout, params);
150
151 // Index is an integer of some bitwidth.
152 if (isa<IndexType>(type))
153 return dataLayout.getTypeABIAlignment(
154 IntegerType::get(type.getContext(), getIndexBitwidth(params)));
155
156 if (auto intType = dyn_cast<IntegerType>(type))
157 return getIntegerTypeABIAlignment(intType, params);
158
159 if (auto ctype = dyn_cast<ComplexType>(type))
160 return getDefaultABIAlignment(ctype.getElementType(), dataLayout, params);
161
162 if (auto typeInterface = dyn_cast<DataLayoutTypeInterface>(type))
163 return typeInterface.getABIAlignment(dataLayout, params);
164
165 reportMissingDataLayout(type);
166}
167
168static uint64_t extractPreferredAlignment(DataLayoutEntryInterface entry) {
169 auto values =
170 cast<DenseIntElementsAttr>(entry.getValue()).getValues<uint64_t>();
171 return *std::next(values.begin(), values.size() - 1) / kDefaultBitsInByte;
172}
173
174static uint64_t
175getIntegerTypePreferredAlignment(IntegerType intType,
176 const DataLayout &dataLayout,
177 ArrayRef<DataLayoutEntryInterface> params) {
178 if (params.empty())
179 return llvm::PowerOf2Ceil(A: dataLayout.getTypeSize(t: intType).getFixedValue());
180
181 return extractPreferredAlignment(findEntryForIntegerType(intType, params));
182}
183
184static uint64_t
185getFloatTypePreferredAlignment(FloatType fltType, const DataLayout &dataLayout,
186 ArrayRef<DataLayoutEntryInterface> params) {
187 assert(params.size() <= 1 && "at most one data layout entry is expected for "
188 "the singleton floating-point type");
189 if (params.empty())
190 return dataLayout.getTypeABIAlignment(t: fltType);
191 return extractPreferredAlignment(params[0]);
192}
193
194uint64_t mlir::detail::getDefaultPreferredAlignment(
195 Type type, const DataLayout &dataLayout,
196 ArrayRef<DataLayoutEntryInterface> params) {
197 // Preferred alignment is same as natural for floats and vectors.
198 if (isa<VectorType>(type))
199 return dataLayout.getTypeABIAlignment(t: type);
200
201 if (auto fltType = dyn_cast<FloatType>(Val&: type))
202 return getFloatTypePreferredAlignment(fltType, dataLayout, params);
203
204 // Preferred alignment is the closest power-of-two number above for integers
205 // (ABI alignment may be smaller).
206 if (auto intType = dyn_cast<IntegerType>(type))
207 return getIntegerTypePreferredAlignment(intType, dataLayout, params);
208
209 if (isa<IndexType>(Val: type)) {
210 return dataLayout.getTypePreferredAlignment(
211 IntegerType::get(type.getContext(), getIndexBitwidth(params)));
212 }
213
214 if (auto ctype = dyn_cast<ComplexType>(type))
215 return getDefaultPreferredAlignment(ctype.getElementType(), dataLayout,
216 params);
217
218 if (auto typeInterface = dyn_cast<DataLayoutTypeInterface>(type))
219 return typeInterface.getPreferredAlignment(dataLayout, params);
220
221 reportMissingDataLayout(type);
222}
223
224std::optional<uint64_t> mlir::detail::getDefaultIndexBitwidth(
225 Type type, const DataLayout &dataLayout,
226 ArrayRef<DataLayoutEntryInterface> params) {
227 if (isa<IndexType>(Val: type))
228 return getIndexBitwidth(params);
229
230 if (auto typeInterface = dyn_cast<DataLayoutTypeInterface>(type))
231 if (std::optional<uint64_t> indexBitwidth =
232 typeInterface.getIndexBitwidth(dataLayout, params))
233 return *indexBitwidth;
234
235 // Return std::nullopt for all other types, which are assumed to be non
236 // pointer-like types.
237 return std::nullopt;
238}
239
240// Returns the endianness if specified in the given entry. If the entry is empty
241// the default endianness represented by an empty attribute is returned.
242Attribute mlir::detail::getDefaultEndianness(DataLayoutEntryInterface entry) {
243 if (entry == DataLayoutEntryInterface())
244 return Attribute();
245
246 return entry.getValue();
247}
248
249// Returns the memory space used for alloca operations if specified in the
250// given entry. If the entry is empty the default memory space represented by
251// an empty attribute is returned.
252Attribute
253mlir::detail::getDefaultAllocaMemorySpace(DataLayoutEntryInterface entry) {
254 if (entry == DataLayoutEntryInterface()) {
255 return Attribute();
256 }
257
258 return entry.getValue();
259}
260
261// Returns the memory space used for the program memory space. if
262// specified in the given entry. If the entry is empty the default
263// memory space represented by an empty attribute is returned.
264Attribute
265mlir::detail::getDefaultProgramMemorySpace(DataLayoutEntryInterface entry) {
266 if (entry == DataLayoutEntryInterface()) {
267 return Attribute();
268 }
269
270 return entry.getValue();
271}
272
273// Returns the memory space used for global the global memory space. if
274// specified in the given entry. If the entry is empty the default memory
275// space represented by an empty attribute is returned.
276Attribute
277mlir::detail::getDefaultGlobalMemorySpace(DataLayoutEntryInterface entry) {
278 if (entry == DataLayoutEntryInterface()) {
279 return Attribute();
280 }
281
282 return entry.getValue();
283}
284
285// Returns the stack alignment if specified in the given entry. If the entry is
286// empty the default alignment zero is returned.
287uint64_t
288mlir::detail::getDefaultStackAlignment(DataLayoutEntryInterface entry) {
289 if (entry == DataLayoutEntryInterface())
290 return 0;
291
292 auto value = cast<IntegerAttr>(entry.getValue());
293 return value.getValue().getZExtValue();
294}
295
296DataLayoutEntryList
297mlir::detail::filterEntriesForType(DataLayoutEntryListRef entries,
298 TypeID typeID) {
299 return llvm::to_vector<4>(Range: llvm::make_filter_range(
300 Range&: entries, Pred: [typeID](DataLayoutEntryInterface entry) {
301 auto type = llvm::dyn_cast_if_present<Type>(entry.getKey());
302 return type && type.getTypeID() == typeID;
303 }));
304}
305
306DataLayoutEntryInterface
307mlir::detail::filterEntryForIdentifier(DataLayoutEntryListRef entries,
308 StringAttr id) {
309 const auto *it = llvm::find_if(Range&: entries, P: [id](DataLayoutEntryInterface entry) {
310 if (!entry.getKey().is<StringAttr>())
311 return false;
312 return entry.getKey().get<StringAttr>() == id;
313 });
314 return it == entries.end() ? DataLayoutEntryInterface() : *it;
315}
316
317static DataLayoutSpecInterface getSpec(Operation *operation) {
318 return llvm::TypeSwitch<Operation *, DataLayoutSpecInterface>(operation)
319 .Case<ModuleOp, DataLayoutOpInterface>(
320 [&](auto op) { return op.getDataLayoutSpec(); })
321 .Default([](Operation *) {
322 llvm_unreachable("expected an op with data layout spec");
323 return DataLayoutSpecInterface();
324 });
325}
326
327/// Populates `opsWithLayout` with the list of proper ancestors of `leaf` that
328/// are either modules or implement the `DataLayoutOpInterface`.
329static void
330collectParentLayouts(Operation *leaf,
331 SmallVectorImpl<DataLayoutSpecInterface> &specs,
332 SmallVectorImpl<Location> *opLocations = nullptr) {
333 if (!leaf)
334 return;
335
336 for (Operation *parent = leaf->getParentOp(); parent != nullptr;
337 parent = parent->getParentOp()) {
338 llvm::TypeSwitch<Operation *>(parent)
339 .Case<ModuleOp>(caseFn: [&](ModuleOp op) {
340 // Skip top-level module op unless it has a layout. Top-level module
341 // without layout is most likely the one implicitly added by the
342 // parser and it doesn't have location. Top-level null specification
343 // would have had the same effect as not having a specification at all
344 // (using type defaults).
345 if (!op->getParentOp() && !op.getDataLayoutSpec())
346 return;
347 specs.push_back(op.getDataLayoutSpec());
348 if (opLocations)
349 opLocations->push_back(Elt: op.getLoc());
350 })
351 .Case<DataLayoutOpInterface>(caseFn: [&](DataLayoutOpInterface op) {
352 specs.push_back(op.getDataLayoutSpec());
353 if (opLocations)
354 opLocations->push_back(Elt: op.getLoc());
355 });
356 }
357}
358
359/// Returns a layout spec that is a combination of the layout specs attached
360/// to the given operation and all its ancestors.
361static DataLayoutSpecInterface getCombinedDataLayout(Operation *leaf) {
362 if (!leaf)
363 return {};
364
365 assert((isa<ModuleOp, DataLayoutOpInterface>(leaf)) &&
366 "expected an op with data layout spec");
367
368 SmallVector<DataLayoutOpInterface> opsWithLayout;
369 SmallVector<DataLayoutSpecInterface> specs;
370 collectParentLayouts(leaf, specs);
371
372 // Fast track if there are no ancestors.
373 if (specs.empty())
374 return getSpec(leaf);
375
376 // Create the list of non-null specs (null/missing specs can be safely
377 // ignored) from the outermost to the innermost.
378 auto nonNullSpecs = llvm::to_vector<2>(llvm::make_filter_range(
379 llvm::reverse(specs),
380 [](DataLayoutSpecInterface iface) { return iface != nullptr; }));
381
382 // Combine the specs using the innermost as anchor.
383 if (DataLayoutSpecInterface current = getSpec(leaf))
384 return current.combineWith(nonNullSpecs);
385 if (nonNullSpecs.empty())
386 return {};
387 return nonNullSpecs.back().combineWith(
388 llvm::ArrayRef(nonNullSpecs).drop_back());
389}
390
391LogicalResult mlir::detail::verifyDataLayoutOp(Operation *op) {
392 DataLayoutSpecInterface spec = getSpec(op);
393 // The layout specification may be missing and it's fine.
394 if (!spec)
395 return success();
396
397 if (failed(spec.verifySpec(op->getLoc())))
398 return failure();
399 if (!getCombinedDataLayout(op)) {
400 InFlightDiagnostic diag =
401 op->emitError()
402 << "data layout does not combine with layouts of enclosing ops";
403 SmallVector<DataLayoutSpecInterface> specs;
404 SmallVector<Location> opLocations;
405 collectParentLayouts(op, specs, &opLocations);
406 for (Location loc : opLocations)
407 diag.attachNote(noteLoc: loc) << "enclosing op with data layout";
408 return diag;
409 }
410 return success();
411}
412
413llvm::TypeSize mlir::detail::divideCeil(llvm::TypeSize numerator,
414 uint64_t denominator) {
415 uint64_t divided =
416 llvm::divideCeil(Numerator: numerator.getKnownMinValue(), Denominator: denominator);
417 return llvm::TypeSize::get(Quantity: divided, Scalable: numerator.isScalable());
418}
419
420//===----------------------------------------------------------------------===//
421// DataLayout
422//===----------------------------------------------------------------------===//
423
424template <typename OpTy>
425void checkMissingLayout(DataLayoutSpecInterface originalLayout, OpTy op) {
426 if (!originalLayout) {
427 assert((!op || !op.getDataLayoutSpec()) &&
428 "could not compute layout information for an op (failed to "
429 "combine attributes?)");
430 }
431}
432
433mlir::DataLayout::DataLayout() : DataLayout(ModuleOp()) {}
434
435mlir::DataLayout::DataLayout(DataLayoutOpInterface op)
436 : originalLayout(getCombinedDataLayout(op)), scope(op),
437 allocaMemorySpace(std::nullopt), programMemorySpace(std::nullopt),
438 globalMemorySpace(std::nullopt), stackAlignment(std::nullopt) {
439#if LLVM_ENABLE_ABI_BREAKING_CHECKS
440 checkMissingLayout(originalLayout, op);
441 collectParentLayouts(op, layoutStack);
442#endif
443}
444
445mlir::DataLayout::DataLayout(ModuleOp op)
446 : originalLayout(getCombinedDataLayout(op)), scope(op),
447 allocaMemorySpace(std::nullopt), programMemorySpace(std::nullopt),
448 globalMemorySpace(std::nullopt), stackAlignment(std::nullopt) {
449#if LLVM_ENABLE_ABI_BREAKING_CHECKS
450 checkMissingLayout(originalLayout, op);
451 collectParentLayouts(op, layoutStack);
452#endif
453}
454
455mlir::DataLayout mlir::DataLayout::closest(Operation *op) {
456 // Search the closest parent either being a module operation or implementing
457 // the data layout interface.
458 while (op) {
459 if (auto module = dyn_cast<ModuleOp>(op))
460 return DataLayout(module);
461 if (auto iface = dyn_cast<DataLayoutOpInterface>(op))
462 return DataLayout(iface);
463 op = op->getParentOp();
464 }
465 return DataLayout();
466}
467
468void mlir::DataLayout::checkValid() const {
469#if LLVM_ENABLE_ABI_BREAKING_CHECKS
470 SmallVector<DataLayoutSpecInterface> specs;
471 collectParentLayouts(scope, specs);
472 assert(specs.size() == layoutStack.size() &&
473 "data layout object used, but no longer valid due to the change in "
474 "number of nested layouts");
475 for (auto pair : llvm::zip(specs, layoutStack)) {
476 Attribute newLayout = std::get<0>(pair);
477 Attribute origLayout = std::get<1>(pair);
478 assert(newLayout == origLayout &&
479 "data layout object used, but no longer valid "
480 "due to the change in layout attributes");
481 }
482#endif
483 assert(((!scope && !this->originalLayout) ||
484 (scope && this->originalLayout == getCombinedDataLayout(scope))) &&
485 "data layout object used, but no longer valid due to the change in "
486 "layout spec");
487}
488
489/// Looks up the value for the given type key in the given cache. If there is no
490/// such value in the cache, compute it using the given callback and put it in
491/// the cache before returning.
492template <typename T>
493static T cachedLookup(Type t, DenseMap<Type, T> &cache,
494 function_ref<T(Type)> compute) {
495 auto it = cache.find(t);
496 if (it != cache.end())
497 return it->second;
498
499 auto result = cache.try_emplace(t, compute(t));
500 return result.first->second;
501}
502
503llvm::TypeSize mlir::DataLayout::getTypeSize(Type t) const {
504 checkValid();
505 return cachedLookup<llvm::TypeSize>(t, sizes, [&](Type ty) {
506 DataLayoutEntryList list;
507 if (originalLayout)
508 list = originalLayout.getSpecForType(ty.getTypeID());
509 if (auto iface = dyn_cast_or_null<DataLayoutOpInterface>(scope))
510 return iface.getTypeSize(ty, *this, list);
511 return detail::getDefaultTypeSize(ty, *this, list);
512 });
513}
514
515llvm::TypeSize mlir::DataLayout::getTypeSizeInBits(Type t) const {
516 checkValid();
517 return cachedLookup<llvm::TypeSize>(t, bitsizes, [&](Type ty) {
518 DataLayoutEntryList list;
519 if (originalLayout)
520 list = originalLayout.getSpecForType(ty.getTypeID());
521 if (auto iface = dyn_cast_or_null<DataLayoutOpInterface>(scope))
522 return iface.getTypeSizeInBits(ty, *this, list);
523 return detail::getDefaultTypeSizeInBits(ty, *this, list);
524 });
525}
526
527uint64_t mlir::DataLayout::getTypeABIAlignment(Type t) const {
528 checkValid();
529 return cachedLookup<uint64_t>(t, abiAlignments, [&](Type ty) {
530 DataLayoutEntryList list;
531 if (originalLayout)
532 list = originalLayout.getSpecForType(ty.getTypeID());
533 if (auto iface = dyn_cast_or_null<DataLayoutOpInterface>(scope))
534 return iface.getTypeABIAlignment(ty, *this, list);
535 return detail::getDefaultABIAlignment(ty, *this, list);
536 });
537}
538
539uint64_t mlir::DataLayout::getTypePreferredAlignment(Type t) const {
540 checkValid();
541 return cachedLookup<uint64_t>(t, preferredAlignments, [&](Type ty) {
542 DataLayoutEntryList list;
543 if (originalLayout)
544 list = originalLayout.getSpecForType(ty.getTypeID());
545 if (auto iface = dyn_cast_or_null<DataLayoutOpInterface>(scope))
546 return iface.getTypePreferredAlignment(ty, *this, list);
547 return detail::getDefaultPreferredAlignment(ty, *this, list);
548 });
549}
550
551std::optional<uint64_t> mlir::DataLayout::getTypeIndexBitwidth(Type t) const {
552 checkValid();
553 return cachedLookup<std::optional<uint64_t>>(t, indexBitwidths, [&](Type ty) {
554 DataLayoutEntryList list;
555 if (originalLayout)
556 list = originalLayout.getSpecForType(ty.getTypeID());
557 if (auto iface = dyn_cast_or_null<DataLayoutOpInterface>(scope))
558 return iface.getIndexBitwidth(ty, *this, list);
559 return detail::getDefaultIndexBitwidth(ty, *this, list);
560 });
561}
562
563mlir::Attribute mlir::DataLayout::getEndianness() const {
564 checkValid();
565 if (endianness)
566 return *endianness;
567 DataLayoutEntryInterface entry;
568 if (originalLayout)
569 entry = originalLayout.getSpecForIdentifier(
570 originalLayout.getEndiannessIdentifier(originalLayout.getContext()));
571
572 if (auto iface = dyn_cast_or_null<DataLayoutOpInterface>(scope))
573 endianness = iface.getEndianness(entry);
574 else
575 endianness = detail::getDefaultEndianness(entry: entry);
576 return *endianness;
577}
578
579mlir::Attribute mlir::DataLayout::getAllocaMemorySpace() const {
580 checkValid();
581 if (allocaMemorySpace)
582 return *allocaMemorySpace;
583 DataLayoutEntryInterface entry;
584 if (originalLayout)
585 entry = originalLayout.getSpecForIdentifier(
586 originalLayout.getAllocaMemorySpaceIdentifier(
587 originalLayout.getContext()));
588 if (auto iface = dyn_cast_or_null<DataLayoutOpInterface>(scope))
589 allocaMemorySpace = iface.getAllocaMemorySpace(entry);
590 else
591 allocaMemorySpace = detail::getDefaultAllocaMemorySpace(entry: entry);
592 return *allocaMemorySpace;
593}
594
595mlir::Attribute mlir::DataLayout::getProgramMemorySpace() const {
596 checkValid();
597 if (programMemorySpace)
598 return *programMemorySpace;
599 DataLayoutEntryInterface entry;
600 if (originalLayout)
601 entry = originalLayout.getSpecForIdentifier(
602 originalLayout.getProgramMemorySpaceIdentifier(
603 originalLayout.getContext()));
604 if (auto iface = dyn_cast_or_null<DataLayoutOpInterface>(scope))
605 programMemorySpace = iface.getProgramMemorySpace(entry);
606 else
607 programMemorySpace = detail::getDefaultProgramMemorySpace(entry: entry);
608 return *programMemorySpace;
609}
610
611mlir::Attribute mlir::DataLayout::getGlobalMemorySpace() const {
612 checkValid();
613 if (globalMemorySpace)
614 return *globalMemorySpace;
615 DataLayoutEntryInterface entry;
616 if (originalLayout)
617 entry = originalLayout.getSpecForIdentifier(
618 originalLayout.getGlobalMemorySpaceIdentifier(
619 originalLayout.getContext()));
620 if (auto iface = dyn_cast_or_null<DataLayoutOpInterface>(scope))
621 globalMemorySpace = iface.getGlobalMemorySpace(entry);
622 else
623 globalMemorySpace = detail::getDefaultGlobalMemorySpace(entry: entry);
624 return *globalMemorySpace;
625}
626
627uint64_t mlir::DataLayout::getStackAlignment() const {
628 checkValid();
629 if (stackAlignment)
630 return *stackAlignment;
631 DataLayoutEntryInterface entry;
632 if (originalLayout)
633 entry = originalLayout.getSpecForIdentifier(
634 originalLayout.getStackAlignmentIdentifier(
635 originalLayout.getContext()));
636 if (auto iface = dyn_cast_or_null<DataLayoutOpInterface>(scope))
637 stackAlignment = iface.getStackAlignment(entry);
638 else
639 stackAlignment = detail::getDefaultStackAlignment(entry: entry);
640 return *stackAlignment;
641}
642
643//===----------------------------------------------------------------------===//
644// DataLayoutSpecInterface
645//===----------------------------------------------------------------------===//
646
647void DataLayoutSpecInterface::bucketEntriesByType(
648 DenseMap<TypeID, DataLayoutEntryList> &types,
649 DenseMap<StringAttr, DataLayoutEntryInterface> &ids) {
650 for (DataLayoutEntryInterface entry : getEntries()) {
651 if (auto type = llvm::dyn_cast_if_present<Type>(entry.getKey()))
652 types[type.getTypeID()].push_back(entry);
653 else
654 ids[entry.getKey().get<StringAttr>()] = entry;
655 }
656}
657
658LogicalResult mlir::detail::verifyDataLayoutSpec(DataLayoutSpecInterface spec,
659 Location loc) {
660 // First, verify individual entries.
661 for (DataLayoutEntryInterface entry : spec.getEntries())
662 if (failed(entry.verifyEntry(loc)))
663 return failure();
664
665 // Second, dispatch verifications of entry groups to types or dialects they
666 // are associated with.
667 DenseMap<TypeID, DataLayoutEntryList> types;
668 DenseMap<StringAttr, DataLayoutEntryInterface> ids;
669 spec.bucketEntriesByType(types, ids);
670
671 for (const auto &kvp : types) {
672 auto sampleType = kvp.second.front().getKey().get<Type>();
673 if (isa<IndexType>(sampleType)) {
674 assert(kvp.second.size() == 1 &&
675 "expected one data layout entry for non-parametric 'index' type");
676 if (!isa<IntegerAttr>(kvp.second.front().getValue()))
677 return emitError(loc)
678 << "expected integer attribute in the data layout entry for "
679 << sampleType;
680 continue;
681 }
682
683 if (isa<IntegerType, FloatType>(sampleType)) {
684 for (DataLayoutEntryInterface entry : kvp.second) {
685 auto value = dyn_cast<DenseIntElementsAttr>(entry.getValue());
686 if (!value || !value.getElementType().isSignlessInteger(64)) {
687 emitError(loc) << "expected a dense i64 elements attribute in the "
688 "data layout entry "
689 << entry;
690 return failure();
691 }
692
693 auto elements = llvm::to_vector<2>(value.getValues<uint64_t>());
694 unsigned numElements = elements.size();
695 if (numElements < 1 || numElements > 2) {
696 emitError(loc) << "expected 1 or 2 elements in the data layout entry "
697 << entry;
698 return failure();
699 }
700
701 uint64_t abi = elements[0];
702 uint64_t preferred = numElements == 2 ? elements[1] : abi;
703 if (preferred < abi) {
704 emitError(loc)
705 << "preferred alignment is expected to be greater than or equal "
706 "to the abi alignment in data layout entry "
707 << entry;
708 return failure();
709 }
710 }
711 continue;
712 }
713
714 if (isa<BuiltinDialect>(&sampleType.getDialect()))
715 return emitError(loc) << "unexpected data layout for a built-in type";
716
717 auto dlType = dyn_cast<DataLayoutTypeInterface>(sampleType);
718 if (!dlType)
719 return emitError(loc)
720 << "data layout specified for a type that does not support it";
721 if (failed(dlType.verifyEntries(kvp.second, loc)))
722 return failure();
723 }
724
725 for (const auto &kvp : ids) {
726 StringAttr identifier = kvp.second.getKey().get<StringAttr>();
727 Dialect *dialect = identifier.getReferencedDialect();
728
729 // Ignore attributes that belong to an unknown dialect, the dialect may
730 // actually implement the relevant interface but we don't know about that.
731 if (!dialect)
732 continue;
733
734 const auto *iface = dyn_cast<DataLayoutDialectInterface>(Val: dialect);
735 if (!iface) {
736 return emitError(loc)
737 << "the '" << dialect->getNamespace()
738 << "' dialect does not support identifier data layout entries";
739 }
740 if (failed(iface->verifyEntry(kvp.second, loc)))
741 return failure();
742 }
743
744 return success();
745}
746
747#include "mlir/Interfaces/DataLayoutAttrInterface.cpp.inc"
748#include "mlir/Interfaces/DataLayoutOpInterface.cpp.inc"
749#include "mlir/Interfaces/DataLayoutTypeInterface.cpp.inc"
750

source code of mlir/lib/Interfaces/DataLayoutInterfaces.cpp