15 #include "llvm/ADT/TypeSwitch.h"
16 #include "llvm/Support/MathExtras.h"
28 llvm::raw_string_ostream os(message);
29 os <<
"neither the scoping op nor the type class provide data layout "
32 llvm::report_fatal_error(Twine(os.str()));
40 auto attr = cast<IntegerAttr>(params.front().getValue());
41 return attr.getValue().getZExtValue();
54 if (isa<IntegerType, FloatType>(type))
57 if (
auto ctype = dyn_cast<ComplexType>(type)) {
58 Type et = ctype.getElementType();
59 uint64_t innerAlignment =
65 return llvm::alignTo(innerSize, innerAlignment) + innerSize;
69 if (isa<IndexType>(type))
78 if (
auto vecType = dyn_cast<VectorType>(type)) {
79 uint64_t baseSize = vecType.getNumElements() / vecType.getShape().back() *
80 llvm::PowerOf2Ceil(vecType.getShape().back()) *
81 dataLayout.
getTypeSize(vecType.getElementType()) * 8;
85 if (
auto typeInterface = dyn_cast<DataLayoutTypeInterface>(type))
86 return typeInterface.getTypeSizeInBits(dataLayout, params);
91 static DataLayoutEntryInterface
94 assert(!params.empty() &&
"expected non-empty parameter list");
95 std::map<unsigned, DataLayoutEntryInterface> sortedParams;
96 for (DataLayoutEntryInterface entry : params) {
97 sortedParams.insert(std::make_pair(
100 auto iter = sortedParams.lower_bound(intType.getWidth());
101 if (iter == sortedParams.end())
102 iter = std::prev(iter);
111 cast<DenseIntElementsAttr>(entry.getValue()).getValues<uint64_t>();
118 constexpr uint64_t kDefaultSmallIntAlignment = 4u;
119 constexpr
unsigned kSmallIntSize = 64;
120 if (params.empty()) {
121 return intType.getWidth() < kSmallIntSize
122 ? llvm::PowerOf2Ceil(
124 : kDefaultSmallIntAlignment;
133 assert(params.size() <= 1 &&
"at most one data layout entry is expected for "
134 "the singleton floating-point type");
136 return llvm::PowerOf2Ceil(dataLayout.
getTypeSize(fltType).getFixedValue());
145 if (isa<VectorType>(type))
146 return llvm::PowerOf2Ceil(dataLayout.
getTypeSize(type).getKnownMinValue());
148 if (
auto fltType = dyn_cast<FloatType>(type))
152 if (isa<IndexType>(type))
156 if (
auto intType = dyn_cast<IntegerType>(type))
159 if (
auto ctype = dyn_cast<ComplexType>(type))
162 if (
auto typeInterface = dyn_cast<DataLayoutTypeInterface>(type))
163 return typeInterface.getABIAlignment(dataLayout, params);
170 cast<DenseIntElementsAttr>(entry.getValue()).getValues<uint64_t>();
179 return llvm::PowerOf2Ceil(dataLayout.
getTypeSize(intType).getFixedValue());
187 assert(params.size() <= 1 &&
"at most one data layout entry is expected for "
188 "the singleton floating-point type");
198 if (isa<VectorType>(type))
201 if (
auto fltType = dyn_cast<FloatType>(type))
206 if (
auto intType = dyn_cast<IntegerType>(type))
209 if (isa<IndexType>(type)) {
214 if (
auto ctype = dyn_cast<ComplexType>(type))
218 if (
auto typeInterface = dyn_cast<DataLayoutTypeInterface>(type))
219 return typeInterface.getPreferredAlignment(dataLayout, params);
227 if (isa<IndexType>(type))
230 if (
auto typeInterface = dyn_cast<DataLayoutTypeInterface>(type))
231 if (std::optional<uint64_t> indexBitwidth =
232 typeInterface.getIndexBitwidth(dataLayout, params))
233 return *indexBitwidth;
243 if (entry == DataLayoutEntryInterface())
246 return entry.getValue();
254 if (entry == DataLayoutEntryInterface()) {
258 return entry.getValue();
266 if (entry == DataLayoutEntryInterface()) {
270 return entry.getValue();
278 if (entry == DataLayoutEntryInterface()) {
282 return entry.getValue();
289 if (entry == DataLayoutEntryInterface())
292 auto value = cast<IntegerAttr>(entry.getValue());
293 return value.getValue().getZExtValue();
299 return llvm::to_vector<4>(llvm::make_filter_range(
300 entries, [typeID](DataLayoutEntryInterface entry) {
301 auto type = llvm::dyn_cast_if_present<Type>(entry.getKey());
302 return type && type.getTypeID() == typeID;
306 DataLayoutEntryInterface
309 const auto *it = llvm::find_if(entries, [
id](DataLayoutEntryInterface entry) {
310 if (!entry.getKey().is<StringAttr>())
312 return entry.getKey().get<StringAttr>() ==
id;
314 return it == entries.end() ? DataLayoutEntryInterface() : *it;
319 .Case<ModuleOp, DataLayoutOpInterface>(
320 [&](
auto op) {
return op.getDataLayoutSpec(); })
322 llvm_unreachable(
"expected an op with data layout spec");
323 return DataLayoutSpecInterface();
339 .Case<ModuleOp>([&](ModuleOp op) {
347 specs.push_back(op.getDataLayoutSpec());
349 opLocations->push_back(op.
getLoc());
351 .Case<DataLayoutOpInterface>([&](DataLayoutOpInterface op) {
352 specs.push_back(op.getDataLayoutSpec());
354 opLocations->push_back(op.
getLoc());
365 assert((isa<ModuleOp, DataLayoutOpInterface>(leaf)) &&
366 "expected an op with data layout spec");
378 auto nonNullSpecs = llvm::to_vector<2>(llvm::make_filter_range(
379 llvm::reverse(specs),
380 [](DataLayoutSpecInterface iface) {
return iface !=
nullptr; }));
383 if (DataLayoutSpecInterface current =
getSpec(leaf))
384 return current.combineWith(nonNullSpecs);
385 if (nonNullSpecs.empty())
387 return nonNullSpecs.back().combineWith(
392 DataLayoutSpecInterface spec =
getSpec(op);
402 <<
"data layout does not combine with layouts of enclosing ops";
407 diag.attachNote(loc) <<
"enclosing op with data layout";
414 uint64_t denominator) {
424 template <
typename OpTy>
426 if (!originalLayout) {
427 assert((!op || !op.getDataLayoutSpec()) &&
428 "could not compute layout information for an op (failed to "
429 "combine attributes?)");
437 allocaMemorySpace(std::nullopt), programMemorySpace(std::nullopt),
438 globalMemorySpace(std::nullopt), stackAlignment(std::nullopt) {
439 #if LLVM_ENABLE_ABI_BREAKING_CHECKS
447 allocaMemorySpace(std::nullopt), programMemorySpace(std::nullopt),
448 globalMemorySpace(std::nullopt), stackAlignment(std::nullopt) {
449 #if LLVM_ENABLE_ABI_BREAKING_CHECKS
459 if (
auto module = dyn_cast<ModuleOp>(op))
461 if (
auto iface = dyn_cast<DataLayoutOpInterface>(op))
468 void mlir::DataLayout::checkValid()
const {
469 #if LLVM_ENABLE_ABI_BREAKING_CHECKS
472 assert(specs.size() == layoutStack.size() &&
473 "data layout object used, but no longer valid due to the change in "
474 "number of nested layouts");
475 for (
auto pair : llvm::zip(specs, layoutStack)) {
477 Attribute origLayout = std::get<1>(pair);
478 assert(newLayout == origLayout &&
479 "data layout object used, but no longer valid "
480 "due to the change in layout attributes");
483 assert(((!scope && !this->originalLayout) ||
485 "data layout object used, but no longer valid due to the change in "
492 template <
typename T>
495 auto it = cache.find(t);
496 if (it != cache.end())
499 auto result = cache.try_emplace(t, compute(t));
500 return result.first->second;
505 return cachedLookup<llvm::TypeSize>(t, sizes, [&](
Type ty) {
508 list = originalLayout.getSpecForType(ty.
getTypeID());
509 if (
auto iface = dyn_cast_or_null<DataLayoutOpInterface>(scope))
510 return iface.getTypeSize(ty, *
this, list);
517 return cachedLookup<llvm::TypeSize>(t, bitsizes, [&](
Type ty) {
520 list = originalLayout.getSpecForType(ty.
getTypeID());
521 if (
auto iface = dyn_cast_or_null<DataLayoutOpInterface>(scope))
522 return iface.getTypeSizeInBits(ty, *
this, list);
529 return cachedLookup<uint64_t>(t, abiAlignments, [&](
Type ty) {
532 list = originalLayout.getSpecForType(ty.
getTypeID());
533 if (
auto iface = dyn_cast_or_null<DataLayoutOpInterface>(scope))
534 return iface.getTypeABIAlignment(ty, *
this, list);
541 return cachedLookup<uint64_t>(t, preferredAlignments, [&](
Type ty) {
544 list = originalLayout.getSpecForType(ty.
getTypeID());
545 if (
auto iface = dyn_cast_or_null<DataLayoutOpInterface>(scope))
546 return iface.getTypePreferredAlignment(ty, *
this, list);
553 return cachedLookup<std::optional<uint64_t>>(t, indexBitwidths, [&](
Type ty) {
556 list = originalLayout.getSpecForType(ty.getTypeID());
557 if (
auto iface = dyn_cast_or_null<DataLayoutOpInterface>(scope))
558 return iface.getIndexBitwidth(ty, *
this, list);
567 DataLayoutEntryInterface entry;
569 entry = originalLayout.getSpecForIdentifier(
570 originalLayout.getEndiannessIdentifier(originalLayout.getContext()));
572 if (
auto iface = dyn_cast_or_null<DataLayoutOpInterface>(scope))
573 endianness = iface.getEndianness(entry);
581 if (allocaMemorySpace)
582 return *allocaMemorySpace;
583 DataLayoutEntryInterface entry;
585 entry = originalLayout.getSpecForIdentifier(
586 originalLayout.getAllocaMemorySpaceIdentifier(
587 originalLayout.getContext()));
588 if (
auto iface = dyn_cast_or_null<DataLayoutOpInterface>(scope))
589 allocaMemorySpace = iface.getAllocaMemorySpace(entry);
592 return *allocaMemorySpace;
597 if (programMemorySpace)
598 return *programMemorySpace;
599 DataLayoutEntryInterface entry;
601 entry = originalLayout.getSpecForIdentifier(
602 originalLayout.getProgramMemorySpaceIdentifier(
603 originalLayout.getContext()));
604 if (
auto iface = dyn_cast_or_null<DataLayoutOpInterface>(scope))
605 programMemorySpace = iface.getProgramMemorySpace(entry);
608 return *programMemorySpace;
613 if (globalMemorySpace)
614 return *globalMemorySpace;
615 DataLayoutEntryInterface entry;
617 entry = originalLayout.getSpecForIdentifier(
618 originalLayout.getGlobalMemorySpaceIdentifier(
619 originalLayout.getContext()));
620 if (
auto iface = dyn_cast_or_null<DataLayoutOpInterface>(scope))
621 globalMemorySpace = iface.getGlobalMemorySpace(entry);
624 return *globalMemorySpace;
630 return *stackAlignment;
631 DataLayoutEntryInterface entry;
633 entry = originalLayout.getSpecForIdentifier(
634 originalLayout.getStackAlignmentIdentifier(
635 originalLayout.getContext()));
636 if (
auto iface = dyn_cast_or_null<DataLayoutOpInterface>(scope))
637 stackAlignment = iface.getStackAlignment(entry);
640 return *stackAlignment;
647 void DataLayoutSpecInterface::bucketEntriesByType(
650 for (DataLayoutEntryInterface entry : getEntries()) {
651 if (
auto type = llvm::dyn_cast_if_present<Type>(entry.getKey()))
652 types[type.getTypeID()].push_back(entry);
654 ids[entry.getKey().get<StringAttr>()] = entry;
661 for (DataLayoutEntryInterface entry : spec.getEntries())
662 if (
failed(entry.verifyEntry(loc)))
669 spec.bucketEntriesByType(types, ids);
671 for (
const auto &kvp : types) {
672 auto sampleType = kvp.second.front().getKey().get<
Type>();
673 if (isa<IndexType>(sampleType)) {
674 assert(kvp.second.size() == 1 &&
675 "expected one data layout entry for non-parametric 'index' type");
676 if (!isa<IntegerAttr>(kvp.second.front().getValue()))
678 <<
"expected integer attribute in the data layout entry for "
683 if (isa<IntegerType, FloatType>(sampleType)) {
684 for (DataLayoutEntryInterface entry : kvp.second) {
685 auto value = dyn_cast<DenseIntElementsAttr>(entry.getValue());
686 if (!value || !value.getElementType().isSignlessInteger(64)) {
687 emitError(loc) <<
"expected a dense i64 elements attribute in the "
693 auto elements = llvm::to_vector<2>(value.getValues<uint64_t>());
694 unsigned numElements = elements.size();
695 if (numElements < 1 || numElements > 2) {
696 emitError(loc) <<
"expected 1 or 2 elements in the data layout entry "
701 uint64_t abi = elements[0];
702 uint64_t preferred = numElements == 2 ? elements[1] : abi;
703 if (preferred < abi) {
705 <<
"preferred alignment is expected to be greater than or equal "
706 "to the abi alignment in data layout entry "
714 if (isa<BuiltinDialect>(&sampleType.getDialect()))
715 return emitError(loc) <<
"unexpected data layout for a built-in type";
717 auto dlType = dyn_cast<DataLayoutTypeInterface>(sampleType);
720 <<
"data layout specified for a type that does not support it";
721 if (
failed(dlType.verifyEntries(kvp.second, loc)))
725 for (
const auto &kvp : ids) {
726 StringAttr identifier = kvp.second.getKey().get<StringAttr>();
727 Dialect *dialect = identifier.getReferencedDialect();
734 const auto *iface = dyn_cast<DataLayoutDialectInterface>(dialect);
737 <<
"the '" << dialect->getNamespace()
738 <<
"' dialect does not support identifier data layout entries";
740 if (
failed(iface->verifyEntry(kvp.second, loc)))
747 #include "mlir/Interfaces/DataLayoutAttrInterface.cpp.inc"
748 #include "mlir/Interfaces/DataLayoutOpInterface.cpp.inc"
749 #include "mlir/Interfaces/DataLayoutTypeInterface.cpp.inc"
static uint64_t getIntegerTypePreferredAlignment(IntegerType intType, const DataLayout &dataLayout, ArrayRef< DataLayoutEntryInterface > params)
static DataLayoutSpecInterface getCombinedDataLayout(Operation *leaf)
Returns a layout spec that is a combination of the layout specs attached to the given operation and a...
constexpr static const uint64_t kDefaultBitsInByte
static uint64_t getIntegerTypeABIAlignment(IntegerType intType, ArrayRef< DataLayoutEntryInterface > params)
static T cachedLookup(Type t, DenseMap< Type, T > &cache, function_ref< T(Type)> compute)
Looks up the value for the given type key in the given cache.
static uint64_t extractPreferredAlignment(DataLayoutEntryInterface entry)
static void collectParentLayouts(Operation *leaf, SmallVectorImpl< DataLayoutSpecInterface > &specs, SmallVectorImpl< Location > *opLocations=nullptr)
Populates opsWithLayout with the list of proper ancestors of leaf that are either modules or implemen...
static uint64_t getFloatTypePreferredAlignment(FloatType fltType, const DataLayout &dataLayout, ArrayRef< DataLayoutEntryInterface > params)
static uint64_t getFloatTypeABIAlignment(FloatType fltType, const DataLayout &dataLayout, ArrayRef< DataLayoutEntryInterface > params)
static DataLayoutSpecInterface getSpec(Operation *operation)
static uint64_t getIndexBitwidth(DataLayoutEntryListRef params)
Returns the bitwidth of the index type if specified in the param list.
static uint64_t extractABIAlignment(DataLayoutEntryInterface entry)
static DataLayoutEntryInterface findEntryForIntegerType(IntegerType intType, ArrayRef< DataLayoutEntryInterface > params)
void checkMissingLayout(DataLayoutSpecInterface originalLayout, OpTy op)
static void reportMissingDataLayout(Type type)
Reports that the given type is missing the data layout information and exits.
static std::string diag(const llvm::Value &value)
Attributes are known-constant values of operations.
The main mechanism for performing data layout queries.
Attribute getAllocaMemorySpace() const
Returns the memory space used for AllocaOps.
static DataLayout closest(Operation *op)
Returns the layout of the closest parent operation carrying layout info.
std::optional< uint64_t > getTypeIndexBitwidth(Type t) const
Returns the bitwidth that should be used when performing index computations for the given pointer-lik...
llvm::TypeSize getTypeSize(Type t) const
Returns the size of the given type in the current scope.
uint64_t getStackAlignment() const
Returns the natural alignment of the stack in bits.
Attribute getProgramMemorySpace() const
Returns the memory space used for program memory operations.
uint64_t getTypePreferredAlignment(Type t) const
Returns the preferred of the given type in the current scope.
Attribute getGlobalMemorySpace() const
Returns the memory space used for global operations.
uint64_t getTypeABIAlignment(Type t) const
Returns the required alignment of the given type in the current scope.
llvm::TypeSize getTypeSizeInBits(Type t) const
Returns the size in bits of the given type in the current scope.
Attribute getEndianness() const
Returns the specified endianness.
Dialects are groups of MLIR operations, types and attributes, as well as behavior associated with the...
This class represents a diagnostic that is inflight and set to be reported.
This class defines the main interface for locations in MLIR and acts as a non-nullable wrapper around...
Operation is the basic unit of execution within MLIR.
Location getLoc()
The source location the operation was defined or derived from.
Operation * getParentOp()
Returns the closest surrounding operation that contains this operation or nullptr if this is a top-le...
InFlightDiagnostic emitError(const Twine &message={})
Emit an error about fatal conditions with this operation, reporting up to any diagnostic handlers tha...
This class provides an efficient unique identifier for a specific C++ type.
Instances of the Type class are uniqued, have an immutable identifier and an optional mutable compone...
MLIRContext * getContext() const
Return the MLIRContext in which this type was uniqued.
TypeID getTypeID()
Return a unique identifier for the concrete type.
unsigned getIntOrFloatBitWidth() const
Return the bit width of an integer or a float type, assert failure on other types.
Attribute getDefaultAllocaMemorySpace(DataLayoutEntryInterface entry)
Default handler for alloca memory space request.
Attribute getDefaultProgramMemorySpace(DataLayoutEntryInterface entry)
Default handler for program memory space request.
Attribute getDefaultEndianness(DataLayoutEntryInterface entry)
Default handler for endianness request.
std::optional< uint64_t > getDefaultIndexBitwidth(Type type, const DataLayout &dataLayout, ArrayRef< DataLayoutEntryInterface > params)
Default handler for the index bitwidth request.
DataLayoutEntryList filterEntriesForType(DataLayoutEntryListRef entries, TypeID typeID)
Given a list of data layout entries, returns a new list containing the entries with keys having the g...
uint64_t getDefaultABIAlignment(Type type, const DataLayout &dataLayout, ArrayRef< DataLayoutEntryInterface > params)
Default handler for the required alignment request.
llvm::TypeSize getDefaultTypeSize(Type type, const DataLayout &dataLayout, DataLayoutEntryListRef params)
Default handler for the type size request.
llvm::TypeSize getDefaultTypeSizeInBits(Type type, const DataLayout &dataLayout, DataLayoutEntryListRef params)
Default handler for the type size in bits request.
uint64_t getDefaultPreferredAlignment(Type type, const DataLayout &dataLayout, ArrayRef< DataLayoutEntryInterface > params)
Default handler for the preferred alignment request.
llvm::TypeSize divideCeil(llvm::TypeSize numerator, uint64_t denominator)
Divides the known min value of the numerator by the denominator and rounds the result up to the next ...
uint64_t getDefaultStackAlignment(DataLayoutEntryInterface entry)
Default handler for the stack alignment request.
Attribute getDefaultGlobalMemorySpace(DataLayoutEntryInterface entry)
Default handler for global memory space request.
LogicalResult verifyDataLayoutOp(Operation *op)
Verifies that the operation implementing the data layout interface, or a module operation,...
LogicalResult verifyDataLayoutSpec(DataLayoutSpecInterface spec, Location loc)
Verifies that a data layout spec is valid.
DataLayoutEntryInterface filterEntryForIdentifier(DataLayoutEntryListRef entries, StringAttr id)
Given a list of data layout entries, returns the entry that has the given identifier as key,...
Include the generated interface declarations.
LogicalResult failure(bool isFailure=true)
Utility function to generate a LogicalResult.
InFlightDiagnostic emitError(Location loc)
Utility method to emit an error message using this location.
LogicalResult success(bool isSuccess=true)
Utility function to generate a LogicalResult.
auto get(MLIRContext *context, Ts &&...params)
Helper method that injects context only if needed, this helps unify some of the attribute constructio...
bool failed(LogicalResult result)
Utility function that returns true if the provided LogicalResult corresponds to a failure value.
This class represents an efficient way to signal success or failure.