15 #include "llvm/ADT/TypeSwitch.h"
16 #include "llvm/Support/MathExtras.h"
28 llvm::raw_string_ostream os(message);
29 os <<
"neither the scoping op nor the type class provide data layout "
32 llvm::report_fatal_error(Twine(os.str()));
40 auto attr = cast<IntegerAttr>(params.front().getValue());
41 return attr.getValue().getZExtValue();
54 if (isa<IntegerType, FloatType>(type))
57 if (
auto ctype = dyn_cast<ComplexType>(type)) {
58 Type et = ctype.getElementType();
59 uint64_t innerAlignment =
65 return llvm::alignTo(innerSize, innerAlignment) + innerSize;
69 if (isa<IndexType>(type))
78 if (
auto vecType = dyn_cast<VectorType>(type))
79 return vecType.getNumElements() / vecType.getShape().back() *
80 llvm::PowerOf2Ceil(vecType.getShape().back()) *
81 dataLayout.
getTypeSize(vecType.getElementType()) * 8;
83 if (
auto typeInterface = dyn_cast<DataLayoutTypeInterface>(type))
84 return typeInterface.getTypeSizeInBits(dataLayout, params);
89 static DataLayoutEntryInterface
92 assert(!params.empty() &&
"expected non-empty parameter list");
93 std::map<unsigned, DataLayoutEntryInterface> sortedParams;
94 for (DataLayoutEntryInterface entry : params) {
95 sortedParams.insert(std::make_pair(
98 auto iter = sortedParams.lower_bound(intType.getWidth());
99 if (iter == sortedParams.end())
100 iter = std::prev(iter);
109 cast<DenseIntElementsAttr>(entry.getValue()).getValues<uint64_t>();
116 constexpr uint64_t kDefaultSmallIntAlignment = 4u;
117 constexpr
unsigned kSmallIntSize = 64;
118 if (params.empty()) {
119 return intType.getWidth() < kSmallIntSize
120 ? llvm::PowerOf2Ceil(
122 : kDefaultSmallIntAlignment;
131 assert(params.size() <= 1 &&
"at most one data layout entry is expected for "
132 "the singleton floating-point type");
134 return llvm::PowerOf2Ceil(dataLayout.
getTypeSize(fltType).getFixedValue());
142 if (isa<VectorType>(type))
143 return llvm::PowerOf2Ceil(dataLayout.
getTypeSize(type));
145 if (
auto fltType = dyn_cast<FloatType>(type))
149 if (isa<IndexType>(type))
153 if (
auto intType = dyn_cast<IntegerType>(type))
156 if (
auto ctype = dyn_cast<ComplexType>(type))
159 if (
auto typeInterface = dyn_cast<DataLayoutTypeInterface>(type))
160 return typeInterface.getABIAlignment(dataLayout, params);
167 cast<DenseIntElementsAttr>(entry.getValue()).getValues<uint64_t>();
176 return llvm::PowerOf2Ceil(dataLayout.
getTypeSize(intType).getFixedValue());
184 assert(params.size() <= 1 &&
"at most one data layout entry is expected for "
185 "the singleton floating-point type");
195 if (isa<VectorType>(type))
198 if (
auto fltType = dyn_cast<FloatType>(type))
203 if (
auto intType = dyn_cast<IntegerType>(type))
206 if (isa<IndexType>(type)) {
211 if (
auto ctype = dyn_cast<ComplexType>(type))
215 if (
auto typeInterface = dyn_cast<DataLayoutTypeInterface>(type))
216 return typeInterface.getPreferredAlignment(dataLayout, params);
226 if (entry == DataLayoutEntryInterface()) {
230 return entry.getValue();
237 if (entry == DataLayoutEntryInterface())
240 auto value = cast<IntegerAttr>(entry.getValue());
241 return value.getValue().getZExtValue();
247 return llvm::to_vector<4>(llvm::make_filter_range(
248 entries, [typeID](DataLayoutEntryInterface entry) {
249 auto type = llvm::dyn_cast_if_present<Type>(entry.getKey());
250 return type && type.getTypeID() == typeID;
254 DataLayoutEntryInterface
257 const auto *it = llvm::find_if(entries, [
id](DataLayoutEntryInterface entry) {
258 if (!entry.getKey().is<StringAttr>())
260 return entry.getKey().get<StringAttr>() ==
id;
262 return it == entries.end() ? DataLayoutEntryInterface() : *it;
267 .Case<ModuleOp, DataLayoutOpInterface>(
268 [&](
auto op) {
return op.getDataLayoutSpec(); })
270 llvm_unreachable(
"expected an op with data layout spec");
271 return DataLayoutSpecInterface();
287 .Case<ModuleOp>([&](ModuleOp op) {
295 specs.push_back(op.getDataLayoutSpec());
297 opLocations->push_back(op.
getLoc());
299 .Case<DataLayoutOpInterface>([&](DataLayoutOpInterface op) {
300 specs.push_back(op.getDataLayoutSpec());
302 opLocations->push_back(op.
getLoc());
313 assert((isa<ModuleOp, DataLayoutOpInterface>(leaf)) &&
314 "expected an op with data layout spec");
326 auto nonNullSpecs = llvm::to_vector<2>(llvm::make_filter_range(
327 llvm::reverse(specs),
328 [](DataLayoutSpecInterface iface) {
return iface !=
nullptr; }));
331 if (DataLayoutSpecInterface current =
getSpec(leaf))
332 return current.combineWith(nonNullSpecs);
333 if (nonNullSpecs.empty())
335 return nonNullSpecs.back().combineWith(
340 DataLayoutSpecInterface spec =
getSpec(op);
350 <<
"data layout does not combine with layouts of enclosing ops";
355 diag.attachNote(loc) <<
"enclosing op with data layout";
362 uint64_t denominator) {
372 template <
typename OpTy>
374 if (!originalLayout) {
375 assert((!op || !op.getDataLayoutSpec()) &&
376 "could not compute layout information for an op (failed to "
377 "combine attributes?)");
385 allocaMemorySpace(std::nullopt), stackAlignment(std::nullopt) {
386 #if LLVM_ENABLE_ABI_BREAKING_CHECKS
394 allocaMemorySpace(std::nullopt), stackAlignment(std::nullopt) {
395 #if LLVM_ENABLE_ABI_BREAKING_CHECKS
405 if (
auto module = dyn_cast<ModuleOp>(op))
407 if (
auto iface = dyn_cast<DataLayoutOpInterface>(op))
414 void mlir::DataLayout::checkValid()
const {
415 #if LLVM_ENABLE_ABI_BREAKING_CHECKS
418 assert(specs.size() == layoutStack.size() &&
419 "data layout object used, but no longer valid due to the change in "
420 "number of nested layouts");
421 for (
auto pair : llvm::zip(specs, layoutStack)) {
423 Attribute origLayout = std::get<1>(pair);
424 assert(newLayout == origLayout &&
425 "data layout object used, but no longer valid "
426 "due to the change in layout attributes");
429 assert(((!scope && !this->originalLayout) ||
431 "data layout object used, but no longer valid due to the change in "
438 template <
typename T>
441 auto it = cache.find(t);
442 if (it != cache.end())
445 auto result = cache.try_emplace(t, compute(t));
446 return result.first->second;
451 return cachedLookup<llvm::TypeSize>(t, sizes, [&](
Type ty) {
454 list = originalLayout.getSpecForType(ty.
getTypeID());
455 if (
auto iface = dyn_cast_or_null<DataLayoutOpInterface>(scope))
456 return iface.getTypeSize(ty, *
this, list);
463 return cachedLookup<llvm::TypeSize>(t, bitsizes, [&](
Type ty) {
466 list = originalLayout.getSpecForType(ty.
getTypeID());
467 if (
auto iface = dyn_cast_or_null<DataLayoutOpInterface>(scope))
468 return iface.getTypeSizeInBits(ty, *
this, list);
475 return cachedLookup<uint64_t>(t, abiAlignments, [&](
Type ty) {
478 list = originalLayout.getSpecForType(ty.
getTypeID());
479 if (
auto iface = dyn_cast_or_null<DataLayoutOpInterface>(scope))
480 return iface.getTypeABIAlignment(ty, *
this, list);
487 return cachedLookup<uint64_t>(t, preferredAlignments, [&](
Type ty) {
490 list = originalLayout.getSpecForType(ty.
getTypeID());
491 if (
auto iface = dyn_cast_or_null<DataLayoutOpInterface>(scope))
492 return iface.getTypePreferredAlignment(ty, *
this, list);
499 if (allocaMemorySpace)
500 return *allocaMemorySpace;
501 DataLayoutEntryInterface entry;
503 entry = originalLayout.getSpecForIdentifier(
504 originalLayout.getAllocaMemorySpaceIdentifier(
505 originalLayout.getContext()));
506 if (
auto iface = dyn_cast_or_null<DataLayoutOpInterface>(scope))
507 allocaMemorySpace = iface.getAllocaMemorySpace(entry);
510 return *allocaMemorySpace;
516 return *stackAlignment;
517 DataLayoutEntryInterface entry;
519 entry = originalLayout.getSpecForIdentifier(
520 originalLayout.getStackAlignmentIdentifier(
521 originalLayout.getContext()));
522 if (
auto iface = dyn_cast_or_null<DataLayoutOpInterface>(scope))
523 stackAlignment = iface.getStackAlignment(entry);
526 return *stackAlignment;
533 void DataLayoutSpecInterface::bucketEntriesByType(
536 for (DataLayoutEntryInterface entry : getEntries()) {
537 if (
auto type = llvm::dyn_cast_if_present<Type>(entry.getKey()))
538 types[type.getTypeID()].push_back(entry);
540 ids[entry.getKey().get<StringAttr>()] = entry;
547 for (DataLayoutEntryInterface entry : spec.getEntries())
548 if (
failed(entry.verifyEntry(loc)))
555 spec.bucketEntriesByType(types, ids);
557 for (
const auto &kvp : types) {
558 auto sampleType = kvp.second.front().getKey().get<
Type>();
559 if (isa<IndexType>(sampleType)) {
560 assert(kvp.second.size() == 1 &&
561 "expected one data layout entry for non-parametric 'index' type");
562 if (!isa<IntegerAttr>(kvp.second.front().getValue()))
564 <<
"expected integer attribute in the data layout entry for "
569 if (isa<IntegerType, FloatType>(sampleType)) {
570 for (DataLayoutEntryInterface entry : kvp.second) {
571 auto value = dyn_cast<DenseIntElementsAttr>(entry.getValue());
572 if (!value || !value.getElementType().isSignlessInteger(64)) {
573 emitError(loc) <<
"expected a dense i64 elements attribute in the "
579 auto elements = llvm::to_vector<2>(value.getValues<uint64_t>());
580 unsigned numElements = elements.size();
581 if (numElements < 1 || numElements > 2) {
582 emitError(loc) <<
"expected 1 or 2 elements in the data layout entry "
587 uint64_t abi = elements[0];
588 uint64_t preferred = numElements == 2 ? elements[1] : abi;
589 if (preferred < abi) {
591 <<
"preferred alignment is expected to be greater than or equal "
592 "to the abi alignment in data layout entry "
600 if (isa<BuiltinDialect>(&sampleType.getDialect()))
601 return emitError(loc) <<
"unexpected data layout for a built-in type";
603 auto dlType = dyn_cast<DataLayoutTypeInterface>(sampleType);
606 <<
"data layout specified for a type that does not support it";
607 if (
failed(dlType.verifyEntries(kvp.second, loc)))
611 for (
const auto &kvp : ids) {
612 StringAttr identifier = kvp.second.getKey().get<StringAttr>();
613 Dialect *dialect = identifier.getReferencedDialect();
620 const auto *iface = dyn_cast<DataLayoutDialectInterface>(dialect);
623 <<
"the '" << dialect->getNamespace()
624 <<
"' dialect does not support identifier data layout entries";
626 if (
failed(iface->verifyEntry(kvp.second, loc)))
633 #include "mlir/Interfaces/DataLayoutAttrInterface.cpp.inc"
634 #include "mlir/Interfaces/DataLayoutOpInterface.cpp.inc"
635 #include "mlir/Interfaces/DataLayoutTypeInterface.cpp.inc"
static uint64_t getIntegerTypePreferredAlignment(IntegerType intType, const DataLayout &dataLayout, ArrayRef< DataLayoutEntryInterface > params)
static DataLayoutSpecInterface getCombinedDataLayout(Operation *leaf)
Returns a layout spec that is a combination of the layout specs attached to the given operation and a...
constexpr static const uint64_t kDefaultBitsInByte
static uint64_t getIntegerTypeABIAlignment(IntegerType intType, ArrayRef< DataLayoutEntryInterface > params)
static T cachedLookup(Type t, DenseMap< Type, T > &cache, function_ref< T(Type)> compute)
Looks up the value for the given type key in the given cache.
static uint64_t extractPreferredAlignment(DataLayoutEntryInterface entry)
static void collectParentLayouts(Operation *leaf, SmallVectorImpl< DataLayoutSpecInterface > &specs, SmallVectorImpl< Location > *opLocations=nullptr)
Populates opsWithLayout with the list of proper ancestors of leaf that are either modules or implemen...
static uint64_t getFloatTypePreferredAlignment(FloatType fltType, const DataLayout &dataLayout, ArrayRef< DataLayoutEntryInterface > params)
static uint64_t getFloatTypeABIAlignment(FloatType fltType, const DataLayout &dataLayout, ArrayRef< DataLayoutEntryInterface > params)
static DataLayoutSpecInterface getSpec(Operation *operation)
static uint64_t getIndexBitwidth(DataLayoutEntryListRef params)
Returns the bitwidth of the index type if specified in the param list.
static uint64_t extractABIAlignment(DataLayoutEntryInterface entry)
static DataLayoutEntryInterface findEntryForIntegerType(IntegerType intType, ArrayRef< DataLayoutEntryInterface > params)
void checkMissingLayout(DataLayoutSpecInterface originalLayout, OpTy op)
static void reportMissingDataLayout(Type type)
Reports that the given type is missing the data layout information and exits.
static std::string diag(const llvm::Value &value)
Attributes are known-constant values of operations.
The main mechanism for performing data layout queries.
Attribute getAllocaMemorySpace() const
Returns the memory space used for AllocaOps.
static DataLayout closest(Operation *op)
Returns the layout of the closest parent operation carrying layout info.
llvm::TypeSize getTypeSize(Type t) const
Returns the size of the given type in the current scope.
uint64_t getStackAlignment() const
Returns the natural alignment of the stack in bits.
uint64_t getTypePreferredAlignment(Type t) const
Returns the preferred of the given type in the current scope.
uint64_t getTypeABIAlignment(Type t) const
Returns the required alignment of the given type in the current scope.
llvm::TypeSize getTypeSizeInBits(Type t) const
Returns the size in bits of the given type in the current scope.
Dialects are groups of MLIR operations, types and attributes, as well as behavior associated with the...
This class represents a diagnostic that is inflight and set to be reported.
This class defines the main interface for locations in MLIR and acts as a non-nullable wrapper around...
Operation is the basic unit of execution within MLIR.
Location getLoc()
The source location the operation was defined or derived from.
Operation * getParentOp()
Returns the closest surrounding operation that contains this operation or nullptr if this is a top-le...
InFlightDiagnostic emitError(const Twine &message={})
Emit an error about fatal conditions with this operation, reporting up to any diagnostic handlers tha...
This class provides an efficient unique identifier for a specific C++ type.
Instances of the Type class are uniqued, have an immutable identifier and an optional mutable compone...
MLIRContext * getContext() const
Return the MLIRContext in which this type was uniqued.
TypeID getTypeID()
Return a unique identifier for the concrete type.
unsigned getIntOrFloatBitWidth() const
Return the bit width of an integer or a float type, assert failure on other types.
Attribute getDefaultAllocaMemorySpace(DataLayoutEntryInterface entry)
Default handler for alloca memory space request.
DataLayoutEntryList filterEntriesForType(DataLayoutEntryListRef entries, TypeID typeID)
Given a list of data layout entries, returns a new list containing the entries with keys having the g...
uint64_t getDefaultABIAlignment(Type type, const DataLayout &dataLayout, ArrayRef< DataLayoutEntryInterface > params)
Default handler for the required alignment request.
llvm::TypeSize getDefaultTypeSize(Type type, const DataLayout &dataLayout, DataLayoutEntryListRef params)
Default handler for the type size request.
llvm::TypeSize getDefaultTypeSizeInBits(Type type, const DataLayout &dataLayout, DataLayoutEntryListRef params)
Default handler for the type size in bits request.
uint64_t getDefaultPreferredAlignment(Type type, const DataLayout &dataLayout, ArrayRef< DataLayoutEntryInterface > params)
Default handler for the preferred alignment request.
llvm::TypeSize divideCeil(llvm::TypeSize numerator, uint64_t denominator)
Divides the known min value of the numerator by the denominator and rounds the result up to the next ...
uint64_t getDefaultStackAlignment(DataLayoutEntryInterface entry)
Default handler for the stack alignment request.
LogicalResult verifyDataLayoutOp(Operation *op)
Verifies that the operation implementing the data layout interface, or a module operation,...
LogicalResult verifyDataLayoutSpec(DataLayoutSpecInterface spec, Location loc)
Verifies that a data layout spec is valid.
DataLayoutEntryInterface filterEntryForIdentifier(DataLayoutEntryListRef entries, StringAttr id)
Given a list of data layout entries, returns the entry that has the given identifier as key,...
Include the generated interface declarations.
LogicalResult failure(bool isFailure=true)
Utility function to generate a LogicalResult.
InFlightDiagnostic emitError(Location loc)
Utility method to emit an error message using this location.
LogicalResult success(bool isSuccess=true)
Utility function to generate a LogicalResult.
auto get(MLIRContext *context, Ts &&...params)
Helper method that injects context only if needed, this helps unify some of the attribute constructio...
bool failed(LogicalResult result)
Utility function that returns true if the provided LogicalResult corresponds to a failure value.
This class represents an efficient way to signal success or failure.