15 #include "llvm/ADT/TypeSwitch.h"
16 #include "llvm/Support/MathExtras.h"
28 llvm::raw_string_ostream os(message);
29 os <<
"neither the scoping op nor the type class provide data layout "
32 llvm::report_fatal_error(Twine(message));
40 auto attr = cast<IntegerAttr>(params.front().getValue());
41 return attr.getValue().getZExtValue();
57 if (
auto ctype = dyn_cast<ComplexType>(type)) {
58 Type et = ctype.getElementType();
59 uint64_t innerAlignment =
65 return llvm::alignTo(innerSize, innerAlignment) + innerSize;
69 if (isa<IndexType>(type))
78 if (
auto vecType = dyn_cast<VectorType>(type)) {
79 uint64_t baseSize = vecType.getNumElements() / vecType.getShape().back() *
80 llvm::PowerOf2Ceil(vecType.getShape().back()) *
81 dataLayout.
getTypeSize(vecType.getElementType()) * 8;
85 if (
auto typeInterface = dyn_cast<DataLayoutTypeInterface>(type))
86 return typeInterface.getTypeSizeInBits(dataLayout, params);
91 static DataLayoutEntryInterface
94 assert(!params.empty() &&
"expected non-empty parameter list");
95 std::map<unsigned, DataLayoutEntryInterface> sortedParams;
96 for (DataLayoutEntryInterface entry : params) {
97 sortedParams.insert(std::make_pair(
98 cast<Type>(entry.getKey()).getIntOrFloatBitWidth(), entry));
100 auto iter = sortedParams.lower_bound(intType.getWidth());
101 if (iter == sortedParams.end())
102 iter = std::prev(iter);
111 cast<DenseIntElementsAttr>(entry.getValue()).getValues<uint64_t>();
118 constexpr uint64_t kDefaultSmallIntAlignment = 4u;
119 constexpr
unsigned kSmallIntSize = 64;
120 if (params.empty()) {
121 return intType.getWidth() < kSmallIntSize
122 ? llvm::PowerOf2Ceil(
124 : kDefaultSmallIntAlignment;
133 assert(params.size() <= 1 &&
"at most one data layout entry is expected for "
134 "the singleton floating-point type");
136 return llvm::PowerOf2Ceil(dataLayout.
getTypeSize(fltType).getFixedValue());
145 if (isa<VectorType>(type))
146 return llvm::PowerOf2Ceil(dataLayout.
getTypeSize(type).getKnownMinValue());
148 if (
auto fltType = dyn_cast<FloatType>(type))
152 if (isa<IndexType>(type))
156 if (
auto intType = dyn_cast<IntegerType>(type))
159 if (
auto ctype = dyn_cast<ComplexType>(type))
162 if (
auto typeInterface = dyn_cast<DataLayoutTypeInterface>(type))
163 return typeInterface.getABIAlignment(dataLayout, params);
170 cast<DenseIntElementsAttr>(entry.getValue()).getValues<uint64_t>();
179 return llvm::PowerOf2Ceil(dataLayout.
getTypeSize(intType).getFixedValue());
187 assert(params.size() <= 1 &&
"at most one data layout entry is expected for "
188 "the singleton floating-point type");
198 if (isa<VectorType>(type))
201 if (
auto fltType = dyn_cast<FloatType>(type))
206 if (
auto intType = dyn_cast<IntegerType>(type))
209 if (isa<IndexType>(type)) {
214 if (
auto ctype = dyn_cast<ComplexType>(type))
218 if (
auto typeInterface = dyn_cast<DataLayoutTypeInterface>(type))
219 return typeInterface.getPreferredAlignment(dataLayout, params);
227 if (isa<IndexType>(type))
230 if (
auto typeInterface = dyn_cast<DataLayoutTypeInterface>(type))
231 if (std::optional<uint64_t> indexBitwidth =
232 typeInterface.getIndexBitwidth(dataLayout, params))
233 return *indexBitwidth;
243 if (entry == DataLayoutEntryInterface())
246 return entry.getValue();
256 return entry.getValue();
264 if (entry == DataLayoutEntryInterface()) {
268 return entry.getValue();
274 if (entry == DataLayoutEntryInterface())
277 return entry.getValue();
285 if (entry == DataLayoutEntryInterface()) {
289 return entry.getValue();
297 if (entry == DataLayoutEntryInterface()) {
301 return entry.getValue();
308 if (entry == DataLayoutEntryInterface())
311 auto value = cast<IntegerAttr>(entry.getValue());
312 return value.getValue().getZExtValue();
315 std::optional<Attribute>
317 if (entry == DataLayoutEntryInterface())
320 return entry.getValue();
326 return llvm::filter_to_vector<4>(
327 entries, [typeID](DataLayoutEntryInterface entry) {
328 auto type = llvm::dyn_cast_if_present<Type>(entry.getKey());
329 return type && type.getTypeID() == typeID;
333 DataLayoutEntryInterface
336 const auto *it = llvm::find_if(entries, [
id](DataLayoutEntryInterface entry) {
337 if (
auto attr = dyn_cast<StringAttr>(entry.getKey()))
341 return it == entries.end() ? DataLayoutEntryInterface() : *it;
346 .Case<ModuleOp, DataLayoutOpInterface>(
347 [&](
auto op) {
return op.getDataLayoutSpec(); })
349 llvm_unreachable(
"expected an op with data layout spec");
350 return DataLayoutSpecInterface();
356 ModuleOp moduleOp = dyn_cast<ModuleOp>(operation);
359 return moduleOp.getTargetSystemSpec();
361 return TargetSystemSpecInterface();
376 .Case<ModuleOp>([&](ModuleOp op) {
382 if (!op->getParentOp() && !op.getDataLayoutSpec())
384 specs.push_back(op.getDataLayoutSpec());
386 opLocations->push_back(op.getLoc());
388 .Case<DataLayoutOpInterface>([&](DataLayoutOpInterface op) {
389 specs.push_back(op.getDataLayoutSpec());
391 opLocations->push_back(op.getLoc());
402 assert((isa<ModuleOp, DataLayoutOpInterface>(leaf)) &&
403 "expected an op with data layout spec");
415 auto nonNullSpecs = llvm::filter_to_vector<2>(
416 llvm::reverse(specs),
417 [](DataLayoutSpecInterface iface) {
return iface !=
nullptr; });
420 if (DataLayoutSpecInterface current =
getSpec(leaf))
421 return current.combineWith(nonNullSpecs);
422 if (nonNullSpecs.empty())
424 return nonNullSpecs.back().combineWith(
429 DataLayoutSpecInterface spec =
getSpec(op);
434 if (failed(spec.verifySpec(op->
getLoc())))
439 <<
"data layout does not combine with layouts of enclosing ops";
444 diag.attachNote(loc) <<
"enclosing op with data layout";
451 uint64_t denominator) {
461 template <
typename OpTy>
463 if (!originalLayout) {
464 assert((!op || !op.getDataLayoutSpec()) &&
465 "could not compute layout information for an op (failed to "
466 "combine attributes?)");
475 allocaMemorySpace(std::nullopt), programMemorySpace(std::nullopt),
476 globalMemorySpace(std::nullopt), stackAlignment(std::nullopt) {
477 #if LLVM_ENABLE_ABI_BREAKING_CHECKS
486 allocaMemorySpace(std::nullopt), programMemorySpace(std::nullopt),
487 globalMemorySpace(std::nullopt), stackAlignment(std::nullopt) {
488 #if LLVM_ENABLE_ABI_BREAKING_CHECKS
498 if (
auto module = dyn_cast<ModuleOp>(op))
500 if (
auto iface = dyn_cast<DataLayoutOpInterface>(op))
507 void mlir::DataLayout::checkValid()
const {
508 #if LLVM_ENABLE_ABI_BREAKING_CHECKS
511 assert(specs.size() == layoutStack.size() &&
512 "data layout object used, but no longer valid due to the change in "
513 "number of nested layouts");
514 for (
auto pair : llvm::zip(specs, layoutStack)) {
516 Attribute origLayout = std::get<1>(pair);
517 assert(newLayout == origLayout &&
518 "data layout object used, but no longer valid "
519 "due to the change in layout attributes");
522 assert(((!scope && !this->originalLayout) ||
524 "data layout object used, but no longer valid due to the change in "
531 template <
typename T>
534 auto it = cache.find(t);
535 if (it != cache.end())
538 auto result = cache.try_emplace(t, compute(t));
539 return result.first->second;
544 return cachedLookup<llvm::TypeSize>(t, sizes, [&](
Type ty) {
547 list = originalLayout.getSpecForType(ty.
getTypeID());
548 if (
auto iface = dyn_cast_or_null<DataLayoutOpInterface>(scope))
549 return iface.getTypeSize(ty, *
this, list);
556 return cachedLookup<llvm::TypeSize>(t, bitsizes, [&](
Type ty) {
559 list = originalLayout.getSpecForType(ty.
getTypeID());
560 if (
auto iface = dyn_cast_or_null<DataLayoutOpInterface>(scope))
561 return iface.getTypeSizeInBits(ty, *
this, list);
568 return cachedLookup<uint64_t>(t, abiAlignments, [&](
Type ty) {
571 list = originalLayout.getSpecForType(ty.
getTypeID());
572 if (
auto iface = dyn_cast_or_null<DataLayoutOpInterface>(scope))
573 return iface.getTypeABIAlignment(ty, *
this, list);
580 return cachedLookup<uint64_t>(t, preferredAlignments, [&](
Type ty) {
583 list = originalLayout.getSpecForType(ty.
getTypeID());
584 if (
auto iface = dyn_cast_or_null<DataLayoutOpInterface>(scope))
585 return iface.getTypePreferredAlignment(ty, *
this, list);
592 return cachedLookup<std::optional<uint64_t>>(t, indexBitwidths, [&](
Type ty) {
595 list = originalLayout.getSpecForType(ty.getTypeID());
596 if (
auto iface = dyn_cast_or_null<DataLayoutOpInterface>(scope))
597 return iface.getIndexBitwidth(ty, *
this, list);
606 DataLayoutEntryInterface entry;
608 entry = originalLayout.getSpecForIdentifier(
609 originalLayout.getEndiannessIdentifier(originalLayout.getContext()));
611 if (
auto iface = dyn_cast_or_null<DataLayoutOpInterface>(scope))
612 endianness = iface.getEndianness(entry);
620 if (defaultMemorySpace)
621 return *defaultMemorySpace;
622 DataLayoutEntryInterface entry;
624 entry = originalLayout.getSpecForIdentifier(
625 originalLayout.getDefaultMemorySpaceIdentifier(
626 originalLayout.getContext()));
627 if (
auto iface = dyn_cast_or_null<DataLayoutOpInterface>(scope))
628 defaultMemorySpace = iface.getDefaultMemorySpace(entry);
631 return *defaultMemorySpace;
636 if (allocaMemorySpace)
637 return *allocaMemorySpace;
638 DataLayoutEntryInterface entry;
640 entry = originalLayout.getSpecForIdentifier(
641 originalLayout.getAllocaMemorySpaceIdentifier(
642 originalLayout.getContext()));
643 if (
auto iface = dyn_cast_or_null<DataLayoutOpInterface>(scope))
644 allocaMemorySpace = iface.getAllocaMemorySpace(entry);
647 return *allocaMemorySpace;
653 return *manglingMode;
654 DataLayoutEntryInterface entry;
656 entry = originalLayout.getSpecForIdentifier(
657 originalLayout.getManglingModeIdentifier(originalLayout.getContext()));
659 if (
auto iface = dyn_cast_or_null<DataLayoutOpInterface>(scope))
660 manglingMode = iface.getManglingMode(entry);
663 return *manglingMode;
668 if (programMemorySpace)
669 return *programMemorySpace;
670 DataLayoutEntryInterface entry;
672 entry = originalLayout.getSpecForIdentifier(
673 originalLayout.getProgramMemorySpaceIdentifier(
674 originalLayout.getContext()));
675 if (
auto iface = dyn_cast_or_null<DataLayoutOpInterface>(scope))
676 programMemorySpace = iface.getProgramMemorySpace(entry);
679 return *programMemorySpace;
684 if (globalMemorySpace)
685 return *globalMemorySpace;
686 DataLayoutEntryInterface entry;
688 entry = originalLayout.getSpecForIdentifier(
689 originalLayout.getGlobalMemorySpaceIdentifier(
690 originalLayout.getContext()));
691 if (
auto iface = dyn_cast_or_null<DataLayoutOpInterface>(scope))
692 globalMemorySpace = iface.getGlobalMemorySpace(entry);
695 return *globalMemorySpace;
701 return *stackAlignment;
702 DataLayoutEntryInterface entry;
704 entry = originalLayout.getSpecForIdentifier(
705 originalLayout.getStackAlignmentIdentifier(
706 originalLayout.getContext()));
707 if (
auto iface = dyn_cast_or_null<DataLayoutOpInterface>(scope))
708 stackAlignment = iface.getStackAlignment(entry);
711 return *stackAlignment;
715 TargetSystemSpecInterface::DeviceID deviceID,
716 StringAttr propertyName)
const {
718 DataLayoutEntryInterface entry;
719 if (originalTargetSystemDesc) {
720 if (std::optional<TargetDeviceSpecInterface> device =
721 originalTargetSystemDesc.getDeviceSpecForDeviceID(deviceID))
722 entry = device->getSpecForIdentifier(propertyName);
728 if (
auto iface = dyn_cast_or_null<DataLayoutOpInterface>(scope))
729 return iface.getDevicePropertyValue(entry);
738 void DataLayoutSpecInterface::bucketEntriesByType(
741 for (DataLayoutEntryInterface entry : getEntries()) {
742 if (
auto type = llvm::dyn_cast_if_present<Type>(entry.getKey()))
743 types[type.getTypeID()].push_back(entry);
745 ids[llvm::cast<StringAttr>(entry.getKey())] = entry;
752 for (DataLayoutEntryInterface entry : spec.getEntries())
753 if (failed(entry.verifyEntry(loc)))
760 spec.bucketEntriesByType(types, ids);
762 for (
const auto &kvp : types) {
763 auto sampleType = cast<Type>(kvp.second.front().getKey());
764 if (isa<IndexType>(sampleType)) {
765 assert(kvp.second.size() == 1 &&
766 "expected one data layout entry for non-parametric 'index' type");
767 if (!isa<IntegerAttr>(kvp.second.front().getValue()))
769 <<
"expected integer attribute in the data layout entry for "
774 if (sampleType.isIntOrFloat()) {
775 for (DataLayoutEntryInterface entry : kvp.second) {
776 auto value = dyn_cast<DenseIntElementsAttr>(entry.getValue());
777 if (!value || !value.getElementType().isSignlessInteger(64)) {
778 emitError(loc) <<
"expected a dense i64 elements attribute in the "
784 auto elements = llvm::to_vector<2>(value.getValues<uint64_t>());
785 unsigned numElements = elements.size();
786 if (numElements < 1 || numElements > 2) {
787 emitError(loc) <<
"expected 1 or 2 elements in the data layout entry "
792 uint64_t abi = elements[0];
793 uint64_t preferred = numElements == 2 ? elements[1] : abi;
794 if (preferred < abi) {
796 <<
"preferred alignment is expected to be greater than or equal "
797 "to the abi alignment in data layout entry "
805 if (isa<BuiltinDialect>(&sampleType.getDialect()))
806 return emitError(loc) <<
"unexpected data layout for a built-in type";
808 auto dlType = dyn_cast<DataLayoutTypeInterface>(sampleType);
811 <<
"data layout specified for a type that does not support it";
812 if (failed(dlType.verifyEntries(kvp.second, loc)))
816 for (
const auto &kvp : ids) {
817 StringAttr identifier = cast<StringAttr>(kvp.second.getKey());
818 Dialect *dialect = identifier.getReferencedDialect();
825 const auto *iface = dyn_cast<DataLayoutDialectInterface>(dialect);
829 <<
"' dialect does not support identifier data layout entries";
831 if (failed(iface->verifyEntry(kvp.second, loc)))
843 for (
const auto &entry : spec.getEntries()) {
844 auto targetDeviceSpec =
845 dyn_cast<TargetDeviceSpecInterface>(entry.getValue());
847 if (!targetDeviceSpec)
851 if (failed(targetDeviceSpec.verifyEntry(loc)))
856 llvm::dyn_cast<TargetSystemSpecInterface::DeviceID>(entry.getKey());
860 if (!deviceIDs.insert(deviceID).second) {
865 for (DataLayoutEntryInterface entry : targetDeviceSpec.getEntries()) {
866 if (
auto type = llvm::dyn_cast_if_present<Type>(entry.getKey())) {
870 deviceDescKeys[cast<StringAttr>(entry.getKey())] = entry;
875 for (
const auto &[keyName, keyVal] : deviceDescKeys) {
876 Dialect *dialect = keyName.getReferencedDialect();
883 const auto *iface = dyn_cast<DataLayoutDialectInterface>(dialect);
887 <<
"' dialect does not support identifier data layout entries";
889 if (failed(iface->verifyEntry(keyVal, loc)))
896 #include "mlir/Interfaces/DataLayoutAttrInterface.cpp.inc"
897 #include "mlir/Interfaces/DataLayoutOpInterface.cpp.inc"
898 #include "mlir/Interfaces/DataLayoutTypeInterface.cpp.inc"
static uint64_t getIntegerTypePreferredAlignment(IntegerType intType, const DataLayout &dataLayout, ArrayRef< DataLayoutEntryInterface > params)
static DataLayoutSpecInterface getCombinedDataLayout(Operation *leaf)
Returns a layout spec that is a combination of the layout specs attached to the given operation and a...
static TargetSystemSpecInterface getTargetSystemSpec(Operation *operation)
constexpr static const uint64_t kDefaultBitsInByte
static uint64_t getIntegerTypeABIAlignment(IntegerType intType, ArrayRef< DataLayoutEntryInterface > params)
static T cachedLookup(Type t, DenseMap< Type, T > &cache, function_ref< T(Type)> compute)
Looks up the value for the given type key in the given cache.
static uint64_t extractPreferredAlignment(DataLayoutEntryInterface entry)
static void collectParentLayouts(Operation *leaf, SmallVectorImpl< DataLayoutSpecInterface > &specs, SmallVectorImpl< Location > *opLocations=nullptr)
Populates opsWithLayout with the list of proper ancestors of leaf that are either modules or implemen...
static uint64_t getFloatTypePreferredAlignment(FloatType fltType, const DataLayout &dataLayout, ArrayRef< DataLayoutEntryInterface > params)
static uint64_t getFloatTypeABIAlignment(FloatType fltType, const DataLayout &dataLayout, ArrayRef< DataLayoutEntryInterface > params)
static DataLayoutSpecInterface getSpec(Operation *operation)
static uint64_t getIndexBitwidth(DataLayoutEntryListRef params)
Returns the bitwidth of the index type if specified in the param list.
static uint64_t extractABIAlignment(DataLayoutEntryInterface entry)
static DataLayoutEntryInterface findEntryForIntegerType(IntegerType intType, ArrayRef< DataLayoutEntryInterface > params)
void checkMissingLayout(DataLayoutSpecInterface originalLayout, OpTy op)
static void reportMissingDataLayout(Type type)
Reports that the given type is missing the data layout information and exits.
static std::string diag(const llvm::Value &value)
Attributes are known-constant values of operations.
The main mechanism for performing data layout queries.
Attribute getAllocaMemorySpace() const
Returns the memory space used for AllocaOps.
static DataLayout closest(Operation *op)
Returns the layout of the closest parent operation carrying layout info.
std::optional< uint64_t > getTypeIndexBitwidth(Type t) const
Returns the bitwidth that should be used when performing index computations for the given pointer-lik...
llvm::TypeSize getTypeSize(Type t) const
Returns the size of the given type in the current scope.
Attribute getManglingMode() const
Returns the mangling mode.
uint64_t getStackAlignment() const
Returns the natural alignment of the stack in bits.
Attribute getProgramMemorySpace() const
Returns the memory space used for program memory operations.
uint64_t getTypePreferredAlignment(Type t) const
Returns the preferred of the given type in the current scope.
Attribute getGlobalMemorySpace() const
Returns the memory space used for global operations.
uint64_t getTypeABIAlignment(Type t) const
Returns the required alignment of the given type in the current scope.
llvm::TypeSize getTypeSizeInBits(Type t) const
Returns the size in bits of the given type in the current scope.
Attribute getDefaultMemorySpace() const
Returns the default memory space used for memory operations.
Attribute getEndianness() const
Returns the specified endianness.
std::optional< Attribute > getDevicePropertyValue(TargetSystemSpecInterface::DeviceID, StringAttr propertyName) const
Returns the value of the specified property if the property is defined for the given device ID,...
Dialects are groups of MLIR operations, types and attributes, as well as behavior associated with the...
StringRef getNamespace() const
This class represents a diagnostic that is inflight and set to be reported.
This class defines the main interface for locations in MLIR and acts as a non-nullable wrapper around...
Operation is the basic unit of execution within MLIR.
Location getLoc()
The source location the operation was defined or derived from.
Operation * getParentOp()
Returns the closest surrounding operation that contains this operation or nullptr if this is a top-le...
InFlightDiagnostic emitError(const Twine &message={})
Emit an error about fatal conditions with this operation, reporting up to any diagnostic handlers tha...
OpTy getParentOfType()
Return the closest surrounding parent operation that is of type 'OpTy'.
This class provides an efficient unique identifier for a specific C++ type.
Instances of the Type class are uniqued, have an immutable identifier and an optional mutable compone...
MLIRContext * getContext() const
Return the MLIRContext in which this type was uniqued.
bool isIntOrFloat() const
Return true if this is an integer (of any signedness) or a float type.
TypeID getTypeID()
Return a unique identifier for the concrete type.
unsigned getIntOrFloatBitWidth() const
Return the bit width of an integer or a float type, assert failure on other types.
Attribute getDefaultAllocaMemorySpace(DataLayoutEntryInterface entry)
Default handler for alloca memory space request.
Attribute getDefaultProgramMemorySpace(DataLayoutEntryInterface entry)
Default handler for program memory space request.
Attribute getDefaultEndianness(DataLayoutEntryInterface entry)
Default handler for endianness request.
std::optional< uint64_t > getDefaultIndexBitwidth(Type type, const DataLayout &dataLayout, ArrayRef< DataLayoutEntryInterface > params)
Default handler for the index bitwidth request.
DataLayoutEntryList filterEntriesForType(DataLayoutEntryListRef entries, TypeID typeID)
Given a list of data layout entries, returns a new list containing the entries with keys having the g...
LogicalResult verifyTargetSystemSpec(TargetSystemSpecInterface spec, Location loc)
Verifies that a target system desc spec is valid.
std::optional< Attribute > getDevicePropertyValue(DataLayoutEntryInterface entry)
Returns the value of the property from the specified DataLayoutEntry.
Attribute getDefaultManglingMode(DataLayoutEntryInterface entry)
Default handler for mangling mode request.
uint64_t getDefaultABIAlignment(Type type, const DataLayout &dataLayout, ArrayRef< DataLayoutEntryInterface > params)
Default handler for the required alignment request.
llvm::TypeSize getDefaultTypeSize(Type type, const DataLayout &dataLayout, DataLayoutEntryListRef params)
Default handler for the type size request.
llvm::TypeSize getDefaultTypeSizeInBits(Type type, const DataLayout &dataLayout, DataLayoutEntryListRef params)
Default handler for the type size in bits request.
uint64_t getDefaultPreferredAlignment(Type type, const DataLayout &dataLayout, ArrayRef< DataLayoutEntryInterface > params)
Default handler for the preferred alignment request.
llvm::TypeSize divideCeil(llvm::TypeSize numerator, uint64_t denominator)
Divides the known min value of the numerator by the denominator and rounds the result up to the next ...
uint64_t getDefaultStackAlignment(DataLayoutEntryInterface entry)
Default handler for the stack alignment request.
Attribute getDefaultGlobalMemorySpace(DataLayoutEntryInterface entry)
Default handler for global memory space request.
Attribute getDefaultMemorySpace(DataLayoutEntryInterface entry)
Default handler for the default memory space request.
LogicalResult verifyDataLayoutOp(Operation *op)
Verifies that the operation implementing the data layout interface, or a module operation,...
LogicalResult verifyDataLayoutSpec(DataLayoutSpecInterface spec, Location loc)
Verifies that a data layout spec is valid.
DataLayoutEntryInterface filterEntryForIdentifier(DataLayoutEntryListRef entries, StringAttr id)
Given a list of data layout entries, returns the entry that has the given identifier as key,...
Include the generated interface declarations.
InFlightDiagnostic emitError(Location loc)
Utility method to emit an error message using this location.
auto get(MLIRContext *context, Ts &&...params)
Helper method that injects context only if needed, this helps unify some of the attribute constructio...