14 #include "llvm/ADT/ScopeExit.h"
15 #include "llvm/Support/Threading.h"
31 return *recursiveStack->second;
39 return *recursiveStackInserted.first->second;
51 : llvmDialect(ctx->getOrLoadDialect<LLVM::LLVMDialect>()),
options(
options),
52 dataLayoutAnalysis(analysis) {
53 assert(
llvmDialect &&
"LLVM IR dialect is not registered");
56 addConversion([&](ComplexType type) {
return convertComplexType(type); });
58 addConversion([&](FunctionType type) {
return convertFunctionType(type); });
59 addConversion([&](IndexType type) {
return convertIndexType(type); });
60 addConversion([&](IntegerType type) {
return convertIntegerType(type); });
61 addConversion([&](MemRefType type) {
return convertMemRefType(type); });
65 FailureOr<Type> llvmType = convertVectorType(type);
80 -> std::optional<LogicalResult> {
83 results.push_back(type);
87 if (type.isIdentified()) {
89 type.getContext(), (
"_Converted." + type.getName()).str());
92 if (llvm::count(recursiveStack, type)) {
93 results.push_back(convertedType);
96 recursiveStack.push_back(type);
97 auto popConversionCallStack = llvm::make_scope_exit(
98 [&recursiveStack]() { recursiveStack.pop_back(); });
101 convertedElemTypes.reserve(type.getBody().size());
102 if (failed(
convertTypes(type.getBody(), convertedElemTypes)))
107 if (!convertedType.isInitialized()) {
109 convertedType.setBody(convertedElemTypes, type.isPacked()))) {
112 results.push_back(convertedType);
120 convertedType.isPacked() == type.isPacked()) {
121 results.push_back(convertedType);
129 convertedSubtypes.reserve(type.getBody().size());
130 if (failed(
convertTypes(type.getBody(), convertedSubtypes)))
134 type.getContext(), convertedSubtypes, type.isPacked()));
137 addConversion([&](LLVM::LLVMArrayType type) -> std::optional<Type> {
138 if (
auto element =
convertType(type.getElementType()))
142 addConversion([&](LLVM::LLVMFunctionType type) -> std::optional<Type> {
144 if (!convertedResType)
148 convertedArgTypes.reserve(type.getNumParams());
149 if (failed(
convertTypes(type.getParams(), convertedArgTypes)))
163 Location loc) -> std::optional<Value> {
164 if (inputs.size() == 1) {
174 return builder.
create<UnrealizedConversionCastOp>(loc, resultType, desc)
179 Location loc) -> std::optional<Value> {
181 if (inputs.size() == 1) {
184 BlockArgument barePtr = dyn_cast<BlockArgument>(inputs.front());
187 Block *block = barePtr.getOwner();
188 if (!block->isEntryBlock() ||
189 !isa<FunctionOpInterface>(block->getParentOp()))
191 desc = MemRefDescriptor::fromStaticShape(builder, loc, *this, resultType,
194 desc = MemRefDescriptor::pack(builder, loc, *this, resultType, inputs);
199 return builder.
create<UnrealizedConversionCastOp>(loc, resultType, desc)
206 Location loc) -> std::optional<Value> {
207 if (inputs.size() != 1)
210 return builder.
create<UnrealizedConversionCastOp>(loc, resultType, inputs)
215 Location loc) -> std::optional<Value> {
216 if (inputs.size() != 1)
219 return builder.
create<UnrealizedConversionCastOp>(loc, resultType, inputs)
225 [](
BaseMemRefType memref, IntegerAttr addrspace) {
return addrspace; });
238 return options.
dataLayout.getPointerSizeInBits(addressSpace);
241 Type LLVMTypeConverter::convertIndexType(IndexType type)
const {
245 Type LLVMTypeConverter::convertIntegerType(IntegerType type)
const {
249 Type LLVMTypeConverter::convertFloatType(
FloatType type)
const {
261 Type LLVMTypeConverter::convertComplexType(ComplexType type)
const {
262 auto elementType =
convertType(type.getElementType());
264 {elementType, elementType});
269 Type LLVMTypeConverter::convertFunctionType(FunctionType type)
const {
278 FunctionType funcTy,
bool isVariadic,
bool useBarePtrCallConv,
287 if (failed(funcArgConverter(*
this, type, converted)))
296 funcTy.getNumResults() == 0
307 std::pair<LLVM::LLVMFunctionType, LLVM::LLVMStructType>
311 Type resultType = type.getNumResults() == 0
318 auto structType = dyn_cast<LLVM::LLVMStructType>(resultType);
322 inputs.push_back(ptrType);
326 for (
Type t : type.getInputs()) {
330 if (isa<MemRefType, UnrankedMemRefType>(t))
332 inputs.push_back(converted);
367 LLVMTypeConverter::getMemRefDescriptorFields(MemRefType type,
368 bool unpackAggregates)
const {
372 "conversion to strided form failed either due to non-strided layout "
373 "maps (which should have been normalized away) or other reasons");
382 if (failed(addressSpace)) {
384 "conversion of memref memory space ")
385 << type.getMemorySpace()
386 <<
" to integer address space "
387 "failed. Consider adding memory space conversions.";
395 auto rank = type.getRank();
399 if (unpackAggregates)
400 results.insert(results.end(), 2 * rank, indexTy);
417 Type LLVMTypeConverter::convertMemRefType(MemRefType type)
const {
421 getMemRefDescriptorFields(type,
false);
435 LLVMTypeConverter::getUnrankedMemRefDescriptorFields()
const {
447 Type LLVMTypeConverter::convertUnrankedMemRefType(
452 getUnrankedMemRefDescriptorFields());
459 std::optional<Attribute> converted =
465 if (
auto explicitSpace = llvm::dyn_cast_if_present<IntegerAttr>(*converted))
466 return explicitSpace.getInt();
472 if (isa<UnrankedMemRefType>(type))
478 auto memrefTy = cast<MemRefType>(type);
479 if (!memrefTy.hasStaticShape())
487 for (int64_t stride : strides)
488 if (ShapedType::isDynamic(stride))
491 return !ShapedType::isDynamic(offset);
502 if (failed(addressSpace))
514 FailureOr<Type> LLVMTypeConverter::convertVectorType(VectorType type)
const {
515 auto elementType =
convertType(type.getElementType());
518 if (type.getShape().empty())
521 type.getScalableDims().back());
523 "expected vector type compatible with the LLVM dialect");
525 if (llvm::is_contained(type.getScalableDims().drop_back(),
true))
527 auto shape = type.getShape();
528 for (
int i = shape.size() - 2; i >= 0; --i)
539 Type type,
bool useBarePtrCallConv)
const {
540 if (useBarePtrCallConv)
541 if (
auto memrefTy = dyn_cast<BaseMemRefType>(type))
542 return convertMemRefToBarePtr(memrefTy);
553 assert(stdTypes.size() == values.size() &&
554 "The number of types and values doesn't match");
555 for (
unsigned i = 0, end = values.size(); i < end; ++i)
556 if (
auto memrefTy = dyn_cast<MemRefType>(stdTypes[i]))
558 memrefTy, values[i]);
566 assert(!types.empty() &&
"expected non-empty list of type");
567 if (types.size() == 1)
571 resultTypes.reserve(types.size());
572 for (
Type type : types) {
576 resultTypes.push_back(converted);
587 bool useBarePtrCallConv)
const {
588 assert(!types.empty() &&
"expected non-empty list of type");
591 if (types.size() == 1)
595 resultTypes.reserve(types.size());
596 for (
auto t : types) {
600 resultTypes.push_back(converted);
614 builder.
create<LLVM::AllocaOp>(loc, ptrType, operand.
getType(), one);
616 builder.
create<LLVM::StoreOp>(loc, operand, allocated);
623 bool useBarePtrCallConv)
const {
625 promotedOperands.reserve(operands.size());
627 for (
auto it : llvm::zip(opOperands, operands)) {
628 auto operand = std::get<0>(it);
629 auto llvmOperand = std::get<1>(it);
631 if (useBarePtrCallConv) {
634 if (dyn_cast<MemRefType>(operand.getType())) {
637 }
else if (isa<UnrankedMemRefType>(operand.getType())) {
638 llvm_unreachable(
"Unranked memrefs are not supported");
641 if (isa<UnrankedMemRefType>(operand.getType())) {
646 if (
auto memrefType = dyn_cast<MemRefType>(operand.getType())) {
653 promotedOperands.push_back(llvmOperand);
655 return promotedOperands;
665 if (
auto memref = dyn_cast<MemRefType>(type)) {
669 converter.getMemRefDescriptorFields(memref,
true);
670 if (converted.empty())
672 result.append(converted.begin(), converted.end());
675 if (isa<UnrankedMemRefType>(type)) {
676 auto converted = converter.getUnrankedMemRefDescriptorFields();
677 if (converted.empty())
679 result.append(converted.begin(), converted.end());
685 result.push_back(converted);
699 result.push_back(llvmTy);
static llvm::ManagedStatic< PassManagerOptions > options
This class provides a shared interface for ranked and unranked memref types.
Attribute getMemorySpace() const
Returns the memory space in which data referred to by this memref resides.
Type getElementType() const
Returns the element type of this memref type.
IntegerAttr getIndexAttr(int64_t value)
MLIRContext * getContext() const
This class implements a pattern rewriter for use with ConversionPatterns.
Stores data layout objects for each operation that specifies the data layout above and below the give...
The main mechanism for performing data layout queries.
llvm::TypeSize getTypeSize(Type t) const
Returns the size of the given type in the current scope.
unsigned getWidth()
Return the bitwidth of this float type.
Conversion from types to the LLVM IR dialect.
LLVM::LLVMDialect * llvmDialect
Pointer to the LLVM dialect.
llvm::sys::SmartRWMutex< true > callStackMutex
Type packOperationResults(TypeRange types) const
Convert a non-empty list of types of values produced by an operation into an LLVM-compatible type.
LLVM::LLVMDialect * getDialect() const
Returns the LLVM dialect.
Type packFunctionResults(TypeRange types, bool useBarePointerCallConv=false) const
Convert a non-empty list of types to be returned from a function into an LLVM-compatible type.
unsigned getUnrankedMemRefDescriptorSize(UnrankedMemRefType type, const DataLayout &layout) const
Returns the size of the unranked memref descriptor object in bytes.
Type convertFunctionSignature(FunctionType funcTy, bool isVariadic, bool useBarePtrCallConv, SignatureConversion &result) const
Convert a function type.
void promoteBarePtrsToDescriptors(ConversionPatternRewriter &rewriter, Location loc, ArrayRef< Type > stdTypes, SmallVectorImpl< Value > &values) const
Promote the bare pointers in 'values' that resulted from memrefs to descriptors.
DenseMap< uint64_t, std::unique_ptr< SmallVector< Type > > > conversionCallStack
SmallVector< Type > & getCurrentThreadRecursiveStack()
Value promoteOneMemRefDescriptor(Location loc, Value operand, OpBuilder &builder) const
Promote the LLVM struct representation of one MemRef descriptor to stack and use pointer to struct to...
Type convertCallingConventionType(Type type, bool useBarePointerCallConv=false) const
Convert a type in the context of the default or bare pointer calling convention.
LogicalResult convertType(Type t, SmallVectorImpl< Type > &results) const
Convert the given type.
SmallVector< Value, 4 > promoteOperands(Location loc, ValueRange opOperands, ValueRange operands, OpBuilder &builder, bool useBarePtrCallConv=false) const
Promote the LLVM representation of all operands including promoting MemRef descriptors to stack and u...
LLVMTypeConverter(MLIRContext *ctx, const DataLayoutAnalysis *analysis=nullptr)
Create an LLVMTypeConverter using the default LowerToLLVMOptions.
unsigned getPointerBitwidth(unsigned addressSpace=0) const
Gets the pointer bitwidth.
FailureOr< unsigned > getMemRefAddressSpace(BaseMemRefType type) const
Return the LLVM address space corresponding to the memory space of the memref type type or failure if...
static bool canConvertToBarePtr(BaseMemRefType type)
Check if a memref type can be converted to a bare pointer.
MLIRContext & getContext() const
Returns the MLIR context.
unsigned getMemRefDescriptorSize(MemRefType type, const DataLayout &layout) const
Returns the size of the memref descriptor object in bytes.
std::pair< LLVM::LLVMFunctionType, LLVM::LLVMStructType > convertFunctionTypeCWrapper(FunctionType type) const
Converts the function type to a C-compatible format, in particular using pointers to memref descripto...
unsigned getIndexTypeBitwidth() const
Gets the bitwidth of the index type when converted to LLVM.
Type getIndexType() const
Gets the LLVM representation of the index type.
friend LogicalResult structFuncArgTypeConverter(const LLVMTypeConverter &converter, Type type, SmallVectorImpl< Type > &result)
Give structFuncArgTypeConverter access to memref-specific functions.
LLVM dialect structure type representing a collection of different-typed elements manipulated togethe...
static LLVMStructType getLiteral(MLIRContext *context, ArrayRef< Type > types, bool isPacked=false)
Gets or creates a literal struct with the given body in the provided context.
static LLVMStructType getIdentified(MLIRContext *context, StringRef name)
Gets or creates an identified struct with the given name in the provided context.
This class defines the main interface for locations in MLIR and acts as a non-nullable wrapper around...
Options to control the LLVM lowering.
llvm::DataLayout dataLayout
The data layout of the module to produce.
MLIRContext is the top-level object for a collection of MLIR operations.
Helper class to produce LLVM dialect operations extracting or inserting elements of a MemRef descript...
Value alignedPtr(OpBuilder &builder, Location loc)
Builds IR extracting the aligned pointer from the descriptor.
static void unpack(OpBuilder &builder, Location loc, Value packed, MemRefType type, SmallVectorImpl< Value > &results)
Builds IR extracting individual elements of a MemRef descriptor structure and returning them as resul...
static MemRefDescriptor fromStaticShape(OpBuilder &builder, Location loc, const LLVMTypeConverter &typeConverter, MemRefType type, Value memory)
Builds IR creating a MemRef descriptor that represents type and populates it with static shape and st...
This class helps build Operations.
Operation * create(const OperationState &state)
Creates an operation given the fields represented as an OperationState.
OpResult getResult(unsigned idx)
Get the 'idx'th result of this operation.
This class provides all of the information necessary to convert a type signature.
void addInputs(unsigned origInputNo, ArrayRef< Type > types)
Remap an input of the original signature with a new set of types.
ArrayRef< Type > getConvertedTypes() const
Return the argument types for the new signature.
std::optional< Attribute > convertTypeAttribute(Type type, Attribute attr) const
Convert an attribute present attr from within the type type using the registered conversion functions...
void addConversion(FnT &&callback)
Register a conversion function.
void addArgumentMaterialization(FnT &&callback)
All of the following materializations require function objects that are convertible to the following ...
void addSourceMaterialization(FnT &&callback)
This method registers a materialization that will be called when converting a legal replacement value...
void addTypeAttributeConversion(FnT &&callback)
Register a conversion function for attributes within types.
void addTargetMaterialization(FnT &&callback)
This method registers a materialization that will be called when converting an illegal (source) value...
LogicalResult convertTypes(TypeRange types, SmallVectorImpl< Type > &results) const
Convert the given set of types, filling 'results' as necessary.
This class provides an abstraction over the various different ranges of value types.
Instances of the Type class are uniqued, have an immutable identifier and an optional mutable compone...
MLIRContext * getContext() const
Return the MLIRContext in which this type was uniqued.
bool isFloat8E4M3FN() const
bool isFloat8E4M3FNUZ() const
bool isFloat8E4M3B11FNUZ() const
bool isFloat8E5M2() const
bool isFloat8E4M3() const
bool isFloat8E5M2FNUZ() const
static Value pack(OpBuilder &builder, Location loc, const LLVMTypeConverter &converter, UnrankedMemRefType type, ValueRange values)
Builds IR populating an unranked MemRef descriptor structure from a list of individual constituent va...
static void unpack(OpBuilder &builder, Location loc, Value packed, SmallVectorImpl< Value > &results)
Builds IR extracting individual elements that compose an unranked memref descriptor and returns them ...
This class provides an abstraction over the different types of ranges over Values.
This class represents an instance of an SSA value in the MLIR system, representing a computable value...
Type getType() const
Return the type of this value.
bool isCompatibleVectorType(Type type)
Returns true if the given type is a vector type compatible with the LLVM dialect.
bool isCompatibleType(Type type)
Returns true if the given type is compatible with the LLVM dialect.
constexpr void enumerate(std::tuple< Tys... > &tuple, CallbackT &&callback)
llvm::TypeSize divideCeil(llvm::TypeSize numerator, uint64_t denominator)
Divides the known min value of the numerator by the denominator and rounds the result up to the next ...
Include the generated interface declarations.
LogicalResult barePtrFuncArgTypeConverter(const LLVMTypeConverter &converter, Type type, SmallVectorImpl< Type > &result)
Callback to convert function argument types.
InFlightDiagnostic emitError(Location loc)
Utility method to emit an error message using this location.
LogicalResult getStridesAndOffset(MemRefType t, SmallVectorImpl< int64_t > &strides, int64_t &offset)
Returns the strides of the MemRef if the layout map is in strided form.
LogicalResult structFuncArgTypeConverter(const LLVMTypeConverter &converter, Type type, SmallVectorImpl< Type > &result)
Callback to convert function argument types.
auto get(MLIRContext *context, Ts &&...params)
Helper method that injects context only if needed, this helps unify some of the attribute constructio...
bool isStrided(MemRefType t)
Return "true" if the layout for t is compatible with strided semantics.