57 return builder.
create<LLVM::ConstantOp>(loc, resultType,
65 auto [strides, offset] = type.getStridesAndOffset();
77 for (
int i = 0, e = indices.size(); i < e; ++i) {
78 Value increment = indices[i];
79 if (strides[i] != 1) {
81 ShapedType::isDynamic(strides[i])
82 ? memRefDescriptor.
stride(rewriter, loc, i)
84 increment = rewriter.
create<LLVM::MulOp>(loc, increment, stride);
87 index ? rewriter.
create<LLVM::AddOp>(loc, index, increment) : increment;
91 return index ? rewriter.
create<LLVM::GEPOp>(
101 MemRefType type)
const {
102 if (!type.getLayout().isIdentity())
109 if (failed(addressSpace))
119 "layout maps must have been normalized away");
120 assert(count(memRefType.getShape(), ShapedType::kDynamic) ==
121 static_cast<ssize_t
>(dynamicSizes.size()) &&
122 "dynamicSizes size doesn't match dynamic sizes count in memref shape");
124 sizes.reserve(memRefType.getRank());
125 unsigned dynamicIndex = 0;
127 for (int64_t size : memRefType.getShape()) {
129 size == ShapedType::kDynamic
130 ? dynamicSizes[dynamicIndex++]
137 strides.resize(memRefType.getRank());
138 for (
auto i = memRefType.getRank(); i-- > 0;) {
139 strides[i] = runningStride;
141 int64_t staticSize = memRefType.getShape()[i];
142 bool useSizeAsStride = stride == 1;
143 if (staticSize == ShapedType::kDynamic)
144 stride = ShapedType::kDynamic;
145 if (stride != ShapedType::kDynamic)
146 stride *= staticSize;
149 runningStride = sizes[i];
150 else if (stride == ShapedType::kDynamic)
152 rewriter.
create<LLVM::MulOp>(loc, runningStride, sizes[i]);
160 Value nullPtr = rewriter.
create<LLVM::ZeroOp>(loc, elementPtrType);
162 loc, elementPtrType, elementType, nullPtr, runningStride);
165 size = runningStride;
178 auto nullPtr = rewriter.
create<LLVM::ZeroOp>(loc, convertedPtrType);
179 auto gep = rewriter.
create<LLVM::GEPOp>(loc, convertedPtrType, llvmType,
187 assert(count(memRefType.getShape(), ShapedType::kDynamic) ==
188 static_cast<ssize_t
>(dynamicSizes.size()) &&
189 "dynamicSizes size doesn't match dynamic sizes count in memref shape");
192 Value numElements = memRefType.getRank() == 0
195 unsigned dynamicIndex = 0;
198 for (int64_t staticSize : memRefType.getShape()) {
201 staticSize == ShapedType::kDynamic
202 ? dynamicSizes[dynamicIndex++]
204 numElements = rewriter.
create<LLVM::MulOp>(loc, numElements, size);
207 staticSize == ShapedType::kDynamic
208 ? dynamicSizes[dynamicIndex++]
224 memRefDescriptor.setAllocatedPtr(rewriter, loc, allocatedPtr);
227 memRefDescriptor.setAlignedPtr(rewriter, loc, alignedPtr);
231 memRefDescriptor.setOffset(
236 memRefDescriptor.setSize(rewriter, loc, en.index(), en.value());
240 memRefDescriptor.setStride(rewriter, loc, en.index(), en.value());
242 return memRefDescriptor;
248 assert(origTypes.size() == operands.size() &&
249 "expected as may original types as operands");
254 for (
unsigned i = 0, e = operands.size(); i < e; ++i) {
255 if (
auto memRefType = dyn_cast<UnrankedMemRefType>(origTypes[i])) {
256 unrankedMemrefs.emplace_back(operands[i]);
257 FailureOr<unsigned> addressSpace =
259 if (failed(addressSpace))
261 unrankedAddressSpaces.emplace_back(*addressSpace);
265 if (unrankedMemrefs.empty())
271 unrankedMemrefs, unrankedAddressSpaces,
279 FailureOr<LLVM::LLVMFuncOp> freeFunc, mallocFunc;
282 if (failed(mallocFunc))
287 if (failed(freeFunc))
291 unsigned unrankedMemrefPos = 0;
292 for (
unsigned i = 0, e = operands.size(); i < e; ++i) {
293 Type type = origTypes[i];
294 if (!isa<UnrankedMemRefType>(type))
296 Value allocationSize = sizes[unrankedMemrefPos++];
303 .
create<LLVM::CallOp>(loc, mallocFunc.value(), allocationSize)
310 builder.
create<LLVM::MemcpyOp>(loc, memory, source, allocationSize,
false);
312 builder.
create<LLVM::CallOp>(loc, freeFunc.value(), source);
324 updatedDesc.setRank(builder, loc, rank);
325 updatedDesc.setMemRefDescPtr(builder, loc, memory);
327 operands[i] = updatedDesc;
338 IntegerOverflowFlags overflowFlags) {
339 if (
auto iface = dyn_cast<IntegerOverflowFlagsInterface>(op))
340 iface.setOverflowFlags(overflowFlags);
349 IntegerOverflowFlags overflowFlags) {
353 if (numResults != 0) {
354 resultTypes.push_back(
356 if (!resultTypes.back())
363 resultTypes, targetAttrs);
369 return rewriter.
eraseOp(op), success();
376 results.reserve(numResults);
377 for (
unsigned i = 0; i < numResults; ++i) {
378 results.push_back(rewriter.
create<LLVM::ExtractValueOp>(
389 auto vec = cast<VectorType>(type);
390 assert(!vec.isScalable() &&
"scalable vectors are not supported");
391 return vec.getNumElements() *
getBitWidth(vec.getElementType());
397 return builder.
create<LLVM::ConstantOp>(loc, i32, value);
403 if (srcType == dstType)
408 if (srcBitWidth == dstBitWidth) {
409 Value cast = builder.
create<LLVM::BitcastOp>(loc, dstType, src);
413 if (dstBitWidth > srcBitWidth) {
415 if (srcType != smallerInt)
416 src = builder.
create<LLVM::BitcastOp>(loc, smallerInt, src);
419 Value res = builder.
create<LLVM::ZExtOp>(loc, largerInt, src);
422 assert(srcBitWidth % dstBitWidth == 0 &&
423 "src bit width must be a multiple of dst bit width");
424 int64_t numElements = srcBitWidth / dstBitWidth;
427 src = builder.
create<LLVM::BitcastOp>(loc, vecType, src);
430 for (
auto i : llvm::seq(numElements)) {
432 Value elem = builder.
create<LLVM::ExtractElementOp>(loc, src, idx);
433 res.emplace_back(elem);
441 assert(!src.empty() &&
"src range must not be empty");
442 if (src.size() == 1) {
443 Value res = src.front();
449 if (dstBitWidth < srcBitWidth) {
451 if (res.
getType() != largerInt)
452 res = builder.
create<LLVM::BitcastOp>(loc, largerInt, res);
455 res = builder.
create<LLVM::TruncOp>(loc, smallerInt, res);
459 res = builder.
create<LLVM::BitcastOp>(loc, dstType, res);
464 int64_t numElements = src.size();
466 Value res = builder.
create<LLVM::PoisonOp>(loc, srcType);
469 res = builder.
create<LLVM::InsertElementOp>(loc, srcType, res, elem, idx);
473 res = builder.
create<LLVM::BitcastOp>(loc, dstType, res);
static Value createI32Constant(OpBuilder &builder, Location loc, int32_t value)
static unsigned getBitWidth(Type type)
IntegerAttr getIndexAttr(int64_t value)
IntegerType getIntegerType(unsigned width)
StringAttr getStringAttr(const Twine &bytes)
MLIRContext * getContext() const
This class implements a pattern rewriter for use with ConversionPatterns.
void replaceOp(Operation *op, ValueRange newValues) override
Replace the given operation with the new values.
void eraseOp(Operation *op) override
PatternRewriter hook for erasing a dead operation.
Base class for the conversion patterns.
const TypeConverter * typeConverter
An optional type converter for use by this pattern.
const TypeConverter * getTypeConverter() const
Return the type converter held by this pattern, or nullptr if the pattern does not require type conve...
Type getVoidType() const
Gets the MLIR type wrapping the LLVM void type.
MemRefDescriptor createMemRefDescriptor(Location loc, MemRefType memRefType, Value allocatedPtr, Value alignedPtr, ArrayRef< Value > sizes, ArrayRef< Value > strides, ConversionPatternRewriter &rewriter) const
Creates and populates a canonical memref descriptor struct.
ConvertToLLVMPattern(StringRef rootOpName, MLIRContext *context, const LLVMTypeConverter &typeConverter, PatternBenefit benefit=1)
void getMemRefDescriptorSizes(Location loc, MemRefType memRefType, ValueRange dynamicSizes, ConversionPatternRewriter &rewriter, SmallVectorImpl< Value > &sizes, SmallVectorImpl< Value > &strides, Value &size, bool sizeInBytes=true) const
Computes sizes, strides and buffer size of memRefType with identity layout.
Type getIndexType() const
Gets the MLIR type wrapping the LLVM integer type whose bit width is defined by the used type convert...
const LLVMTypeConverter * getTypeConverter() const
Value getStridedElementPtr(Location loc, MemRefType type, Value memRefDesc, ValueRange indices, ConversionPatternRewriter &rewriter) const
Value getNumElements(Location loc, MemRefType memRefType, ValueRange dynamicSizes, ConversionPatternRewriter &rewriter) const
Computes total number of elements for the given MemRef and dynamicSizes.
LLVM::LLVMDialect & getDialect() const
Returns the LLVM dialect.
Value getSizeInBytes(Location loc, Type type, ConversionPatternRewriter &rewriter) const
Computes the size of type in bytes.
Type getIntPtrType(unsigned addressSpace=0) const
Gets the MLIR type wrapping the LLVM integer type whose bit width corresponds to that of a LLVM point...
LogicalResult copyUnrankedDescriptors(OpBuilder &builder, Location loc, TypeRange origTypes, SmallVectorImpl< Value > &operands, bool toDynamic) const
Copies the memory descriptor for any operands that were unranked descriptors originally to heap-alloc...
Type getElementPtrType(MemRefType type) const
Returns the type of a pointer to an element of the memref.
static Value createIndexAttrConstant(OpBuilder &builder, Location loc, Type resultType, int64_t value)
Create a constant Op producing a value of resultType from an index-typed integer attribute.
bool isConvertibleAndHasIdentityMaps(MemRefType type) const
Returns if the given memref type is convertible to LLVM and has an identity layout map.
Type getVoidPtrType() const
Get the MLIR type wrapping the LLVM i8* type.
Conversion from types to the LLVM IR dialect.
Type packOperationResults(TypeRange types) const
Convert a non-empty list of types of values produced by an operation into an LLVM-compatible type.
LLVM::LLVMDialect * getDialect() const
Returns the LLVM dialect.
LogicalResult convertType(Type t, SmallVectorImpl< Type > &results) const
Convert the given type.
FailureOr< unsigned > getMemRefAddressSpace(BaseMemRefType type) const
Return the LLVM address space corresponding to the memory space of the memref type type or failure if...
Type getIndexType() const
Gets the LLVM representation of the index type.
This class defines the main interface for locations in MLIR and acts as a non-nullable wrapper around...
MLIRContext is the top-level object for a collection of MLIR operations.
Helper class to produce LLVM dialect operations extracting or inserting elements of a MemRef descript...
Value bufferPtr(OpBuilder &builder, Location loc, const LLVMTypeConverter &converter, MemRefType type)
Builds IR for getting the start address of the buffer represented by this memref: memref....
LLVM::LLVMPointerType getElementPtrType()
Returns the (LLVM) pointer type this descriptor contains.
Value stride(OpBuilder &builder, Location loc, unsigned pos)
Builds IR extracting the pos-th size from the descriptor.
static MemRefDescriptor poison(OpBuilder &builder, Location loc, Type descriptorType)
Builds IR creating a poison value of the descriptor type.
This class helps build Operations.
Block::iterator getInsertionPoint() const
Returns the current insertion point of the builder.
Operation * create(const OperationState &state)
Creates an operation given the fields represented as an OperationState.
Operation is the basic unit of execution within MLIR.
OpResult getResult(unsigned idx)
Get the 'idx'th result of this operation.
Location getLoc()
The source location the operation was defined or derived from.
result_type_range getResultTypes()
unsigned getNumResults()
Return the number of results held by this operation.
This class represents the benefit of a pattern match in a unitless scheme that ranges from 0 (very li...
MLIRContext * getContext() const
Return the MLIRContext used to create this pattern.
LogicalResult convertType(Type t, SmallVectorImpl< Type > &results) const
Convert the given type.
This class provides an abstraction over the various different ranges of value types.
Instances of the Type class are uniqued, have an immutable identifier and an optional mutable compone...
bool isIntOrFloat() const
Return true if this is an integer (of any signedness) or a float type.
unsigned getIntOrFloatBitWidth() const
Return the bit width of an integer or a float type, assert failure on other types.
Value memRefDescPtr(OpBuilder &builder, Location loc) const
Builds IR extracting ranked memref descriptor ptr.
static UnrankedMemRefDescriptor poison(OpBuilder &builder, Location loc, Type descriptorType)
Builds IR creating an undef value of the descriptor type.
static void computeSizes(OpBuilder &builder, Location loc, const LLVMTypeConverter &typeConverter, ArrayRef< UnrankedMemRefDescriptor > values, ArrayRef< unsigned > addressSpaces, SmallVectorImpl< Value > &sizes)
Builds IR computing the sizes in bytes (suitable for opaque allocation) and appends the corresponding...
Value rank(OpBuilder &builder, Location loc) const
Builds IR extracting the rank from the descriptor.
This class provides an abstraction over the different types of ranges over Values.
type_range getType() const
This class represents an instance of an SSA value in the MLIR system, representing a computable value...
Type getType() const
Return the type of this value.
LogicalResult oneToOneRewrite(Operation *op, StringRef targetOp, ValueRange operands, ArrayRef< NamedAttribute > targetAttrs, const LLVMTypeConverter &typeConverter, ConversionPatternRewriter &rewriter, IntegerOverflowFlags overflowFlags=IntegerOverflowFlags::none)
Replaces the given operation "op" with a new operation of type "targetOp" and given operands.
void setNativeProperties(Operation *op, IntegerOverflowFlags overflowFlags)
Handle generically setting flags as native properties on LLVM operations.
Value composeValue(OpBuilder &builder, Location loc, ValueRange src, Type dstType)
Composes a set of src values into a single value of type dstType through series of bitcasts and vecto...
FailureOr< LLVM::LLVMFuncOp > lookupOrCreateFreeFn(OpBuilder &b, Operation *moduleOp)
SmallVector< Value > decomposeValue(OpBuilder &builder, Location loc, Value src, Type dstType)
Decomposes a src value into a set of values of type dstType through series of bitcasts and vector ops...
FailureOr< LLVM::LLVMFuncOp > lookupOrCreateMallocFn(OpBuilder &b, Operation *moduleOp, Type indexType)
constexpr void enumerate(std::tuple< Tys... > &tuple, CallbackT &&callback)
Include the generated interface declarations.
auto get(MLIRContext *context, Ts &&...params)
Helper method that injects context only if needed, this helps unify some of the attribute constructio...