#include "mlir/Dialect/Arith/IR/Arith.h"
#include "mlir/Dialect/Arith/Utils/Utils.h"
#include "mlir/Dialect/MemRef/IR/MemRef.h"
#include "mlir/Dialect/Utils/StaticValueUtils.h"
#include "mlir/IR/AffineMap.h"
#include "mlir/IR/Builders.h"
#include "mlir/IR/BuiltinTypes.h"
#include "mlir/IR/Matchers.h"
#include "mlir/IR/OpDefinition.h"
#include "mlir/IR/PatternMatch.h"
#include "mlir/IR/TypeUtilities.h"
#include "mlir/Interfaces/InferTypeOpInterface.h"
#include "mlir/Interfaces/SideEffectInterfaces.h"
#include "mlir/Interfaces/ViewLikeInterface.h"
#include "llvm/ADT/STLExtras.h"
#include "llvm/ADT/SmallBitVector.h"
#include "mlir/Dialect/MemRef/IR/MemRefOps.cpp.inc"

Classes
struct	AllocaScopeInliner
	Inline an AllocaScopeOp if either the direct parent is an allocation scope or it contains no allocation. More...

struct	AllocaScopeHoister
	Move allocations into an allocation scope, if it is legal to move them (e.g. More...

struct	CollapseShapeOpMemRefCastFolder

struct	SubViewReturnTypeCanonicalizer
	Return the canonical type of the result of a subview. More...

struct	SubViewCanonicalizer
	A canonicalizer wrapper to replace SubViewOps. More...

Macros
#define	GET_OP_CLASSES

Functions
static void	constifyIndexValues (SmallVectorImpl< OpFoldResult > &values, MemRefType memRefTy, MLIRContext *ctxt, llvm::function_ref< SmallVector< int64_t >(MemRefType)> getAttributes, llvm::function_ref< bool(int64_t)> isDynamic)
	Helper function that infers the constant values from a list of `values`, a `memRefTy`, and another helper function `getAttributes`. More...

static SmallVector< int64_t >	getConstantSizes (MemRefType memRefTy)
	Wrapper around `getShape` that conforms to the function signature expected for `getAttributes` in `constifyIndexValues`. More...

static SmallVector< int64_t >	getConstantOffset (MemRefType memrefType)
	Wrapper around `getStridesAndOffset` that returns only the offset and conforms to the function signature expected for `getAttributes` in `constifyIndexValues`. More...

static SmallVector< int64_t >	getConstantStrides (MemRefType memrefType)
	Wrapper around `getStridesAndOffset` that returns only the strides and conforms to the function signature expected for `getAttributes` in `constifyIndexValues`. More...

template<typename AllocLikeOp >
static LogicalResult	verifyAllocLikeOp (AllocLikeOp op)

static bool	isGuaranteedAutomaticAllocation (Operation *op)
	Given an operation, return whether this op is guaranteed to allocate an AutomaticAllocationScopeResource. More...

static bool	isOpItselfPotentialAutomaticAllocation (Operation *op)
	Given an operation, return whether this op itself could allocate an AutomaticAllocationScopeResource. More...

static bool	lastNonTerminatorInRegion (Operation *op)
	Return whether this op is the last non terminating op in a region. More...

static std::map< int64_t, unsigned >	getNumOccurences (ArrayRef< int64_t > vals)
	Return a map with key being elements in `vals` and data being number of occurences of it. More...

static FailureOr< llvm::SmallBitVector >	computeMemRefRankReductionMask (MemRefType originalType, MemRefType reducedType, ArrayRef< OpFoldResult > sizes)
	Given the `originalType` and a `candidateReducedType` whose shape is assumed to be a subset of `originalType` with some `1` entries erased, return the set of indices that specifies which of the entries of `originalShape` are dropped to obtain `reducedShape`. More...

template<typename Container >
static bool	replaceConstantUsesOf (OpBuilder &rewriter, Location loc, Container values, ArrayRef< OpFoldResult > maybeConstants)
	Helper function to perform the replacement of all constant uses of `values` by a materialized constant extracted from `maybeConstants`. More...

static void	printGlobalMemrefOpTypeAndInitialValue (OpAsmPrinter &p, GlobalOp op, TypeAttr type, Attribute initialValue)

static ParseResult	parseGlobalMemrefOpTypeAndInitialValue (OpAsmParser &parser, TypeAttr &typeAttr, Attribute &initialValue)

static LogicalResult	verifyCollapsedShape (Operation *op, ArrayRef< int64_t > collapsedShape, ArrayRef< int64_t > expandedShape, ArrayRef< ReassociationIndices > reassociation, bool allowMultipleDynamicDimsPerGroup)
	Helper function for verifying the shape of ExpandShapeOp and ResultShapeOp result and operand. More...

static FailureOr< StridedLayoutAttr >	computeExpandedLayoutMap (MemRefType srcType, ArrayRef< int64_t > resultShape, ArrayRef< ReassociationIndices > reassociation)
	Compute the layout map after expanding a given source MemRef type with the specified reassociation indices. More...

static FailureOr< StridedLayoutAttr >	computeCollapsedLayoutMap (MemRefType srcType, ArrayRef< ReassociationIndices > reassociation, bool strict=false)
	Compute the layout map after collapsing a given source MemRef type with the specified reassociation indices. More...

static bool	haveCompatibleOffsets (MemRefType t1, MemRefType t2)
	Return true if `t1` and `t2` have equal offsets (both dynamic or of same static value). More...

static bool	haveCompatibleStrides (MemRefType t1, MemRefType t2, const llvm::SmallBitVector &droppedDims)
	Return true if `t1` and `t2` have equal strides (both dynamic or of same static value). More...

static LogicalResult	produceSubViewErrorMsg (SliceVerificationResult result, Operation *op, Type expectedType)

static MemRefType	getCanonicalSubViewResultType (MemRefType currentResultType, MemRefType currentSourceType, MemRefType sourceType, ArrayRef< OpFoldResult > mixedOffsets, ArrayRef< OpFoldResult > mixedSizes, ArrayRef< OpFoldResult > mixedStrides)
	Compute the canonical result type of a SubViewOp. More...

static bool	isTrivialSubViewOp (SubViewOp subViewOp)
	Helper method to check if a `subview` operation is trivially a no-op. More...

static MemRefType	inferTransposeResultType (MemRefType memRefType, AffineMap permutationMap)
	Build a strided memref type by applying `permutationMap` to `memRefType`. More...

Macro Definition Documentation

◆ GET_OP_CLASSES

#define GET_OP_CLASSES

Definition at line 3614 of file MemRefOps.cpp.

Function Documentation

◆ computeCollapsedLayoutMap()

static FailureOr<StridedLayoutAttr> computeCollapsedLayoutMap	(	MemRefType	srcType,
		ArrayRef< ReassociationIndices >	reassociation,
		bool	strict = `false`
	)

static

Compute the layout map after collapsing a given source MemRef type with the specified reassociation indices.

Note: All collapsed dims in a reassociation group must be contiguous. It is not possible to check this by inspecting a MemRefType in the general case. If non-contiguity cannot be checked statically, the collapse is assumed to be valid (and thus accepted by this function) unless strict = true.

Definition at line 2419 of file MemRefOps.cpp.

References mlir::get(), mlir::getStridesAndOffset(), and mlir::SaturatedInteger::wrap().

◆ computeExpandedLayoutMap()

static FailureOr<StridedLayoutAttr> computeExpandedLayoutMap	(	MemRefType	srcType,
		ArrayRef< int64_t >	resultShape,
		ArrayRef< ReassociationIndices >	reassociation
	)

static

Compute the layout map after expanding a given source MemRef type with the specified reassociation indices.

Definition at line 2224 of file MemRefOps.cpp.

References mlir::get(), mlir::getStridesAndOffset(), and mlir::SaturatedInteger::wrap().

◆ computeMemRefRankReductionMask()

static FailureOr<llvm::SmallBitVector> computeMemRefRankReductionMask	(	MemRefType	originalType,
		MemRefType	reducedType,
		ArrayRef< OpFoldResult >	sizes
	)

static

Given the originalType and a candidateReducedType whose shape is assumed to be a subset of originalType with some 1 entries erased, return the set of indices that specifies which of the entries of originalShape are dropped to obtain reducedShape.

This accounts for cases where there are multiple unit-dims, but only a subset of those are dropped. For MemRefTypes these can be disambiguated using the strides. If a dimension is dropped the stride must be dropped too.

Definition at line 938 of file MemRefOps.cpp.

References mlir::detail::enumerate(), getNumOccurences(), and mlir::getStridesAndOffset().

Referenced by getCanonicalSubViewResultType().

◆ constifyIndexValues()

static void constifyIndexValues	(	SmallVectorImpl< OpFoldResult > &	values,
		MemRefType	memRefTy,
		MLIRContext *	ctxt,
		llvm::function_ref< SmallVector< int64_t >(MemRefType)>	getAttributes,
		llvm::function_ref< bool(int64_t)>	isDynamic
	)

static

Helper function that infers the constant values from a list of values, a memRefTy, and another helper function getAttributes.

The inferred constant values replace the related OpFoldResult in values.

Note: This function shouldn't be used directly, instead, use the getConstifiedMixedXXX methods from the related operations.

getAttributes retuns a list of potentially constant values, as determined by isDynamic, from the given memRefTy. The returned list must have as many elements as values or be empty.

E.g., consider the following example:

memref.reinterpret_cast %base to <...> strides: [2, %dyn_stride] :

memref<f32> to memref<?x?xf32, strided<[?, 1], offset: ?>>

ReinterpretCastOp::getMixedStrides() will return [2, dyn_stride]. Now using this helper function with:

values == [2, dyn_stride],
memRefTy == memref<?x?xf32, strided<[?, 1], offset: ?>>
getAttributes == getConstantStrides (i.e., a wrapper around getStridesAndOffset), and
isDynamic == ShapedType::isDynamic Will yield: values == [2, 1]

Definition at line 115 of file MemRefOps.cpp.

References mlir::detail::enumerate(), mlir::getConstantIntValue(), and mlir::Builder::getIndexAttr().

◆ getCanonicalSubViewResultType()

static MemRefType getCanonicalSubViewResultType	(	MemRefType	currentResultType,
		MemRefType	currentSourceType,
		MemRefType	sourceType,
		ArrayRef< OpFoldResult >	mixedOffsets,
		ArrayRef< OpFoldResult >	mixedSizes,
		ArrayRef< OpFoldResult >	mixedStrides
	)

static

Compute the canonical result type of a SubViewOp.

Call inferResultType to deduce the result type for the given sourceType. Additionally, reduce the rank of the inferred result type if currentResultType is lower rank than currentSourceType. Use this signature if sourceType is updated together with the result type. In this case, it is important to compute the dropped dimensions using currentSourceType whose strides align with currentResultType.

Definition at line 3069 of file MemRefOps.cpp.

References computeMemRefRankReductionMask(), and mlir::get().

◆ getConstantOffset()

static SmallVector<int64_t> getConstantOffset ( MemRefType memrefType )

static

Wrapper around getStridesAndOffset that returns only the offset and conforms to the function signature expected for getAttributes in constifyIndexValues.

Definition at line 163 of file MemRefOps.cpp.

References mlir::getStridesAndOffset().

◆ getConstantSizes()

static SmallVector<int64_t> getConstantSizes ( MemRefType memRefTy )

static

Wrapper around getShape that conforms to the function signature expected for getAttributes in constifyIndexValues.

Definition at line 155 of file MemRefOps.cpp.

◆ getConstantStrides()

static SmallVector<int64_t> getConstantStrides ( MemRefType memrefType )

static

Wrapper around getStridesAndOffset that returns only the strides and conforms to the function signature expected for getAttributes in constifyIndexValues.

Definition at line 176 of file MemRefOps.cpp.

References mlir::getStridesAndOffset().

◆ getNumOccurences()

static std::map<int64_t, unsigned> getNumOccurences ( ArrayRef< int64_t > vals )

static

Return a map with key being elements in vals and data being number of occurences of it.

Use std::map, since the vals here are strides and the dynamic stride value is the same as the tombstone value for DenseMap<int64_t>.

Definition at line 923 of file MemRefOps.cpp.

Referenced by computeMemRefRankReductionMask().

◆ haveCompatibleOffsets()

static bool haveCompatibleOffsets	(	MemRefType	t1,
		MemRefType	t2
	)

static

Return true if t1 and t2 have equal offsets (both dynamic or of same static value).

Definition at line 2914 of file MemRefOps.cpp.

References mlir::getStridesAndOffset().

◆ haveCompatibleStrides()

static bool haveCompatibleStrides	(	MemRefType	t1,
		MemRefType	t2,
		const llvm::SmallBitVector &	droppedDims
	)

static

Return true if t1 and t2 have equal strides (both dynamic or of same static value).

Dimensions of t1 may be dropped in t2; these must be marked as dropped in droppedDims.

Definition at line 2925 of file MemRefOps.cpp.

References mlir::getStridesAndOffset().

◆ inferTransposeResultType()

static MemRefType inferTransposeResultType	(	MemRefType	memRefType,
		AffineMap	permutationMap
	)

static

Build a strided memref type by applying permutationMap to memRefType.

Definition at line 3343 of file MemRefOps.cpp.

References mlir::get(), mlir::getStridesAndOffset(), mlir::MemRefType::Builder::setLayout(), and mlir::MemRefType::Builder::setShape().

◆ isGuaranteedAutomaticAllocation()

static bool isGuaranteedAutomaticAllocation ( Operation * op )

static

Given an operation, return whether this op is guaranteed to allocate an AutomaticAllocationScopeResource.

Definition at line 430 of file MemRefOps.cpp.

References mlir::Operation::getResults().

◆ isOpItselfPotentialAutomaticAllocation()

static bool isOpItselfPotentialAutomaticAllocation ( Operation * op )

static

Given an operation, return whether this op itself could allocate an AutomaticAllocationScopeResource.

Note that this will not check whether an operation contained within the op can allocate.

Definition at line 449 of file MemRefOps.cpp.

References mlir::Operation::getResults(), and mlir::Operation::hasTrait().

Referenced by AllocaScopeInliner::matchAndRewrite().

◆ isTrivialSubViewOp()

static bool isTrivialSubViewOp ( SubViewOp subViewOp )

static

Helper method to check if a subview operation is trivially a no-op.

This is the case if the all offsets are zero, all strides are 1, and the source shape is same as the size of the subview. In such cases, the subview can be folded into its source.

Definition at line 3134 of file MemRefOps.cpp.

References mlir::detail::enumerate(), and mlir::getConstantIntValue().

◆ lastNonTerminatorInRegion()

static bool lastNonTerminatorInRegion ( Operation * op )

static

Return whether this op is the last non terminating op in a region.

That is to say, it is in a one-block region and is only followed by a terminator. This prevents extending the lifetime of allocations.

Definition at line 472 of file MemRefOps.cpp.

References mlir::Operation::getBlock(), mlir::Region::getBlocks(), mlir::Operation::getParentRegion(), and mlir::Block::getTerminator().

Referenced by AllocaScopeInliner::matchAndRewrite(), and AllocaScopeHoister::matchAndRewrite().

◆ parseGlobalMemrefOpTypeAndInitialValue()

static ParseResult parseGlobalMemrefOpTypeAndInitialValue	(	OpAsmParser &	parser,
		TypeAttr &	typeAttr,
		Attribute &	initialValue
	)

static

Definition at line 1565 of file MemRefOps.cpp.

References mlir::AsmParser::emitError(), mlir::get(), mlir::AsmParser::getContext(), mlir::AsmParser::getNameLoc(), mlir::memref::getTensorTypeFromMemRefType(), mlir::AsmParser::parseAttribute(), mlir::AsmParser::parseOptionalEqual(), mlir::AsmParser::parseOptionalKeyword(), and mlir::AsmParser::parseType().

◆ printGlobalMemrefOpTypeAndInitialValue()

static void printGlobalMemrefOpTypeAndInitialValue	(	OpAsmPrinter &	p,
		GlobalOp	op,
		TypeAttr	type,
		Attribute	initialValue
	)

static

Definition at line 1551 of file MemRefOps.cpp.

References mlir::AsmPrinter::printAttributeWithoutType().

◆ produceSubViewErrorMsg()

static LogicalResult produceSubViewErrorMsg	(	SliceVerificationResult	result,
		Operation *	op,
		Type	expectedType
	)

static

Definition at line 2947 of file MemRefOps.cpp.

References mlir::ElemTypeMismatch, mlir::Operation::emitError(), mlir::LayoutMismatch, mlir::MemSpaceMismatch, mlir::RankTooLarge, mlir::SizeMismatch, and mlir::Success.

◆ replaceConstantUsesOf()

template<typename Container >

static bool replaceConstantUsesOf	(	OpBuilder &	rewriter,
		Location	loc,
		Container	values,
		ArrayRef< OpFoldResult >	maybeConstants
	)

static

Helper function to perform the replacement of all constant uses of values by a materialized constant extracted from maybeConstants.

values and maybeConstants are expected to have the same size.

Definition at line 1398 of file MemRefOps.cpp.

References mlir::OpBuilder::create(), mlir::getAsOpFoldResult(), and mlir::Operation::replaceUsesOfWith().

◆ verifyAllocLikeOp()

template<typename AllocLikeOp >

static LogicalResult verifyAllocLikeOp ( AllocLikeOp op )

static

Definition at line 201 of file MemRefOps.cpp.

◆ verifyCollapsedShape()

static LogicalResult verifyCollapsedShape	(	Operation *	op,
		ArrayRef< int64_t >	collapsedShape,
		ArrayRef< int64_t >	expandedShape,
		ArrayRef< ReassociationIndices >	reassociation,
		bool	allowMultipleDynamicDimsPerGroup
	)

static

Helper function for verifying the shape of ExpandShapeOp and ResultShapeOp result and operand.

Layout maps are verified separately.

If allowMultipleDynamicDimsPerGroup, multiple dynamic dimensions are allowed in a reassocation group.

Definition at line 2131 of file MemRefOps.cpp.

References mlir::Operation::emitOpError(), and mlir::detail::enumerate().

Classes

Macros

Functions

Macro Definition Documentation

◆ GET_OP_CLASSES

Function Documentation

◆ computeCollapsedLayoutMap()

◆ computeExpandedLayoutMap()

◆ computeMemRefRankReductionMask()

◆ constifyIndexValues()

◆ getCanonicalSubViewResultType()

◆ getConstantOffset()

◆ getConstantSizes()

◆ getConstantStrides()

◆ getNumOccurences()

◆ haveCompatibleOffsets()

◆ haveCompatibleStrides()

◆ inferTransposeResultType()

◆ isGuaranteedAutomaticAllocation()

◆ isOpItselfPotentialAutomaticAllocation()

◆ isTrivialSubViewOp()

◆ lastNonTerminatorInRegion()

◆ parseGlobalMemrefOpTypeAndInitialValue()

◆ printGlobalMemrefOpTypeAndInitialValue()

◆ produceSubViewErrorMsg()

◆ replaceConstantUsesOf()

◆ verifyAllocLikeOp()

◆ verifyCollapsedShape()