#include "mlir/Dialect/Vector/IR/VectorOps.h"
#include "mlir/Dialect/Affine/IR/ValueBoundsOpInterfaceImpl.h"
#include "mlir/Dialect/Arith/IR/Arith.h"
#include "mlir/Dialect/Arith/Utils/Utils.h"
#include "mlir/Dialect/Bufferization/IR/BufferizableOpInterface.h"
#include "mlir/Dialect/MemRef/IR/MemRef.h"
#include "mlir/Dialect/Tensor/IR/Tensor.h"
#include "mlir/Dialect/Utils/IndexingUtils.h"
#include "mlir/Dialect/Utils/StructuredOpsUtils.h"
#include "mlir/IR/AffineExpr.h"
#include "mlir/IR/AffineMap.h"
#include "mlir/IR/Builders.h"
#include "mlir/IR/BuiltinAttributes.h"
#include "mlir/IR/BuiltinOps.h"
#include "mlir/IR/BuiltinTypes.h"
#include "mlir/IR/DialectImplementation.h"
#include "mlir/IR/IRMapping.h"
#include "mlir/IR/OpImplementation.h"
#include "mlir/IR/PatternMatch.h"
#include "mlir/IR/TypeUtilities.h"
#include "mlir/Interfaces/SubsetOpInterface.h"
#include "mlir/Interfaces/ValueBoundsOpInterface.h"
#include "mlir/Support/LLVM.h"
#include "mlir/Transforms/InliningUtils.h"
#include "llvm/ADT/ArrayRef.h"
#include "llvm/ADT/STLExtras.h"
#include "llvm/ADT/SmallVector.h"
#include "llvm/ADT/StringSet.h"
#include "llvm/ADT/TypeSwitch.h"
#include "llvm/ADT/bit.h"
#include <cassert>
#include <cstdint>
#include <numeric>
#include "mlir/Dialect/Vector/IR/VectorDialect.cpp.inc"
#include "mlir/Dialect/Vector/IR/VectorEnums.cpp.inc"
#include "mlir/Dialect/Vector/IR/VectorAttributes.cpp.inc"
#include "mlir/Dialect/Vector/IR/VectorOps.cpp.inc"

Classes
struct	mlir::vector::detail::BitmaskEnumStorage

struct	CanonicalizeContractAdd< AddOpType >
	Return a fused vector::ContractionOp which represents a patterns such as: More...

Namespaces
	mlir
	Include the generated interface declarations.

	mlir::vector

	mlir::vector::detail

Macros
#define	GET_ATTRDEF_LIST

#define	GET_OP_LIST

#define	GET_ATTRDEF_CLASSES

#define	GET_OP_CLASSES

Enumerations
enum class	MaskFormat { AllTrue = 0 , AllFalse = 1 , Unknown = 2 }
	Helper enum to classify mask value. More...

Functions
static MaskFormat	getMaskFormat (Value mask)
	Helper method to classify a mask value. More...

static bool	isSupportedCombiningKind (CombiningKind combiningKind, Type elementType)

static LogicalResult	incSlicePosition (MutableArrayRef< int64_t > position, ArrayRef< int64_t > shape, ArrayRef< int64_t > offsets)

static bool	verifyDimMap (VectorType lhsType, VectorType rhsType, const std::vector< std::pair< int64_t, int64_t >> &map)

static LogicalResult	verifyOutputShape (ContractionOp op, VectorType lhsType, VectorType rhsType, Type accType, Type resType, const std::vector< std::pair< int64_t, int64_t >> &contractingDimMap, const std::vector< std::pair< int64_t, int64_t >> &batchDimMap)

static int64_t	getResultIndex (AffineMap map, AffineExpr targetExpr)

static std::vector< std::pair< int64_t, int64_t > >	getDimMap (ArrayRef< AffineMap > indexingMaps, ArrayAttr iteratorTypes, IteratorType targetIteratorType, MLIRContext *context)

template<typename IntType >
static SmallVector< IntType >	extractVector (ArrayAttr arrayAttr)

static LogicalResult	foldExtractOpFromExtractChain (ExtractOp extractOp)
	Fold the result of chains of ExtractOp in place by simply concatenating the positions. More...

static bool	hasZeroDimVectors (Operation *op)
	Returns true if the operation has a 0-D vector type operand or result. More...

static Value	foldExtractFromBroadcast (ExtractOp extractOp)
	Fold extractOp with scalar result coming from BroadcastOp or SplatOp. More...

static Value	foldExtractFromShapeCast (ExtractOp extractOp)

static Value	foldExtractFromExtractStrided (ExtractOp extractOp)
	Fold an ExtractOp from ExtractStridedSliceOp. More...

static Value	foldExtractStridedOpFromInsertChain (ExtractOp extractOp)
	Fold extract_op fed from a chain of insertStridedSlice ops. More...

static void	populateFromInt64AttrArray (ArrayAttr arrayAttr, SmallVectorImpl< int64_t > &results)

static llvm::SetVector< int64_t >	computeBroadcastedUnitDims (ArrayRef< int64_t > srcShape, ArrayRef< int64_t > dstShape)
	Return the dimensions of the result vector that were formerly ones in the source tensor and thus correspond to "dim-1" broadcasting. More...

static bool	isStepIndexArray (ArrayAttr idxArr, uint64_t begin, size_t width)

template<typename OpType >
static LogicalResult	isIntegerArrayAttrSmallerThanShape (OpType op, ArrayAttr arrayAttr, ArrayRef< int64_t > shape, StringRef attrName)

template<typename OpType >
static LogicalResult	isIntegerArrayAttrConfinedToRange (OpType op, ArrayAttr arrayAttr, int64_t min, int64_t max, StringRef attrName, bool halfOpen=true)

template<typename OpType >
static LogicalResult	isIntegerArrayAttrConfinedToShape (OpType op, ArrayAttr arrayAttr, ArrayRef< int64_t > shape, StringRef attrName, bool halfOpen=true, int64_t min=0)

template<typename OpType >
static LogicalResult	isSumOfIntegerArrayAttrConfinedToShape (OpType op, ArrayAttr arrayAttr1, ArrayAttr arrayAttr2, ArrayRef< int64_t > shape, StringRef attrName1, StringRef attrName2, bool halfOpen=true, int64_t min=1)

static ArrayAttr	makeI64ArrayAttr (ArrayRef< int64_t > values, MLIRContext *context)

static Type	inferStridedSliceOpResultType (VectorType vectorType, ArrayAttr offsets, ArrayAttr sizes, ArrayAttr strides)

static LogicalResult	foldExtractStridedOpFromInsertChain (ExtractStridedSliceOp op)

template<typename EmitFun >
static LogicalResult	verifyPermutationMap (AffineMap permutationMap, EmitFun emitOpError)

static LogicalResult	verifyTransferOp (VectorTransferOpInterface op, ShapedType shapedType, VectorType vectorType, VectorType maskType, VectorType inferredMaskType, AffineMap permutationMap, ArrayAttr inBounds)

static void	printTransferAttrs (OpAsmPrinter &p, VectorTransferOpInterface op)

template<typename TransferOp >
static bool	isInBounds (TransferOp op, int64_t resultIdx, int64_t indicesIdx)

template<typename TransferOp >
static LogicalResult	foldTransferInBoundsAttribute (TransferOp op)

template<typename TransferOp >
static LogicalResult	foldTransferFullMask (TransferOp op)

Macro Definition Documentation

◆ GET_ATTRDEF_CLASSES

#define GET_ATTRDEF_CLASSES

◆ GET_ATTRDEF_LIST

#define GET_ATTRDEF_LIST

◆ GET_OP_CLASSES

#define GET_OP_CLASSES

◆ GET_OP_LIST

#define GET_OP_LIST

Enumeration Type Documentation

◆ MaskFormat

enum MaskFormat

strong

Helper enum to classify mask value.

Enumerator
AllTrue
AllFalse
Unknown

Definition at line 58 of file VectorOps.cpp.

Function Documentation

◆ computeBroadcastedUnitDims()

static llvm::SetVector<int64_t> computeBroadcastedUnitDims	(	ArrayRef< int64_t >	srcShape,
		ArrayRef< int64_t >	dstShape
	)

static

Return the dimensions of the result vector that were formerly ones in the source tensor and thus correspond to "dim-1" broadcasting.

Definition at line 2103 of file VectorOps.cpp.

◆ extractVector()

template<typename IntType >

static SmallVector<IntType> extractVector ( ArrayAttr arrayAttr )

static

Definition at line 1314 of file VectorOps.cpp.

◆ foldExtractFromBroadcast()

static Value foldExtractFromBroadcast ( ExtractOp extractOp )

static

Fold extractOp with scalar result coming from BroadcastOp or SplatOp.

Definition at line 1594 of file VectorOps.cpp.

References mlir::Operation::getOperand(), mlir::Value::getType(), and hasZeroDimVectors().

◆ foldExtractFromExtractStrided()

static Value foldExtractFromExtractStrided ( ExtractOp extractOp )

static

Fold an ExtractOp from ExtractStridedSliceOp.

Definition at line 1728 of file VectorOps.cpp.

References hasZeroDimVectors().

◆ foldExtractFromShapeCast()

static Value foldExtractFromShapeCast ( ExtractOp extractOp )

static

Definition at line 1658 of file VectorOps.cpp.

References delinearize(), hasZeroDimVectors(), and mlir::linearize().

◆ foldExtractOpFromExtractChain()

static LogicalResult foldExtractOpFromExtractChain ( ExtractOp extractOp )

static

Fold the result of chains of ExtractOp in place by simply concatenating the positions.

Definition at line 1322 of file VectorOps.cpp.

References mlir::failure(), and mlir::success().

◆ foldExtractStridedOpFromInsertChain() [1/2]

static Value foldExtractStridedOpFromInsertChain ( ExtractOp extractOp )

static

Fold extract_op fed from a chain of insertStridedSlice ops.

Definition at line 1780 of file VectorOps.cpp.

References hasZeroDimVectors().

◆ foldExtractStridedOpFromInsertChain() [2/2]

static LogicalResult foldExtractStridedOpFromInsertChain ( ExtractStridedSliceOp op )

static

Definition at line 3360 of file VectorOps.cpp.

◆ foldTransferFullMask()

template<typename TransferOp >

static LogicalResult foldTransferFullMask ( TransferOp op )

static

Definition at line 4030 of file VectorOps.cpp.

◆ foldTransferInBoundsAttribute()

template<typename TransferOp >

static LogicalResult foldTransferInBoundsAttribute ( TransferOp op )

static

Definition at line 3996 of file VectorOps.cpp.

◆ getDimMap()

static std::vector<std::pair<int64_t, int64_t> > getDimMap	(	ArrayRef< AffineMap >	indexingMaps,
		ArrayAttr	iteratorTypes,
		IteratorType	targetIteratorType,
		MLIRContext *	context
	)

static

Definition at line 1041 of file VectorOps.cpp.

References mlir::detail::enumerate(), mlir::getAffineDimExpr(), and getResultIndex().

◆ getMaskFormat()

static MaskFormat getMaskFormat ( Value mask )

static

Helper method to classify a mask value.

Currently, the method looks "under the hood" of a constant value with dense attributes and a constant mask operation (since the client may be called at various stages during progressive lowering).

Definition at line 68 of file VectorOps.cpp.

References AllFalse, AllTrue, mlir::Value::getDefiningOp(), and Unknown.

◆ getResultIndex()

static int64_t getResultIndex	(	AffineMap	map,
		AffineExpr	targetExpr
	)

static

Definition at line 1033 of file VectorOps.cpp.

References mlir::AffineMap::getNumResults(), and mlir::AffineMap::getResult().

Referenced by getDimMap().

◆ hasZeroDimVectors()

static bool hasZeroDimVectors ( Operation * op )

static

Returns true if the operation has a 0-D vector type operand or result.

Definition at line 1583 of file VectorOps.cpp.

Referenced by foldExtractFromBroadcast(), foldExtractFromExtractStrided(), foldExtractFromShapeCast(), and foldExtractStridedOpFromInsertChain().

◆ incSlicePosition()

static LogicalResult incSlicePosition	(	MutableArrayRef< int64_t >	position,
		ArrayRef< int64_t >	shape,
		ArrayRef< int64_t >	offsets
	)

static

Definition at line 266 of file VectorOps.cpp.

References mlir::failure(), and mlir::success().

◆ inferStridedSliceOpResultType()

static Type inferStridedSliceOpResultType	(	VectorType	vectorType,
		ArrayAttr	offsets,
		ArrayAttr	sizes,
		ArrayAttr	strides
	)

static

Definition at line 3268 of file VectorOps.cpp.

References mlir::get().

◆ isInBounds()

template<typename TransferOp >

static bool isInBounds	(	TransferOp	op,
		int64_t	resultIdx,
		int64_t	indicesIdx
	)

static

Definition at line 3979 of file VectorOps.cpp.

◆ isIntegerArrayAttrConfinedToRange()

template<typename OpType >

static LogicalResult isIntegerArrayAttrConfinedToRange	(	OpType	op,
		ArrayAttr	arrayAttr,
		int64_t	min,
		int64_t	max,
		StringRef	attrName,
		bool	halfOpen = `true`
	)

static

Definition at line 2805 of file VectorOps.cpp.

◆ isIntegerArrayAttrConfinedToShape()

template<typename OpType >

static LogicalResult isIntegerArrayAttrConfinedToShape	(	OpType	op,
		ArrayAttr	arrayAttr,
		ArrayRef< int64_t >	shape,
		StringRef	attrName,
		bool	halfOpen = `true`,
		int64_t	min = `0`
	)

static

Definition at line 2825 of file VectorOps.cpp.

◆ isIntegerArrayAttrSmallerThanShape()

template<typename OpType >

static LogicalResult isIntegerArrayAttrSmallerThanShape	(	OpType	op,
		ArrayAttr	arrayAttr,
		ArrayRef< int64_t >	shape,
		StringRef	attrName
	)

static

Definition at line 2790 of file VectorOps.cpp.

◆ isStepIndexArray()

static bool isStepIndexArray	(	ArrayAttr	idxArr,
		uint64_t	begin,
		size_t	width
	)

static

Definition at line 2391 of file VectorOps.cpp.

◆ isSumOfIntegerArrayAttrConfinedToShape()

template<typename OpType >

static LogicalResult isSumOfIntegerArrayAttrConfinedToShape	(	OpType	op,
		ArrayAttr	arrayAttr1,
		ArrayAttr	arrayAttr2,
		ArrayRef< int64_t >	shape,
		StringRef	attrName1,
		StringRef	attrName2,
		bool	halfOpen = `true`,
		int64_t	min = `1`
	)

static

Definition at line 2848 of file VectorOps.cpp.

◆ isSupportedCombiningKind()

static bool isSupportedCombiningKind	(	CombiningKind	combiningKind,
		Type	elementType
	)

static

Definition at line 131 of file VectorOps.cpp.

References mlir::Type::isIntOrIndex(), mlir::Type::isIntOrIndexOrFloat(), and MINUI.

◆ makeI64ArrayAttr()

static ArrayAttr makeI64ArrayAttr	(	ArrayRef< int64_t >	values,
		MLIRContext *	context
	)

static

Definition at line 2869 of file VectorOps.cpp.

References mlir::get().

◆ populateFromInt64AttrArray()

static void populateFromInt64AttrArray	(	ArrayAttr	arrayAttr,
		SmallVectorImpl< int64_t > &	results
	)

static

Definition at line 2082 of file VectorOps.cpp.

◆ printTransferAttrs()

static void printTransferAttrs	(	OpAsmPrinter &	p,
		VectorTransferOpInterface	op
	)

static

Definition at line 3828 of file VectorOps.cpp.

◆ verifyDimMap()

static bool verifyDimMap	(	VectorType	lhsType,
		VectorType	rhsType,
		const std::vector< std::pair< int64_t, int64_t >> &	map
	)

static

Definition at line 822 of file VectorOps.cpp.

◆ verifyOutputShape()

static LogicalResult verifyOutputShape	(	ContractionOp	op,
		VectorType	lhsType,
		VectorType	rhsType,
		Type	accType,
		Type	resType,
		const std::vector< std::pair< int64_t, int64_t >> &	contractingDimMap,
		const std::vector< std::pair< int64_t, int64_t >> &	batchDimMap
	)

static

Definition at line 833 of file VectorOps.cpp.

◆ verifyPermutationMap()

template<typename EmitFun >

static LogicalResult verifyPermutationMap	(	AffineMap	permutationMap,
		EmitFun	emitOpError
	)

static

Definition at line 3714 of file VectorOps.cpp.

References mlir::AffineMap::getNumInputs(), mlir::AffineMap::getResults(), and mlir::success().

◆ verifyTransferOp()

static LogicalResult verifyTransferOp	(	VectorTransferOpInterface	op,
		ShapedType	shapedType,
		VectorType	vectorType,
		VectorType	maskType,
		VectorType	inferredMaskType,
		AffineMap	permutationMap,
		ArrayAttr	inBounds
	)

static

Definition at line 3743 of file VectorOps.cpp.

Classes

Namespaces

Macros

Enumerations

Functions

Macro Definition Documentation

◆ GET_ATTRDEF_CLASSES

◆ GET_ATTRDEF_LIST

◆ GET_OP_CLASSES

◆ GET_OP_LIST

Enumeration Type Documentation

◆ MaskFormat

Function Documentation

◆ computeBroadcastedUnitDims()

◆ extractVector()

◆ foldExtractFromBroadcast()

◆ foldExtractFromExtractStrided()

◆ foldExtractFromShapeCast()

◆ foldExtractOpFromExtractChain()

◆ foldExtractStridedOpFromInsertChain() [1/2]

◆ foldExtractStridedOpFromInsertChain() [2/2]

◆ foldTransferFullMask()

◆ foldTransferInBoundsAttribute()

◆ getDimMap()

◆ getMaskFormat()

◆ getResultIndex()

◆ hasZeroDimVectors()

◆ incSlicePosition()

◆ inferStridedSliceOpResultType()

◆ isInBounds()

◆ isIntegerArrayAttrConfinedToRange()

◆ isIntegerArrayAttrConfinedToShape()

◆ isIntegerArrayAttrSmallerThanShape()

◆ isStepIndexArray()

◆ isSumOfIntegerArrayAttrConfinedToShape()

◆ isSupportedCombiningKind()

◆ makeI64ArrayAttr()

◆ populateFromInt64AttrArray()

◆ printTransferAttrs()

◆ verifyDimMap()

◆ verifyOutputShape()

◆ verifyPermutationMap()

◆ verifyTransferOp()