#include "mlir/Dialect/Vector/IR/VectorOps.h"
#include "mlir/Conversion/ConvertToLLVM/ToLLVMInterface.h"
#include "mlir/Dialect/Affine/IR/ValueBoundsOpInterfaceImpl.h"
#include "mlir/Dialect/Arith/IR/Arith.h"
#include "mlir/Dialect/Arith/Utils/Utils.h"
#include "mlir/Dialect/Bufferization/IR/BufferizableOpInterface.h"
#include "mlir/Dialect/MemRef/IR/MemRef.h"
#include "mlir/Dialect/MemRef/IR/MemoryAccessOpInterfaces.h"
#include "mlir/Dialect/Tensor/IR/Tensor.h"
#include "mlir/Dialect/UB/IR/UBMatchers.h"
#include "mlir/Dialect/Utils/IndexingUtils.h"
#include "mlir/Dialect/Utils/StructuredOpsUtils.h"
#include "mlir/Dialect/Utils/VerificationUtils.h"
#include "mlir/IR/AffineExpr.h"
#include "mlir/IR/AffineMap.h"
#include "mlir/IR/Builders.h"
#include "mlir/IR/BuiltinAttributes.h"
#include "mlir/IR/BuiltinTypes.h"
#include "mlir/IR/DialectImplementation.h"
#include "mlir/IR/IRMapping.h"
#include "mlir/IR/OpImplementation.h"
#include "mlir/IR/PatternMatch.h"
#include "mlir/IR/TypeUtilities.h"
#include "mlir/IR/ValueRange.h"
#include "mlir/Interfaces/SubsetOpInterface.h"
#include "mlir/Interfaces/ValueBoundsOpInterface.h"
#include "mlir/Support/LLVM.h"
#include "mlir/Transforms/InliningUtils.h"
#include "llvm/ADT/ArrayRef.h"
#include "llvm/ADT/Repeated.h"
#include "llvm/ADT/STLExtras.h"
#include "llvm/ADT/SmallVector.h"
#include "llvm/ADT/SmallVectorExtras.h"
#include "llvm/ADT/StringSet.h"
#include "llvm/ADT/TypeSwitch.h"
#include "llvm/Support/Casting.h"
#include <cassert>
#include <cstdint>
#include <numeric>
#include "mlir/Dialect/Vector/IR/VectorDialect.cpp.inc"
#include "mlir/Dialect/Vector/IR/VectorEnums.cpp.inc"
#include "mlir/Dialect/Vector/IR/VectorAttributes.cpp.inc"
#include "mlir/Dialect/Vector/IR/VectorOps.cpp.inc"

Classes
struct	mlir::vector::detail::BitmaskEnumStorage
struct	CanonicalizeContractAdd< AddOpType >
	Return a fused vector::ContractionOp which represents a patterns such as: More...
struct	ToElementsOfBroadcast
	Canonicalize vector.to_elements(vector.broadcast(v)) where v is a vector. More...
class	FromElementsToShapeCast
	Rewrite from_elements on multiple scalar extracts as a shape_cast on a single extract. More...

Namespaces
namespace	mlir
	Include the generated interface declarations.
namespace	mlir::vector
namespace	mlir::vector::detail

Macros
#define	GET_ATTRDEF_LIST
#define	GET_OP_LIST
#define	GET_ATTRDEF_CLASSES
#define	GET_OP_CLASSES

Enumerations
enum class	MaskFormat { AllTrue = 0 , AllFalse = 1 , Unknown = 2 }
	Helper enum to classify mask value. More...

Functions
static MaskFormat	getMaskFormat (Value mask)
	Helper method to classify a mask value.
static bool	isSupportedCombiningKind (CombiningKind combiningKind, Type elementType)
static unsigned	getEffectiveVectorRankForXferOp (ShapedType shapedType, VectorType vectorType)
	Returns the effective rank of the vector to read/write for Xfer Ops.
static bool	isSplatWriteConsistentWithMaskedRead (vector::TransferWriteOp write, vector::TransferReadOp read)
	Check if write is of a constant splat and the masked read is padded with the same splat value – meaning it could be the same value as the initial constant splat.
static LogicalResult	incSlicePosition (MutableArrayRef< int64_t > position, ArrayRef< int64_t > shape, ArrayRef< int64_t > offsets)
static Attribute	convertNumericAttr (Attribute attr, Type expectedType)
	Converts numeric attributes to the expected type.
static bool	verifyDimMap (VectorType lhsType, VectorType rhsType, const std::vector< std::pair< int64_t, int64_t > > &map)
static LogicalResult	verifyOutputShape (ContractionOp op, VectorType lhsType, VectorType rhsType, Type accType, Type resType, const std::vector< std::pair< int64_t, int64_t > > &contractingDimMap, const std::vector< std::pair< int64_t, int64_t > > &batchDimMap)
static int64_t	getResultIndex (AffineMap map, AffineExpr targetExpr)
static std::vector< std::pair< int64_t, int64_t > >	getDimMap (ArrayRef< AffineMap > indexingMaps, ArrayAttr iteratorTypes, IteratorType targetIteratorType, MLIRContext *context)
static bool	isValidPositiveIndexOrPoison (int64_t index, int64_t poisonValue, int64_t maxIndex)
template<typename IntType>
static SmallVector< IntType >	extractVector (ArrayAttr arrayAttr)
static LogicalResult	foldExtractOpFromExtractChain (ExtractOp extractOp)
	Fold the result of chains of ExtractOp in place by simply concatenating the positions.
static bool	hasZeroDimVectors (Operation *op)
	Returns true if the operation has a 0-D vector type operand or result.
static bool	isBroadcastLike (Operation *op)
	All BroadcastOps, as well as ShapeCastOps that only prepend 1s, are considered to be 'broadcastlike'.
static Value	foldExtractFromBroadcast (ExtractOp extractOp)
	Fold extract(broadcast(X)) to either extract(X) or just X.
static Value	foldExtractFromShuffle (ExtractOp extractOp)
	Fold extractOp coming from ShuffleOp.
static Value	foldExtractFromShapeCast (ExtractOp extractOp)
static Value	foldExtractFromExtractStrided (ExtractOp extractOp)
	Fold an ExtractOp from ExtractStridedSliceOp.
static Value	foldExtractStridedOpFromInsertChain (ExtractOp extractOp)
	Fold extract_op fed from a chain of insertStridedSlice ops.
static Value	foldScalarExtractFromFromElements (ExtractOp extractOp)
	Try to fold the extraction of a scalar from a vector defined by vector.from_elements.
template<typename OpType, typename AdaptorType>
static Value	extractInsertFoldConstantOp (OpType op, AdaptorType adaptor, SmallVectorImpl< Value > &operands)
	If the dynamic indices of extractOp or insertOp are in fact constants, then fold it.
static Attribute	foldPoisonIndexInsertExtractOp (MLIRContext *context, ArrayRef< int64_t > staticPos, int64_t poisonVal)
	Fold an insert or extract operation into an poison value when a poison index is found at any dimension of the static position.
static Attribute	foldPoisonSrcExtractOp (Attribute srcAttr)
	Fold a vector extract from is a poison source.
static Attribute	foldDenseElementsAttrSrcExtractOp (ExtractOp extractOp, Attribute srcAttr)
	Fold a vector extract extracting from a DenseElementsAttr.
static void	populateFromInt64AttrArray (ArrayAttr arrayAttr, SmallVectorImpl< int64_t > &results)
static bool	haveSameDefiningOp (OperandRange operands, Operation *defOp)
	Returns true if all the operands are defined by defOp.
static LogicalResult	foldToElementsFromElements (ToElementsOp toElementsOp, SmallVectorImpl< OpFoldResult > &results)
	Folds vector.to_elements(vector.from_elements(e0, e1, ...)) into (e0, e1, ...).
static LogicalResult	foldToElementsOfBroadcast (ToElementsOp toElementsOp, SmallVectorImpl< OpFoldResult > &results)
	Folds vector.to_elements(vector.broadcast(x)) for the scalar case only.
static OpFoldResult	foldFromElementsToElements (FromElementsOp fromElementsOp)
	Folds vector.from_elements(vector.to_elements(vector)) into vector.
static OpFoldResult	foldFromElementsToConstant (FromElementsOp fromElementsOp, ArrayRef< Attribute > elements)
	Fold vector.from_elements to a constant when all operands are constants.
static LogicalResult	rewriteFromElementsAsBroadcast (FromElementsOp fromElementsOp, PatternRewriter &rewriter)
	Rewrite vector.from_elements as vector.broadcast if the elements are the same.
static llvm::SetVector< int64_t >	computeBroadcastedUnitDims (ArrayRef< int64_t > srcShape, ArrayRef< int64_t > dstShape)
	Return the dimensions of the result vector that were formerly ones in the source tensor and thus correspond to "dim-1" broadcasting.
static LogicalResult	foldBroadcastOfShapeCast (BroadcastOp broadcastOp)
template<typename T>
static bool	isStepIndexArray (ArrayRef< T > idxArr, uint64_t begin, size_t width)
static OpFoldResult	foldShuffleIdentityMask (ShuffleOp op)
	Fold shuffle V1, V2, [0, 1, 2, 3] : <4xi32>, <2xi32> -> V1.
static OpFoldResult	foldShufflePoisonOperandToMask (ShuffleOp op)
	If a shuffle operand is poison, replace all mask indices that reference it with kPoisonIndex.
static OpFoldResult	foldShufflePoisonInputs (MLIRContext *context, Attribute v1Attr, Attribute v2Attr)
	Fold shuffle poison, poison -> poison.
static OpFoldResult	foldShuffleConstantInputs (ShuffleOp op, Attribute v1Attr, Attribute v2Attr)
	Fold a shuffle of constant 1-D inputs by evaluating the mask.
static int64_t	calculateInsertPosition (VectorType destTy, ArrayRef< int64_t > positions)
static Attribute	foldDenseElementsAttrDestInsertOp (InsertOp insertOp, Attribute srcAttr, Attribute dstAttr, int64_t maxVectorSizeFoldThreshold)
static Value	foldInsertUseChain (InsertOp insertOp)
	Folder to replace the dest operand of the insert op with the root dest of the insert op use chain.
template<typename OpType>
static LogicalResult	isIntegerArrayAttrSmallerThanShape (OpType op, ArrayAttr arrayAttr, ArrayRef< int64_t > shape, StringRef attrName)
template<typename OpType>
static LogicalResult	isIntegerArrayAttrConfinedToRange (OpType op, ArrayAttr arrayAttr, int64_t min, int64_t max, StringRef attrName, bool halfOpen=true)
template<typename OpType>
static LogicalResult	isIntegerArrayAttrConfinedToShape (OpType op, ArrayAttr arrayAttr, ArrayRef< int64_t > shape, StringRef attrName, bool halfOpen=true, int64_t min=0)
template<typename OpType>
static LogicalResult	isSumOfIntegerArrayAttrConfinedToShape (OpType op, ArrayAttr arrayAttr1, ArrayAttr arrayAttr2, ArrayRef< int64_t > shape, StringRef attrName1, StringRef attrName2, bool halfOpen=true, int64_t min=1)
static ArrayAttr	makeI64ArrayAttr (ArrayRef< int64_t > values, MLIRContext *context)
static Type	inferStridedSliceOpResultType (VectorType vectorType, ArrayAttr offsets, ArrayAttr sizes, ArrayAttr strides)
static LogicalResult	foldExtractStridedOpFromInsertChain (ExtractStridedSliceOp op)
static OpFoldResult	foldExtractStridedSliceNonSplatConstant (ExtractStridedSliceOp op, Attribute foldInput)
template<typename EmitFun>
static LogicalResult	verifyPermutationMap (AffineMap permutationMap, EmitFun emitOpError)
static LogicalResult	verifyTransferOp (VectorTransferOpInterface op, ShapedType shapedType, VectorType vectorType, VectorType maskType, VectorType inferredMaskType, AffineMap permutationMap, ArrayAttr inBounds)
static void	printTransferAttrs (OpAsmPrinter &p, VectorTransferOpInterface op)
template<typename TransferOp>
static bool	isInBounds (TransferOp op, int64_t resultIdx, int64_t indicesIdx)
template<typename TransferOp>
static LogicalResult	foldTransferInBoundsAttribute (TransferOp op)
template<typename TransferOp>
static LogicalResult	foldTransferFullMask (TransferOp op)

Macro Definition Documentation

◆ GET_ATTRDEF_CLASSES

#define GET_ATTRDEF_CLASSES

◆ GET_ATTRDEF_LIST

#define GET_ATTRDEF_LIST

◆ GET_OP_CLASSES

#define GET_OP_CLASSES

◆ GET_OP_LIST

#define GET_OP_LIST

Enumeration Type Documentation

◆ MaskFormat

enum class MaskFormat

strong

Helper enum to classify mask value.

Enumerator
AllTrue
AllFalse
Unknown

Definition at line 64 of file VectorOps.cpp.

Function Documentation

◆ calculateInsertPosition()

int64_t calculateInsertPosition	(	VectorType	destTy,
		ArrayRef< int64_t >	positions )

static

Definition at line 3754 of file VectorOps.cpp.

References mlir::computeStrides(), copy(), and mlir::linearize().

Referenced by foldDenseElementsAttrDestInsertOp().

◆ computeBroadcastedUnitDims()

llvm::SetVector< int64_t > computeBroadcastedUnitDims	(	ArrayRef< int64_t >	srcShape,
		ArrayRef< int64_t >	dstShape )

static

Return the dimensions of the result vector that were formerly ones in the source tensor and thus correspond to "dim-1" broadcasting.

Definition at line 2971 of file VectorOps.cpp.

◆ convertNumericAttr()

Attribute convertNumericAttr	(	Attribute	attr,
		Type	expectedType )

static

Converts numeric attributes to the expected type.

Supports integer-to-integer and float-to-integer conversions. Returns the original attribute if no conversion is needed or supported.

Definition at line 407 of file VectorOps.cpp.

Referenced by foldDenseElementsAttrDestInsertOp(), and foldFromElementsToConstant().

◆ extractInsertFoldConstantOp()

template<typename OpType, typename AdaptorType>

Value extractInsertFoldConstantOp	(	OpType	op,
		AdaptorType	adaptor,
		SmallVectorImpl< Value > &	operands )

static

If the dynamic indices of extractOp or insertOp are in fact constants, then fold it.

Definition at line 2059 of file VectorOps.cpp.

References vectorShape().

◆ extractVector()

template<typename IntType>

SmallVector< IntType > extractVector ( ArrayAttr arrayAttr )

static

Definition at line 1408 of file VectorOps.cpp.

References ArrayAttr().

Referenced by foldExtractFromExtractStrided(), and foldExtractStridedOpFromInsertChain().

◆ foldBroadcastOfShapeCast()

LogicalResult foldBroadcastOfShapeCast ( BroadcastOp broadcastOp )

static

Definition at line 3183 of file VectorOps.cpp.

References mlir::vector::isBroadcastableTo(), mlir::vector::Success, and success().

◆ foldDenseElementsAttrDestInsertOp()

Attribute foldDenseElementsAttrDestInsertOp	(	InsertOp	insertOp,
		Attribute	srcAttr,
		Attribute	dstAttr,
		int64_t	maxVectorSizeFoldThreshold )

static

Converts attribute to the expected type if there's a mismatch.

Definition at line 3929 of file VectorOps.cpp.

References calculateInsertPosition(), convertNumericAttr(), copy(), and mlir::DenseElementsAttr::get().

◆ foldDenseElementsAttrSrcExtractOp()

Attribute foldDenseElementsAttrSrcExtractOp	(	ExtractOp	extractOp,
		Attribute	srcAttr )

static

Fold a vector extract extracting from a DenseElementsAttr.

Definition at line 2126 of file VectorOps.cpp.

References mlir::computeStrides(), copy(), mlir::DenseElementsAttr::get(), and mlir::linearize().

◆ foldExtractFromBroadcast()

Value foldExtractFromBroadcast ( ExtractOp extractOp )

static

Fold extract(broadcast(X)) to either extract(X) or just X.

Example:

   broadcast             extract [1][2]

(3, 4) --------> (2, 3, 4) ----------------> (4)

becomes extract [1] (3,4) -------------------------------------> (4)

The variable names used in this implementation correspond to the above shapes as,

(3, 4) is input shape.
(2, 3, 4) is broadcast shape.
(4) is extract shape.

This folding is possible when the suffix of input shape is the same as extract shape.

Definition at line 1729 of file VectorOps.cpp.

References mlir::decomposeMixedValues(), mlir::Builder::getIndexAttr(), mlir::Operation::getOperand(), mlir::Value::getType(), isBroadcastLike(), and ValueRange.

◆ foldExtractFromExtractStrided()

Value foldExtractFromExtractStrided ( ExtractOp extractOp )

static

Fold an ExtractOp from ExtractStridedSliceOp.

Definition at line 1891 of file VectorOps.cpp.

References b, extractVector(), and hasZeroDimVectors().

◆ foldExtractFromShapeCast()

Value foldExtractFromShapeCast ( ExtractOp extractOp )

static

Definition at line 1826 of file VectorOps.cpp.

References b, mlir::delinearize(), and mlir::linearize().

◆ foldExtractFromShuffle()

Value foldExtractFromShuffle ( ExtractOp extractOp )

static

Fold extractOp coming from ShuffleOp.

Example:

shuffle = vector.shuffle a, b [0, 8, 7, 15] : vector<8xf32>, vector<8xf32> extract = vector.extract shuffle[3] : f32 from vector<4xf32> -> extract = vector.extract b[7] : f32 from vector<8xf32>

Definition at line 1794 of file VectorOps.cpp.

◆ foldExtractOpFromExtractChain()

LogicalResult foldExtractOpFromExtractChain ( ExtractOp extractOp )

static

Fold the result of chains of ExtractOp in place by simply concatenating the positions.

Definition at line 1416 of file VectorOps.cpp.

References b, and success().

◆ foldExtractStridedOpFromInsertChain() [1/2]

Value foldExtractStridedOpFromInsertChain ( ExtractOp extractOp )

static

Fold extract_op fed from a chain of insertStridedSlice ops.

Definition at line 1943 of file VectorOps.cpp.

References b, extractVector(), and hasZeroDimVectors().

◆ foldExtractStridedOpFromInsertChain() [2/2]

LogicalResult foldExtractStridedOpFromInsertChain ( ExtractStridedSliceOp op )

static

Definition at line 4559 of file VectorOps.cpp.

References ArrayAttr(), b, extractStrides(), and success().

◆ foldExtractStridedSliceNonSplatConstant()

OpFoldResult foldExtractStridedSliceNonSplatConstant	(	ExtractStridedSliceOp	op,
		Attribute	foldInput )

static

Definition at line 4623 of file VectorOps.cpp.

References mlir::computeStrides(), copy(), mlir::DenseElementsAttr::get(), mlir::getI64SubArray(), incSlicePosition(), and mlir::linearize().

◆ foldFromElementsToConstant()

OpFoldResult foldFromElementsToConstant	(	FromElementsOp	fromElementsOp,
		ArrayRef< Attribute >	elements )

static

Fold vector.from_elements to a constant when all operands are constants.

Example: c1 = arith.constant 1 : i32 c2 = arith.constant 2 : i32 v = vector.from_elements c1, c2 : vector<2xi32> => v = arith.constant dense<[1, 2]> : vector<2xi32>

Definition at line 2783 of file VectorOps.cpp.

References convertNumericAttr(), mlir::DenseElementsAttr::get(), mlir::ub::m_Poison(), and mlir::matchPattern().

◆ foldFromElementsToElements()

OpFoldResult foldFromElementsToElements ( FromElementsOp fromElementsOp )

static

Folds vector.from_elements(vector.to_elements(vector)) into vector.

Case #1: Input and output vectors are the same.

%0:3 = vector.to_elements a : vector<3xf32> %1 = vector.from_elements %0#0, %0#1, %0#2 : vector<3xf32> user_op %1

becomes:

user_op a

Definition at line 2750 of file VectorOps.cpp.

References mlir::Value::getType(), and haveSameDefiningOp().

◆ foldInsertUseChain()

Value foldInsertUseChain ( InsertOp insertOp )

static

Folder to replace the dest operand of the insert op with the root dest of the insert op use chain.

Definition at line 3981 of file VectorOps.cpp.

References mlir::Value::getDefiningOp(), and mlir::Operation::setOperand().

◆ foldPoisonIndexInsertExtractOp()

Attribute foldPoisonIndexInsertExtractOp	(	MLIRContext *	context,
		ArrayRef< int64_t >	staticPos,
		int64_t	poisonVal )

static

Fold an insert or extract operation into an poison value when a poison index is found at any dimension of the static position.

Definition at line 2108 of file VectorOps.cpp.

◆ foldPoisonSrcExtractOp()

Attribute foldPoisonSrcExtractOp ( Attribute srcAttr )

static

Fold a vector extract from is a poison source.

Definition at line 2118 of file VectorOps.cpp.

References mlir::ub::m_Poison(), and mlir::matchPattern().

◆ foldScalarExtractFromFromElements()

Value foldScalarExtractFromFromElements ( ExtractOp extractOp )

static

Try to fold the extraction of a scalar from a vector defined by vector.from_elements.

E.g.:

%0 = vector.from_elements a, b : vector<2xf32> %1 = vector.extract %0[0] : f32 from vector<2xf32> ==> fold to a

Definition at line 2023 of file VectorOps.cpp.

References mlir::Value::getDefiningOp(), and indices.

◆ foldShuffleConstantInputs()

OpFoldResult foldShuffleConstantInputs	(	ShuffleOp	op,
		Attribute	v1Attr,
		Attribute	v2Attr )

static

Fold a shuffle of constant 1-D inputs by evaluating the mask.

Definition at line 3420 of file VectorOps.cpp.

References mlir::DenseElementsAttr::get(), mlir::ub::m_Poison(), and mlir::matchPattern().

◆ foldShuffleIdentityMask()

OpFoldResult foldShuffleIdentityMask ( ShuffleOp op )

static

Fold shuffle V1, V2, [0, 1, 2, 3] : <4xi32>, <2xi32> -> V1.

Fold shuffle V1, V2, [4, 5] : <4xi32>, <2xi32> -> V2.

Definition at line 3371 of file VectorOps.cpp.

References isStepIndexArray().

◆ foldShufflePoisonInputs()

OpFoldResult foldShufflePoisonInputs	(	MLIRContext *	context,
		Attribute	v1Attr,
		Attribute	v2Attr )

static

Fold shuffle poison, poison -> poison.

Definition at line 3410 of file VectorOps.cpp.

References mlir::ub::m_Poison(), and mlir::matchPattern().

◆ foldShufflePoisonOperandToMask()

OpFoldResult foldShufflePoisonOperandToMask ( ShuffleOp op )

static

If a shuffle operand is poison, replace all mask indices that reference it with kPoisonIndex.

This is an in-place fold.

Definition at line 3384 of file VectorOps.cpp.

References mlir::ub::m_Poison(), and mlir::matchPattern().

◆ foldToElementsFromElements()

LogicalResult foldToElementsFromElements	(	ToElementsOp	toElementsOp,
		SmallVectorImpl< OpFoldResult > &	results )

static

Folds vector.to_elements(vector.from_elements(e0, e1, ...)) into (e0, e1, ...).

For example:

%0 = vector.from_elements a, b, c : vector<3xf32> %1:3 = vector.to_elements %0 : vector<3xf32> user_op %1#0, %1#1, %1#2

becomes:

user_op a, b, c

Definition at line 2585 of file VectorOps.cpp.

References success().

◆ foldToElementsOfBroadcast()

LogicalResult foldToElementsOfBroadcast	(	ToElementsOp	toElementsOp,
		SmallVectorImpl< OpFoldResult > &	results )

static

Folds vector.to_elements(vector.broadcast(x)) for the scalar case only.

Example: b = vector.broadcast x : i32 to vector<3xf32> e:3 = vector.to_elements b : vector<3xf32> user_op e#0, e#1, e#2 becomes: user_op x, x, x

The vector source case is handled by a canonicalization pattern.

Definition at line 2607 of file VectorOps.cpp.

References success().

◆ foldTransferFullMask()

template<typename TransferOp>

LogicalResult foldTransferFullMask ( TransferOp op )

static

Definition at line 5444 of file VectorOps.cpp.

References AllTrue, getMaskFormat(), and success().

◆ foldTransferInBoundsAttribute()

template<typename TransferOp>

LogicalResult foldTransferInBoundsAttribute ( TransferOp op )

static

Definition at line 5389 of file VectorOps.cpp.

References b, mlir::AffineMap::getBroadcastDims(), mlir::AffineMap::getPermutationMap(), mlir::AffineMap::getResult(), isInBounds(), and success().

◆ getDimMap()

std::vector< std::pair< int64_t, int64_t > > getDimMap	(	ArrayRef< AffineMap >	indexingMaps,
		ArrayAttr	iteratorTypes,
		IteratorType	targetIteratorType,
		MLIRContext *	context )

static

Definition at line 1177 of file VectorOps.cpp.

References ArrayAttr(), mlir::getAffineDimExpr(), and getResultIndex().

◆ getEffectiveVectorRankForXferOp()

unsigned getEffectiveVectorRankForXferOp	(	ShapedType	shapedType,
		VectorType	vectorType )

static

Returns the effective rank of the vector to read/write for Xfer Ops.

When the element type of the shaped type is a scalar, this will simply return the rank of the vector ( the result for xfer_read or the value to store for xfer_write).

When the element type of the base shaped type is a vector, returns the difference between the original vector type and the element type of the shaped type.

EXAMPLE 1 (element type is a scalar):

shapedType = tensor<10x20xf32>, vectorType = vector<2x4xf32>
- shapedType.getElementType() = f32 (rank 0)
- vectorType.getRank() = 2
- Result = 2 - 0 = 2

EXAMPLE 2 (element type is a vector):

shapedType = tensor<10xvector<20xf32>>, vectorType = vector<20xf32>
- shapedType.getElementType() = vector<20xf32> (rank 1)
- vectorType.getRank() = 1
- Result = 1 - 1 = 0

This is used to determine the number of minor dimensions for identity maps in vector transfer Ops.

Definition at line 183 of file VectorOps.cpp.

Referenced by mlir::vector::getTransferMinorIdentityMap().

◆ getMaskFormat()

MaskFormat getMaskFormat ( Value mask )

static

Helper method to classify a mask value.

Currently, the method looks "under the hood" of a constant value with dense attributes and a constant mask operation (since the client may be called at various stages during progressive lowering).

Definition at line 74 of file VectorOps.cpp.

References AllFalse, AllTrue, b, mlir::Value::getDefiningOp(), and Unknown.

Referenced by foldTransferFullMask().

◆ getResultIndex()

int64_t getResultIndex	(	AffineMap	map,
		AffineExpr	targetExpr )

static

Definition at line 1169 of file VectorOps.cpp.

References mlir::AffineMap::getNumResults(), and mlir::AffineMap::getResult().

Referenced by getDimMap().

◆ hasZeroDimVectors()

bool hasZeroDimVectors ( Operation * op )

static

Returns true if the operation has a 0-D vector type operand or result.

Definition at line 1677 of file VectorOps.cpp.

References mlir::Operation::getOperandTypes(), and mlir::Operation::getResultTypes().

Referenced by foldExtractFromExtractStrided(), and foldExtractStridedOpFromInsertChain().

◆ haveSameDefiningOp()

bool haveSameDefiningOp	(	OperandRange	operands,
		Operation *	defOp )

static

Returns true if all the operands are defined by defOp.

Otherwise, returns false.

Definition at line 2563 of file VectorOps.cpp.

References mlir::Value::getDefiningOp().

Referenced by foldFromElementsToElements().

◆ incSlicePosition()

LogicalResult incSlicePosition	(	MutableArrayRef< int64_t >	position,
		ArrayRef< int64_t >	shape,
		ArrayRef< int64_t >	offsets )

static

Definition at line 332 of file VectorOps.cpp.

References success().

Referenced by foldExtractStridedSliceNonSplatConstant().

◆ inferStridedSliceOpResultType()

Type inferStridedSliceOpResultType	(	VectorType	vectorType,
		ArrayAttr	offsets,
		ArrayAttr	sizes,
		ArrayAttr	strides )

static

Definition at line 4467 of file VectorOps.cpp.

References ArrayAttr().

◆ isBroadcastLike()

bool isBroadcastLike ( Operation * op )

static

All BroadcastOps, as well as ShapeCastOps that only prepend 1s, are considered to be 'broadcastlike'.

Definition at line 1689 of file VectorOps.cpp.

Referenced by foldExtractFromBroadcast().

◆ isInBounds()

template<typename TransferOp>

bool isInBounds	(	TransferOp	op,
		int64_t	resultIdx,
		int64_t	indicesIdx )

static

Definition at line 5372 of file VectorOps.cpp.

References mlir::getConstantIntValue().

Referenced by foldTransferInBoundsAttribute().

◆ isIntegerArrayAttrConfinedToRange()

template<typename OpType>

LogicalResult isIntegerArrayAttrConfinedToRange	(	OpType	op,
		ArrayAttr	arrayAttr,
		int64_t	min,
		int64_t	max,
		StringRef	attrName,
		bool	halfOpen = true )

static

Definition at line 4063 of file VectorOps.cpp.

References ArrayAttr(), max(), min(), and success().

◆ isIntegerArrayAttrConfinedToShape()

template<typename OpType>

LogicalResult isIntegerArrayAttrConfinedToShape	(	OpType	op,
		ArrayAttr	arrayAttr,
		ArrayRef< int64_t >	shape,
		StringRef	attrName,
		bool	halfOpen = true,
		int64_t	min = 0 )

static

Definition at line 4083 of file VectorOps.cpp.

References ArrayAttr(), max(), min(), and success().

◆ isIntegerArrayAttrSmallerThanShape()

template<typename OpType>

LogicalResult isIntegerArrayAttrSmallerThanShape	(	OpType	op,
		ArrayAttr	arrayAttr,
		ArrayRef< int64_t >	shape,
		StringRef	attrName )

static

Definition at line 4048 of file VectorOps.cpp.

References ArrayAttr(), and success().

◆ isSplatWriteConsistentWithMaskedRead()

bool isSplatWriteConsistentWithMaskedRead	(	vector::TransferWriteOp	write,
		vector::TransferReadOp	read )

static

Check if write is of a constant splat and the masked read is padded with the same splat value – meaning it could be the same value as the initial constant splat.

Definition at line 211 of file VectorOps.cpp.

References mlir::DenseElementsAttr::getSplatValue(), mlir::DenseElementsAttr::isSplat(), mlir::m_Constant(), and mlir::matchPattern().

◆ isStepIndexArray()

template<typename T>

bool isStepIndexArray	(	ArrayRef< T >	idxArr,
		uint64_t	begin,
		size_t	width )

static

Definition at line 3362 of file VectorOps.cpp.

Referenced by foldShuffleIdentityMask().

◆ isSumOfIntegerArrayAttrConfinedToShape()

template<typename OpType>

LogicalResult isSumOfIntegerArrayAttrConfinedToShape	(	OpType	op,
		ArrayAttr	arrayAttr1,
		ArrayAttr	arrayAttr2,
		ArrayRef< int64_t >	shape,
		StringRef	attrName1,
		StringRef	attrName2,
		bool	halfOpen = true,
		int64_t	min = 1 )

static

Definition at line 4106 of file VectorOps.cpp.

References ArrayAttr(), max(), min(), and success().

◆ isSupportedCombiningKind()

bool isSupportedCombiningKind	(	CombiningKind	combiningKind,
		Type	elementType )

static

Definition at line 136 of file VectorOps.cpp.

References mlir::Type::isIntOrIndex(), and mlir::Type::isIntOrIndexOrFloat().

◆ isValidPositiveIndexOrPoison()

bool isValidPositiveIndexOrPoison	(	int64_t	index,
		int64_t	poisonValue,
		int64_t	maxIndex )

static

Definition at line 1311 of file VectorOps.cpp.

◆ makeI64ArrayAttr()

ArrayAttr makeI64ArrayAttr	(	ArrayRef< int64_t >	values,
		MLIRContext *	context )

static

Definition at line 4127 of file VectorOps.cpp.

References ArrayAttr().

◆ populateFromInt64AttrArray()

void populateFromInt64AttrArray	(	ArrayAttr	arrayAttr,
		SmallVectorImpl< int64_t > &	results )

static

Definition at line 2543 of file VectorOps.cpp.

References ArrayAttr().

◆ printTransferAttrs()

void printTransferAttrs	(	OpAsmPrinter &	p,
		VectorTransferOpInterface	op )

static

Definition at line 5198 of file VectorOps.cpp.

References b, and mlir::OpAsmPrinter::printOptionalAttrDict().

◆ rewriteFromElementsAsBroadcast()

LogicalResult rewriteFromElementsAsBroadcast	(	FromElementsOp	fromElementsOp,
		PatternRewriter &	rewriter )

static

Rewrite vector.from_elements as vector.broadcast if the elements are the same.

Example: %0 = vector.from_elements a, a, a : vector<3xf32> => %0 = vector.broadcast a : f32 to vector<3xf32>

Definition at line 2821 of file VectorOps.cpp.

References mlir::RewriterBase::replaceOpWithNewOp(), and success().

◆ verifyDimMap()

bool verifyDimMap	(	VectorType	lhsType,
		VectorType	rhsType,
		const std::vector< std::pair< int64_t, int64_t > > &	map )

static

Definition at line 952 of file VectorOps.cpp.

◆ verifyOutputShape()

LogicalResult verifyOutputShape	(	ContractionOp	op,
		VectorType	lhsType,
		VectorType	rhsType,
		Type	accType,
		Type	resType,
		const std::vector< std::pair< int64_t, int64_t > > &	contractingDimMap,
		const std::vector< std::pair< int64_t, int64_t > > &	batchDimMap )

static

Definition at line 963 of file VectorOps.cpp.

References mlir::AffineMap::compose(), mlir::AffineMap::get(), mlir::getAffineConstantExpr(), mlir::AffineMap::getNumInputs(), mlir::AffineMap::getResults(), mlir::getUnusedDimsBitVector(), mlir::simplifyAffineMap(), and success().

◆ verifyPermutationMap()

template<typename EmitFun>

LogicalResult verifyPermutationMap	(	AffineMap	permutationMap,
		EmitFun	emitOpError )

static

Definition at line 5090 of file VectorOps.cpp.

References emitOpError(), mlir::AffineMap::getNumInputs(), mlir::AffineMap::getResults(), and success().

◆ verifyTransferOp()

LogicalResult verifyTransferOp	(	VectorTransferOpInterface	op,
		ShapedType	shapedType,
		VectorType	vectorType,
		VectorType	maskType,
		VectorType	inferredMaskType,
		AffineMap	permutationMap,
		ArrayAttr	inBounds )

static

Definition at line 5119 of file VectorOps.cpp.

References ArrayAttr(), mlir::DataLayout::closest(), mlir::AffineMap::getNumInputs(), mlir::AffineMap::getNumResults(), mlir::AffineMap::getNumSymbols(), mlir::DataLayout::getTypeSizeInBits(), and success().

Classes

Namespaces

Macros

Enumerations

Functions

Macro Definition Documentation

◆ GET_ATTRDEF_CLASSES

◆ GET_ATTRDEF_LIST

◆ GET_OP_CLASSES

◆ GET_OP_LIST

Enumeration Type Documentation

◆ MaskFormat

Function Documentation

◆ calculateInsertPosition()

◆ computeBroadcastedUnitDims()

◆ convertNumericAttr()

◆ extractInsertFoldConstantOp()

◆ extractVector()

◆ foldBroadcastOfShapeCast()

◆ foldDenseElementsAttrDestInsertOp()

◆ foldDenseElementsAttrSrcExtractOp()

◆ foldExtractFromBroadcast()

◆ foldExtractFromExtractStrided()

◆ foldExtractFromShapeCast()

◆ foldExtractFromShuffle()

◆ foldExtractOpFromExtractChain()

◆ foldExtractStridedOpFromInsertChain() [1/2]

◆ foldExtractStridedOpFromInsertChain() [2/2]

◆ foldExtractStridedSliceNonSplatConstant()

◆ foldFromElementsToConstant()

◆ foldFromElementsToElements()

◆ foldInsertUseChain()

◆ foldPoisonIndexInsertExtractOp()

◆ foldPoisonSrcExtractOp()

◆ foldScalarExtractFromFromElements()

◆ foldShuffleConstantInputs()

◆ foldShuffleIdentityMask()

◆ foldShufflePoisonInputs()

◆ foldShufflePoisonOperandToMask()

◆ foldToElementsFromElements()

◆ foldToElementsOfBroadcast()

◆ foldTransferFullMask()

◆ foldTransferInBoundsAttribute()

◆ getDimMap()

◆ getEffectiveVectorRankForXferOp()

◆ getMaskFormat()

◆ getResultIndex()

◆ hasZeroDimVectors()

◆ haveSameDefiningOp()

◆ incSlicePosition()

◆ inferStridedSliceOpResultType()

◆ isBroadcastLike()

◆ isInBounds()

◆ isIntegerArrayAttrConfinedToRange()

◆ isIntegerArrayAttrConfinedToShape()

◆ isIntegerArrayAttrSmallerThanShape()

◆ isSplatWriteConsistentWithMaskedRead()

◆ isStepIndexArray()

◆ isSumOfIntegerArrayAttrConfinedToShape()

◆ isSupportedCombiningKind()

◆ isValidPositiveIndexOrPoison()

◆ makeI64ArrayAttr()

◆ populateFromInt64AttrArray()

◆ printTransferAttrs()

◆ rewriteFromElementsAsBroadcast()

◆ verifyDimMap()

◆ verifyOutputShape()

◆ verifyPermutationMap()

◆ verifyTransferOp()