Namespaces
	detail

Classes
struct	LowerVectorsOptions
	Helper structure used to hold the different options of LowerVectorsOp. More...

struct	WarpExecuteOnLane0LoweringOptions

struct	UnrollVectorOptions
	Options that control the vector unrolling. More...

struct	VectorTransformsOptions
	Structure to control the behavior of vector transform patterns. More...

Typedefs
using	DistributionMapFn = std::function< AffineMap(Value)>

Enumerations
enum class	BroadcastableToResult { Success = 0 , SourceRankHigher = 1 , DimensionMismatch = 2 , SourceTypeNotAVector = 3 }
	Return whether `srcType` can be broadcast to `dstVectorType` under the semantics of the `vector.broadcast` op. More...

Functions
void	buildTerminatedBody (OpBuilder &builder, Location loc)
	Default callback to build a region with a 'vector.yield' terminator with no arguments. More...

BroadcastableToResult	isBroadcastableTo (Type srcType, VectorType dstVectorType, std::pair< int, int > *mismatchingDims=nullptr)

void	populateVectorToVectorCanonicalizationPatterns (RewritePatternSet &patterns, PatternBenefit benefit=1)
	Collect a set of vector-to-vector canonicalization patterns. More...

void	populateFoldArithExtensionPatterns (RewritePatternSet &patterns)
	Collect a set of patterns that fold arithmetic extension on floating point into vector contract for the backends with native support. More...

IntegerType	getVectorSubscriptType (Builder &builder)
	Returns the integer type required for subscripts in the vector dialect. More...

ArrayAttr	getVectorSubscriptAttr (Builder &b, ArrayRef< int64_t > values)
	Returns an integer array attribute containing the given values using the integer type required for subscripts in the vector dialect. More...

Value	getVectorReductionOp (arith::AtomicRMWKind op, OpBuilder &builder, Location loc, Value vector)
	Returns the value obtained by reducing the vector into a scalar using the operation kind associated with a binary AtomicRMWKind op. More...

AffineMap	getTransferMinorIdentityMap (ShapedType shapedType, VectorType vectorType)
	Build the default minor identity map suitable for a vector transfer. More...

bool	checkSameValueRAW (TransferWriteOp defWrite, TransferReadOp read)
	Return true if the transfer_write fully writes the data accessed by the transfer_read. More...

bool	checkSameValueWAW (TransferWriteOp write, TransferWriteOp priorWrite)
	Return true if the write op fully over-write the priorWrite transfer_write op. More...

bool	isDisjointTransferIndices (VectorTransferOpInterface transferA, VectorTransferOpInterface transferB)
	Same behavior as `isDisjointTransferSet` but doesn't require the operations to have the same tensor/memref. More...

bool	isDisjointTransferSet (VectorTransferOpInterface transferA, VectorTransferOpInterface transferB)
	Return true if we can prove that the transfer operations access disjoint memory. More...

Value	makeArithReduction (OpBuilder &b, Location loc, CombiningKind kind, Value v1, Value acc, Value mask=Value())
	Return the result value of reducing two scalar/vector values with the corresponding arith operation. More...

bool	isParallelIterator (Attribute attr)
	Returns true if `attr` has "parallel" iterator type semantics. More...

bool	isReductionIterator (Attribute attr)
	Returns true if `attr` has "reduction" iterator type semantics. More...

SmallVector< int64_t >	getAsIntegers (ArrayRef< Value > values)
	Returns the integer numbers in `values`. More...

SmallVector< int64_t >	getAsIntegers (ArrayRef< OpFoldResult > foldResults)
	Returns the integer numbers in `foldResults`. More...

SmallVector< Value >	getAsValues (OpBuilder &builder, Location loc, ArrayRef< OpFoldResult > foldResults)
	Convert `foldResults` into Values. More...

SmallVector< arith::ConstantIndexOp >	getAsConstantIndexOps (ArrayRef< Value > values)
	Returns the constant index ops in `values`. More...

void	createMaskOpRegion (OpBuilder &builder, Operation *maskableOp)
	Create the vector.yield-ended region of a vector.mask op with `maskableOp` as masked operation. More...

Operation *	maskOperation (OpBuilder &builder, Operation *maskableOp, Value mask, Value passthru=Value())
	Creates a vector.mask operation around a maskable operation. More...

Value	selectPassthru (OpBuilder &builder, Value mask, Value newValue, Value passthru)
	Creates a vector select operation that picks values from `newValue` or `passthru` for each result vector lane based on `mask`. More...

void	registerTransformDialectExtension (DialectRegistry &registry)

void	registerBufferizableOpInterfaceExternalModels (DialectRegistry &registry)

void	populateVectorContractLoweringPatterns (RewritePatternSet &patterns, VectorTransformsOptions options, PatternBenefit benefit=1, bool disableOuterProductLowering=false)
	Populate the pattern set with the following patterns: More...

void	populateVectorOuterProductLoweringPatterns (RewritePatternSet &patterns, PatternBenefit benefit=1)
	Populate the pattern set with the following patterns: More...

void	populateVectorMultiReductionLoweringPatterns (RewritePatternSet &patterns, VectorMultiReductionLowering options, PatternBenefit benefit=1)
	Collect a set of patterns to convert vector.multi_reduction op into a sequence of vector.reduction ops. More...

void	populateVectorBroadcastLoweringPatterns (RewritePatternSet &patterns, PatternBenefit benefit=1)
	Populate the pattern set with the following patterns: More...

void	populateVectorMaskOpLoweringPatterns (RewritePatternSet &patterns, PatternBenefit benefit=1)
	Populate the pattern set with the following patterns: More...

void	populateScalarVectorTransferLoweringPatterns (RewritePatternSet &patterns, PatternBenefit benefit, bool allowMultipleUses)
	Collects patterns that lower scalar vector transfer ops to memref loads and stores when beneficial. More...

void	populateVectorShapeCastLoweringPatterns (RewritePatternSet &patterns, PatternBenefit benefit=1)
	Populate the pattern set with the following patterns: More...

void	populateVectorTransposeLoweringPatterns (RewritePatternSet &patterns, VectorTransformsOptions options, PatternBenefit benefit=1)
	Populate the pattern set with the following patterns: More...

void	populateVectorTransferLoweringPatterns (RewritePatternSet &patterns, std::optional< unsigned > maxTransferRank=std::nullopt, PatternBenefit benefit=1)
	Populate the pattern set with the following patterns: More...

void	populateVectorTransferPermutationMapLoweringPatterns (RewritePatternSet &patterns, PatternBenefit benefit=1)
	Collect a set of transfer read/write lowering patterns that simplify the permutation map (e.g., converting it to a minor identity map) by inserting broadcasts and transposes. More...

void	populateVectorScanLoweringPatterns (RewritePatternSet &patterns, PatternBenefit benefit=1)
	Populate the pattern set with the following patterns: More...

void	populateVectorGatherLoweringPatterns (RewritePatternSet &patterns, PatternBenefit benefit=1)
	Populate the pattern set with the following patterns: More...

void	populateVectorMaskLoweringPatternsForSideEffectingOps (RewritePatternSet &patterns)
	Populates instances of `MaskOpRewritePattern` to lower masked operations with `vector.mask`. More...

std::unique_ptr< Pass >	createVectorBufferizePass ()
	Creates an instance of the `vector` dialect bufferization pass. More...

std::unique_ptr< Pass >	createLowerVectorMaskPass ()
	Creates an instance of the `vector.mask` lowering pass. More...

void	populateWarpExecuteOnLane0OpToScfForPattern (RewritePatternSet &patterns, const WarpExecuteOnLane0LoweringOptions &options, PatternBenefit benefit=1)

void	populateVectorContractCanonicalizeMatmulToMMT (RewritePatternSet &patterns, std::function< LogicalResult(vector::ContractionOp)> constraint=[](vector::ContractionOp) { return success();}, PatternBenefit=1)
	Canonicalization of a `vector.contraction a, b, c` with row-major matmul semantics to a contraction with MMT semantics (matrix matrix multiplication with the RHS transposed). More...

void	populateVectorReductionToContractPatterns (RewritePatternSet &patterns, PatternBenefit benefit=1)
	Collect patterns to convert reduction op to vector.contract and fold transpose/broadcast ops into the contract. More...

void	populateVectorTransferFullPartialPatterns (RewritePatternSet &patterns, const VectorTransformsOptions &options)
	Populate `patterns` with the following patterns. More...

void	populateVectorTransferCollapseInnerMostContiguousDimsPatterns (RewritePatternSet &patterns, PatternBenefit benefit=1)
	Collect a set of patterns to reduce the rank of the operands of vector transfer ops to operate on the largest contigious vector. More...

void	populateSinkVectorBroadcastPatterns (RewritePatternSet &patterns, PatternBenefit benefit=1)
	Patterns that remove redundant vector broadcasts. More...

void	populateVectorInsertExtractStridedSliceDecompositionPatterns (RewritePatternSet &patterns, PatternBenefit benefit=1)
	Populate `patterns` with the following patterns. More...

void	populateVectorExtractStridedSliceToExtractInsertChainPatterns (RewritePatternSet &patterns, std::function< bool(ExtractStridedSliceOp)> controlFn=nullptr, PatternBenefit benefit=1)
	Populate `patterns` with a pattern to breaks down 1-D extract_strided_slice ops into a chain of Extract ops to extract each element from the source, and then a chain of Insert ops to insert to the target vector. More...

void	populateBreakDownVectorBitCastOpPatterns (RewritePatternSet &patterns, std::function< bool(BitCastOp)> controlFn=nullptr, PatternBenefit benefit=1)
	Populate `patterns` with a pattern to break down 1-D vector.bitcast ops based on the destination vector shape. More...

void	populateVectorInsertExtractStridedSliceTransforms (RewritePatternSet &patterns, PatternBenefit benefit=1)
	Populate `patterns` with the following patterns. More...

void	populateVectorUnrollPatterns (RewritePatternSet &patterns, const UnrollVectorOptions &options, PatternBenefit benefit=1)
	Collect a set of pattern to unroll vector operations to a smaller shapes. More...

void	populateShapeCastFoldingPatterns (RewritePatternSet &patterns, PatternBenefit benefit=1)
	Collect a set of vector.shape_cast folding patterns. More...

void	populateCastAwayVectorLeadingOneDimPatterns (RewritePatternSet &patterns, PatternBenefit benefit=1)
	Collect a set of leading one dimension removal patterns. More...

void	populateVectorTransferDropUnitDimsPatterns (RewritePatternSet &patterns, PatternBenefit benefit=1)
	Collect a set of one dimension removal patterns. More...

void	populateFlattenVectorTransferPatterns (RewritePatternSet &patterns, PatternBenefit benefit=1)
	Collect a set of patterns to flatten n-D vector transfers on contiguous memref. More...

void	populateBubbleVectorBitCastOpPatterns (RewritePatternSet &patterns, PatternBenefit benefit=1)
	Collect a set of patterns that bubble up/down bitcast ops. More...

void	populateVectorMaskMaterializationPatterns (RewritePatternSet &patterns, bool force32BitVectorIndices, PatternBenefit benefit=1)
	These patterns materialize masks for various vector ops such as transfers. More...

void	populateVectorNarrowTypeEmulationPatterns (arith::NarrowTypeEmulationConverter &typeConverter, RewritePatternSet &patterns)
	Appends patterns for emulating vector operations over narrow types with ops over wider types. More...

FailureOr< Value >	rewriteBitCastOfTruncI (RewriterBase &rewriter, vector::BitCastOp bitCastOp, arith::TruncIOp truncOp, vector::BroadcastOp maybeBroadcastOp)
	Rewrite a vector `bitcast(trunci)` to use a more efficient sequence of vector operations comprising `shuffle` and `bitwise` ops. More...

FailureOr< Value >	rewriteExtOfBitCast (RewriterBase &rewriter, Operation *extOp, vector::BitCastOp bitCastOp, vector::BroadcastOp maybeBroadcastOp)
	Rewrite a vector `ext(bitcast)` to use a more efficient sequence of vector operations comprising `shuffle` and `bitwise` ops. More...

void	populateVectorNarrowTypeRewritePatterns (RewritePatternSet &patterns, PatternBenefit benefit=1)
	Appends patterns for rewriting vector operations over narrow types with ops over wider types. More...

LogicalResult	splitFullAndPartialTransfer (RewriterBase &b, VectorTransferOpInterface xferOp, VectorTransformsOptions options=VectorTransformsOptions(), scf::IfOp *ifOp=nullptr)
	Split a vector.transfer operation into an in-bounds (i.e., no out-of-bounds masking) fastpath and a slowpath. More...

void	transferOpflowOpt (RewriterBase &rewriter, Operation *rootOp)
	Implements transfer op write to read forwarding and dead transfer write optimizations. More...

LogicalResult	castAwayContractionLeadingOneDim (vector::ContractionOp contractOp, RewriterBase &rewriter)
	Cast away the leading unit dim, if exists, for the given contract op. More...

Value	createOrFoldDimOp (OpBuilder &b, Location loc, Value source, int64_t dim)
	Helper function that creates a memref::DimOp or tensor::DimOp depending on the type of `source`. More...

FailureOr< std::pair< int, int > >	isTranspose2DSlice (vector::TransposeOp op)
	Returns two dims that are greater than one if the transposition is applied on a 2D slice. More...

Typedef Documentation

◆ DistributionMapFn

using mlir::vector::DistributionMapFn = typedef std::function<AffineMap(Value)>

Definition at line 43 of file VectorDistribution.h.

Enumeration Type Documentation

◆ BroadcastableToResult

enum mlir::vector::BroadcastableToResult

strong

Return whether srcType can be broadcast to dstVectorType under the semantics of the vector.broadcast op.

Enumerator
Success
SourceRankHigher
DimensionMismatch
SourceTypeNotAVector

Definition at line 64 of file VectorOps.h.

Function Documentation

◆ buildTerminatedBody()

void mlir::vector::buildTerminatedBody	(	OpBuilder &	builder,
		Location	loc
	)

Default callback to build a region with a 'vector.yield' terminator with no arguments.

Definition at line 107 of file VectorOps.cpp.

References mlir::OpBuilder::create().

◆ castAwayContractionLeadingOneDim()

LogicalResult mlir::vector::castAwayContractionLeadingOneDim	(	vector::ContractionOp	contractOp,
		RewriterBase &	rewriter
	)

Cast away the leading unit dim, if exists, for the given contract op.

Return success if the transformation applies; return failure otherwise.

Definition at line 297 of file VectorDropLeadUnitDim.cpp.

References mlir::OpBuilder::create(), mlir::detail::enumerate(), mlir::failure(), mlir::AffineMap::get(), mlir::Builder::getAffineDimExpr(), mlir::Builder::getAffineMapArrayAttr(), mlir::Builder::getArrayAttr(), isParallelIterator(), mlir::RewriterBase::replaceOpWithNewOp(), splatZero(), and mlir::success().

◆ checkSameValueRAW()

bool mlir::vector::checkSameValueRAW	(	TransferWriteOp	defWrite,
		TransferReadOp	read
	)

Return true if the transfer_write fully writes the data accessed by the transfer_read.

◆ checkSameValueWAW()

bool mlir::vector::checkSameValueWAW	(	TransferWriteOp	write,
		TransferWriteOp	priorWrite
	)

Return true if the write op fully over-write the priorWrite transfer_write op.

◆ createLowerVectorMaskPass()

std::unique_ptr< Pass > mlir::vector::createLowerVectorMaskPass ( )

Creates an instance of the vector.mask lowering pass.

Definition at line 306 of file LowerVectorMask.cpp.

◆ createMaskOpRegion()

void mlir::vector::createMaskOpRegion	(	OpBuilder &	builder,
		Operation *	maskableOp
	)

Create the vector.yield-ended region of a vector.mask op with maskableOp as masked operation.

◆ createOrFoldDimOp()

Value mlir::vector::createOrFoldDimOp	(	OpBuilder &	b,
		Location	loc,
		Value	source,
		int64_t	dim
	)

Helper function that creates a memref::DimOp or tensor::DimOp depending on the type of source.

Definition at line 37 of file VectorUtils.cpp.

References mlir::OpBuilder::createOrFold(), and mlir::Value::getType().

◆ createVectorBufferizePass()

std::unique_ptr< Pass > mlir::vector::createVectorBufferizePass ( )

Creates an instance of the vector dialect bufferization pass.

Definition at line 53 of file Bufferize.cpp.

◆ getAsConstantIndexOps()

SmallVector<arith::ConstantIndexOp> mlir::vector::getAsConstantIndexOps ( ArrayRef< Value > values )

Returns the constant index ops in values.

values are expected to be constant operations.

◆ getAsIntegers() [1/2]

SmallVector< int64_t > mlir::vector::getAsIntegers ( ArrayRef< OpFoldResult > foldResults )

Returns the integer numbers in foldResults.

foldResults are expected to be constant operations.

Definition at line 240 of file VectorOps.cpp.

◆ getAsIntegers() [2/2]

SmallVector< int64_t > mlir::vector::getAsIntegers ( ArrayRef< Value > values )

Returns the integer numbers in values.

values are expected to be constant operations.

Definition at line 228 of file VectorOps.cpp.

References mlir::Value::getDefiningOp().

◆ getAsValues()

SmallVector< Value > mlir::vector::getAsValues	(	OpBuilder &	builder,
		Location	loc,
		ArrayRef< OpFoldResult >	foldResults
	)

Convert foldResults into Values.

Integer attributes are converted to constant op.

Definition at line 252 of file VectorOps.cpp.

References mlir::OpBuilder::create().

◆ getTransferMinorIdentityMap()

AffineMap mlir::vector::getTransferMinorIdentityMap	(	ShapedType	shapedType,
		VectorType	vectorType
	)

Build the default minor identity map suitable for a vector transfer.

This also handles the case memref<... x vector<...>> -> vector<...> in which the rank of the identity map must take the vector element type into account.

Definition at line 135 of file VectorOps.cpp.

References mlir::AffineMap::get(), mlir::getAffineConstantExpr(), and mlir::AffineMap::getMinorIdentityMap().

◆ getVectorReductionOp()

Value mlir::vector::getVectorReductionOp	(	arith::AtomicRMWKind	op,
		OpBuilder &	builder,
		Location	loc,
		Value	vector
	)

Returns the value obtained by reducing the vector into a scalar using the operation kind associated with a binary AtomicRMWKind op.

Definition at line 539 of file VectorOps.cpp.

◆ getVectorSubscriptAttr()

ArrayAttr mlir::vector::getVectorSubscriptAttr	(	Builder &	b,
		ArrayRef< int64_t >	values
	)

Returns an integer array attribute containing the given values using the integer type required for subscripts in the vector dialect.

Definition at line 322 of file VectorOps.cpp.

References mlir::Builder::getI64ArrayAttr().

◆ getVectorSubscriptType()

IntegerType mlir::vector::getVectorSubscriptType ( Builder & builder )

Returns the integer type required for subscripts in the vector dialect.

Definition at line 318 of file VectorOps.cpp.

References mlir::Builder::getIntegerType().

◆ isBroadcastableTo()

BroadcastableToResult mlir::vector::isBroadcastableTo	(	Type	srcType,
		VectorType	dstVectorType,
		std::pair< int, int > *	mismatchingDims = `nullptr`
	)

Definition at line 2120 of file VectorOps.cpp.

References mlir::getElementTypeOrSelf(), and mlir::Type::isIntOrIndexOrFloat().

Referenced by broadcastIfNeeded().

◆ isDisjointTransferIndices()

bool mlir::vector::isDisjointTransferIndices	(	VectorTransferOpInterface	transferA,
		VectorTransferOpInterface	transferB
	)

Same behavior as isDisjointTransferSet but doesn't require the operations to have the same tensor/memref.

This allows comparing operations accessing different tensors.

Definition at line 170 of file VectorOps.cpp.

References mlir::presburger::abs(), and mlir::getConstantIntValue().

Referenced by findHoistableMatchingTransferRead(), isDisjointTransferSet(), and isTensorChunkAccessedByUnknownOp().

◆ isDisjointTransferSet()

bool mlir::vector::isDisjointTransferSet	(	VectorTransferOpInterface	transferA,
		VectorTransferOpInterface	transferB
	)

Return true if we can prove that the transfer operations access disjoint memory.

Definition at line 199 of file VectorOps.cpp.

References isDisjointTransferIndices().

Referenced by mlir::linalg::hoistRedundantVectorTransfers().

◆ isParallelIterator()

bool mlir::vector::isParallelIterator ( Attribute attr )

inline

Returns true if attr has "parallel" iterator type semantics.

Definition at line 125 of file VectorOps.h.

Referenced by castAwayContractionLeadingOneDim(), contractSupportsMMAMatrixType(), and gpuMmaUnrollOrder().

◆ isReductionIterator()

bool mlir::vector::isReductionIterator ( Attribute attr )

inline

Returns true if attr has "reduction" iterator type semantics.

Definition at line 130 of file VectorOps.h.

Referenced by contractSupportsMMAMatrixType(), getReductionIndex(), and gpuMmaUnrollOrder().

◆ isTranspose2DSlice()

FailureOr< std::pair< int, int > > mlir::vector::isTranspose2DSlice ( vector::TransposeOp op )

Returns two dims that are greater than one if the transposition is applied on a 2D slice.

Otherwise, returns a failure.

Definition at line 80 of file VectorUtils.cpp.

◆ makeArithReduction()

Value mlir::vector::makeArithReduction	(	OpBuilder &	b,
		Location	loc,
		CombiningKind	kind,
		Value	v1,
		Value	acc,
		Value	mask = `Value()`
	)

Return the result value of reducing two scalar/vector values with the corresponding arith operation.

◆ maskOperation()

Operation* mlir::vector::maskOperation	(	OpBuilder &	builder,
		Operation *	maskableOp,
		Value	mask,
		Value	passthru = `Value()`
	)

Creates a vector.mask operation around a maskable operation.

Returns the vector.mask operation if the mask provided is valid. Otherwise, returns the maskable operation itself.

Referenced by VectorizationState::maskOperation(), and vectorizeAsTensorPadOp().

◆ populateBreakDownVectorBitCastOpPatterns()

void mlir::vector::populateBreakDownVectorBitCastOpPatterns	(	RewritePatternSet &	patterns,
		std::function< bool(BitCastOp)>	controlFn = `nullptr`,
		PatternBenefit	benefit = `1`
	)

Populate patterns with a pattern to break down 1-D vector.bitcast ops based on the destination vector shape.

Bitcasts from a lower bitwidth element type to a higher bitwidth one are extracted from the lower bitwidth based on the native destination vector shape and inserted based on the ratio of the bitwidths.

This acts as a last resort way to break down vector.bitcast ops to smaller vector sizes. Because this pattern composes until it is bitcasting to a single element of the higher bitwidth, the is an optional control function. If controlFn is not nullptr, the pattern will only apply to ops where controlFn returns true, otherwise applies to all bitcast ops.

◆ populateBubbleVectorBitCastOpPatterns()

void mlir::vector::populateBubbleVectorBitCastOpPatterns	(	RewritePatternSet &	patterns,
		PatternBenefit	benefit = `1`
	)

Collect a set of patterns that bubble up/down bitcast ops.

These patterns move vector.bitcast ops to be before insert ops or after extract ops where suitable. With them, bitcast will happen on smaller vectors and there are more chances to share extract/insert ops.

Definition at line 1411 of file VectorTransforms.cpp.

References mlir::RewritePatternSet::add(), and mlir::RewritePatternSet::getContext().

◆ populateCastAwayVectorLeadingOneDimPatterns()

void mlir::vector::populateCastAwayVectorLeadingOneDimPatterns	(	RewritePatternSet &	patterns,
		PatternBenefit	benefit = `1`
	)

Collect a set of leading one dimension removal patterns.

These patterns insert vector.shape_cast to remove leading one dimensions to expose more canonical forms of read/write/insert/extract operations. With them, there are more chances that we can cancel out extract-insert pairs or forward write-read pairs.

Definition at line 472 of file VectorDropLeadUnitDim.cpp.

References mlir::RewritePatternSet::add(), mlir::RewritePatternSet::getContext(), and populateShapeCastFoldingPatterns().

◆ populateFlattenVectorTransferPatterns()

void mlir::vector::populateFlattenVectorTransferPatterns	(	RewritePatternSet &	patterns,
		PatternBenefit	benefit = `1`
	)

Collect a set of patterns to flatten n-D vector transfers on contiguous memref.

These patterns insert memref.collapse_shape + vector.shape_cast patterns to transform multiple small n-D transfers into a larger 1-D transfer where the memref contiguity properties allow it.

Definition at line 811 of file VectorTransferOpTransforms.cpp.

References mlir::RewritePatternSet::add(), mlir::RewritePatternSet::getContext(), and populateShapeCastFoldingPatterns().

◆ populateFoldArithExtensionPatterns()

void mlir::vector::populateFoldArithExtensionPatterns ( RewritePatternSet & patterns )

Collect a set of patterns that fold arithmetic extension on floating point into vector contract for the backends with native support.

Definition at line 1391 of file VectorTransforms.cpp.

References mlir::RewritePatternSet::add(), and mlir::RewritePatternSet::getContext().

◆ populateScalarVectorTransferLoweringPatterns()

void mlir::vector::populateScalarVectorTransferLoweringPatterns	(	RewritePatternSet &	patterns,
		PatternBenefit	benefit,
		bool	allowMultipleUses
	)

Collects patterns that lower scalar vector transfer ops to memref loads and stores when beneficial.

If allowMultipleUses is set to true, the patterns are applied to vector transfer reads with any number of uses. Otherwise, only vector transfer reads with a single use will be lowered.

Definition at line 794 of file VectorTransferOpTransforms.cpp.

References mlir::RewritePatternSet::add(), and mlir::RewritePatternSet::getContext().

◆ populateShapeCastFoldingPatterns()

void mlir::vector::populateShapeCastFoldingPatterns	(	RewritePatternSet &	patterns,
		PatternBenefit	benefit = `1`
	)

Collect a set of vector.shape_cast folding patterns.

Definition at line 1406 of file VectorTransforms.cpp.

References mlir::RewritePatternSet::add(), and mlir::RewritePatternSet::getContext().

Referenced by populateCastAwayVectorLeadingOneDimPatterns(), populateFlattenVectorTransferPatterns(), and populateVectorTransferDropUnitDimsPatterns().

◆ populateSinkVectorBroadcastPatterns()

void mlir::vector::populateSinkVectorBroadcastPatterns	(	RewritePatternSet &	patterns,
		PatternBenefit	benefit = `1`
	)

Patterns that remove redundant vector broadcasts.

Definition at line 1448 of file VectorTransforms.cpp.

References mlir::RewritePatternSet::add(), and mlir::RewritePatternSet::getContext().

◆ populateVectorBroadcastLoweringPatterns()

void mlir::vector::populateVectorBroadcastLoweringPatterns	(	RewritePatternSet &	patterns,
		PatternBenefit	benefit = `1`
	)

Populate the pattern set with the following patterns:

[TransferReadToVectorLoadLowering] Progressive lowering of BroadcastOp to ExtractOp + InsertOp + lower-D BroadcastOp until dim 1.

Definition at line 152 of file LowerVectorBroadcast.cpp.

References mlir::RewritePatternSet::add(), and mlir::RewritePatternSet::getContext().

◆ populateVectorContractCanonicalizeMatmulToMMT()

void mlir::vector::populateVectorContractCanonicalizeMatmulToMMT	(	RewritePatternSet &	patterns,
		std::function< LogicalResult(vector::ContractionOp)>	constraint = `[](vector::ContractionOp) { return success(); }`,
		PatternBenefit	benefit = `1`
	)

Canonicalization of a vector.contraction a, b, c with row-major matmul semantics to a contraction with MMT semantics (matrix matrix multiplication with the RHS transposed).

This specific form is meant to have the vector operands are organized such that the reduction dimension is contiguous. Example:

vector.contract {indexing_maps = [affine_map<(m, n, k) -> (m, k)>,
                                  affine_map<(m, n, k) -> (n, k)>,
                                  affine_map<(m, n, k) -> (m, n)>],
                 iterator_types = ["parallel", "parallel", "reduction"],
                 kind = #vector.kind<add>} %a, %b, %c : ...

The constraint predicate is used to decide which vector.contraction ops to filter out.

Definition at line 1426 of file VectorTransforms.cpp.

References mlir::RewritePatternSet::add(), and mlir::RewritePatternSet::getContext().

Referenced by mlir::populatePrepareVectorToMMAPatterns().

◆ populateVectorContractLoweringPatterns()

void mlir::vector::populateVectorContractLoweringPatterns	(	RewritePatternSet &	patterns,
		VectorTransformsOptions	options,
		PatternBenefit	benefit = `1`,
		bool	disableOuterProductLowering = `false`
	)

Populate the pattern set with the following patterns:

[OuterProductOpLowering] Progressively lower a vector.outerproduct to linearized vector.extract + vector.fma + vector.insert.

[ContractionOpLowering] Progressive lowering of ContractionOp. One: x = vector.contract with at least one free/batch dimension is replaced by: a = vector.contract with one less free/batch dimension b = vector.contract with one less free/batch dimension

[ContractionOpToMatmulOpLowering] Progressively lower a vector.contract with row-major matmul semantics to linearized vector.shape_cast + vector.matmul on the way to llvm.matrix.multiply.

[ContractionOpToDotLowering] Progressively lower a vector.contract with row-major matmul semantics to linearized vector.extract + vector.reduce + vector.insert.

[ContractionOpToOuterProductOpLowering] Progressively lower a vector.contract with row-major matmul semantics to linearized vector.extract + vector.outerproduct + vector.insert.

Definition at line 1303 of file LowerVectorContract.cpp.

References mlir::RewritePatternSet::add(), mlir::RewritePatternSet::getContext(), and options.

◆ populateVectorExtractStridedSliceToExtractInsertChainPatterns()

void mlir::vector::populateVectorExtractStridedSliceToExtractInsertChainPatterns	(	RewritePatternSet &	patterns,
		std::function< bool(ExtractStridedSliceOp)>	controlFn = `nullptr`,
		PatternBenefit	benefit = `1`
	)

Populate patterns with a pattern to breaks down 1-D extract_strided_slice ops into a chain of Extract ops to extract each element from the source, and then a chain of Insert ops to insert to the target vector.

If controlFn is not nullptr, the pattern will only be invoked on ops that controlFn returns true. Otherwise runs on ops.

Definition at line 339 of file VectorInsertExtractStridedSliceRewritePatterns.cpp.

References mlir::RewritePatternSet::add(), and mlir::RewritePatternSet::getContext().

◆ populateVectorGatherLoweringPatterns()

void mlir::vector::populateVectorGatherLoweringPatterns	(	RewritePatternSet &	patterns,
		PatternBenefit	benefit = `1`
	)

Populate the pattern set with the following patterns:

[FlattenGather] Flattens 2 or more dimensional vector.gather ops by unrolling the outermost dimension.

[Gather1DToConditionalLoads] Turns 1-d vector.gather into a scalarized sequence of vector.loads or tensor.extracts. To avoid out-of-bounds memory accesses, these loads/extracts are made conditional using scf.if ops.

Definition at line 169 of file LowerVectorGather.cpp.

References mlir::RewritePatternSet::add(), and mlir::RewritePatternSet::getContext().

◆ populateVectorInsertExtractStridedSliceDecompositionPatterns()

void mlir::vector::populateVectorInsertExtractStridedSliceDecompositionPatterns	(	RewritePatternSet &	patterns,
		PatternBenefit	benefit = `1`
	)

Populate patterns with the following patterns.

[DecomposeDifferentRankInsertStridedSlice]

RewritePattern for InsertStridedSliceOp where source and destination vectors have different ranks.

When ranks are different, InsertStridedSlice needs to extract a properly ranked vector from the destination vector into which to insert. This pattern only takes care of this extraction part and forwards the rest to [VectorInsertStridedSliceOpSameRankRewritePattern].

For a k-D source and n-D destination vector (k < n), we emit:

ExtractOp to extract the (unique) (n-1)-D subvector into which to insert the k-D source.
k-D -> (n-1)-D InsertStridedSlice op
InsertOp that is the reverse of 1.

[DecomposeNDExtractStridedSlice]

For such cases, we can rewrite it to ExtractOp/ExtractElementOp + lower rank ExtractStridedSliceOp + InsertOp/InsertElementOp for the n-D case.

Definition at line 333 of file VectorInsertExtractStridedSliceRewritePatterns.cpp.

References mlir::RewritePatternSet::add(), and mlir::RewritePatternSet::getContext().

Referenced by populateVectorInsertExtractStridedSliceTransforms().

◆ populateVectorInsertExtractStridedSliceTransforms()

void mlir::vector::populateVectorInsertExtractStridedSliceTransforms	(	RewritePatternSet &	patterns,
		PatternBenefit	benefit = `1`
	)

Populate patterns with the following patterns.

Populate the given list with patterns that convert from Vector to LLVM.

Patterns in populateVectorInsertExtractStridedSliceDecompositionPatterns();

[ConvertSameRankInsertStridedSliceIntoShuffle]

RewritePattern for InsertStridedSliceOp where source and destination vectors have the same rank. For each outermost index in the slice: begin end stride [offset : offset+size*stride : stride]

ExtractOp one (k-1)-D source subvector and one (n-1)-D dest subvector.
InsertStridedSlice (k-1)-D into (n-1)-D
the destination subvector is inserted back in the proper place

InsertOp that is the reverse of 1.

[Convert1DExtractStridedSliceIntoShuffle]

For such cases, we can lower it to a ShuffleOp.

Definition at line 348 of file VectorInsertExtractStridedSliceRewritePatterns.cpp.

References mlir::RewritePatternSet::add(), mlir::RewritePatternSet::getContext(), and populateVectorInsertExtractStridedSliceDecompositionPatterns().

Referenced by mlir::populateVectorToLLVMConversionPatterns().

◆ populateVectorMaskLoweringPatternsForSideEffectingOps()

void mlir::vector::populateVectorMaskLoweringPatternsForSideEffectingOps ( RewritePatternSet & patterns )

Populates instances of MaskOpRewritePattern to lower masked operations with vector.mask.

Patterns should rewrite the vector.mask operation and not its nested MaskableOpInterface.

Definition at line 300 of file LowerVectorMask.cpp.

References mlir::RewritePatternSet::add(), and mlir::RewritePatternSet::getContext().

◆ populateVectorMaskMaterializationPatterns()

void mlir::vector::populateVectorMaskMaterializationPatterns	(	RewritePatternSet &	patterns,
		bool	force32BitVectorIndices,
		PatternBenefit	benefit = `1`
	)

These patterns materialize masks for various vector ops such as transfers.

Definition at line 1396 of file VectorTransforms.cpp.

References mlir::RewritePatternSet::add(), and mlir::RewritePatternSet::getContext().

◆ populateVectorMaskOpLoweringPatterns()

void mlir::vector::populateVectorMaskOpLoweringPatterns	(	RewritePatternSet &	patterns,
		PatternBenefit	benefit = `1`
	)

Populate the pattern set with the following patterns:

[CreateMaskOp] Progressive lowering of CreateMaskOp to lower-D CreateMaskOp until dim 1.

[ConstantMaskOp] Progressive lowering of ConstantMaskOp to lower-D ConstantMaskOp until dim 1.

Definition at line 162 of file LowerVectorMask.cpp.

References mlir::RewritePatternSet::add(), and mlir::RewritePatternSet::getContext().

◆ populateVectorMultiReductionLoweringPatterns()

void mlir::vector::populateVectorMultiReductionLoweringPatterns	(	RewritePatternSet &	patterns,
		VectorMultiReductionLowering	options,
		PatternBenefit	benefit = `1`
	)

Collect a set of patterns to convert vector.multi_reduction op into a sequence of vector.reduction ops.

The patterns comprise:

[InnerOuterDimReductionConversion] Rewrites vector.multi_reduction such that all reduction dimensions are either innermost or outermost, by adding the proper vector.transpose operations.

[ReduceMultiDimReductionRank] Once in innermost or outermost reduction form, rewrites n-D vector.multi_reduction into 2-D vector.multi_reduction, by introducing vector.shape_cast ops to collapse + multi-reduce + expand back.

[TwoDimMultiReductionToElementWise] Once in 2-D vector.multi_reduction form, with an outermost reduction dimension, unroll the outer dimension to obtain a sequence of 1-D vector ops. This also has an opportunity for tree-reduction (in the future).

[TwoDimMultiReductionToReduction] Once in 2-D vector.multi_reduction form, with an innermost reduction dimension, unroll the outer dimension to obtain a sequence of extract + vector.reduction + insert. This can further lower to horizontal reduction ops.

[OneDimMultiReductionToTwoDim] For cases that reduce to 1-D vector<k> reduction (and are thus missing either a parallel or a reduction), we lift them back up to 2-D with a simple vector.shape_cast to vector<1xk> so that the other patterns can kick in, thus fully exiting out of the vector.multi_reduction abstraction.

Definition at line 466 of file LowerVectorMultiReduction.cpp.

References mlir::RewritePatternSet::add(), mlir::RewritePatternSet::getContext(), and options.

◆ populateVectorNarrowTypeEmulationPatterns()

void mlir::vector::populateVectorNarrowTypeEmulationPatterns	(	arith::NarrowTypeEmulationConverter &	typeConverter,
		RewritePatternSet &	patterns
	)

Appends patterns for emulating vector operations over narrow types with ops over wider types.

Definition at line 586 of file VectorEmulateNarrowType.cpp.

References mlir::RewritePatternSet::add(), and mlir::RewritePatternSet::getContext().

◆ populateVectorNarrowTypeRewritePatterns()

void mlir::vector::populateVectorNarrowTypeRewritePatterns	(	RewritePatternSet &	patterns,
		PatternBenefit	benefit = `1`
	)

Appends patterns for rewriting vector operations over narrow types with ops over wider types.

Warning: these patterns currently only work for little endian targets.

Definition at line 595 of file VectorEmulateNarrowType.cpp.

References mlir::RewritePatternSet::add(), and mlir::RewritePatternSet::getContext().

◆ populateVectorOuterProductLoweringPatterns()

void mlir::vector::populateVectorOuterProductLoweringPatterns	(	RewritePatternSet &	patterns,
		PatternBenefit	benefit = `1`
	)

Populate the pattern set with the following patterns:

[OuterProductOpLowering] Progressively lower a vector.outerproduct to linearized vector.extract + vector.fma + vector.insert.

Definition at line 1313 of file LowerVectorContract.cpp.

References mlir::RewritePatternSet::add(), and mlir::RewritePatternSet::getContext().

◆ populateVectorReductionToContractPatterns()

void mlir::vector::populateVectorReductionToContractPatterns	(	RewritePatternSet &	patterns,
		PatternBenefit	benefit = `1`
	)

Collect patterns to convert reduction op to vector.contract and fold transpose/broadcast ops into the contract.

Definition at line 1434 of file VectorTransforms.cpp.

References mlir::RewritePatternSet::add(), and mlir::RewritePatternSet::getContext().

◆ populateVectorScanLoweringPatterns()

void mlir::vector::populateVectorScanLoweringPatterns	(	RewritePatternSet &	patterns,
		PatternBenefit	benefit = `1`
	)

Populate the pattern set with the following patterns:

[ScanToArithOps] Convert vector.scan op into arith ops and vector.insert_strided_slice / vector.extract_strided_slice.

Definition at line 252 of file LowerVectorScan.cpp.

References mlir::RewritePatternSet::add(), and mlir::RewritePatternSet::getContext().

◆ populateVectorShapeCastLoweringPatterns()

void mlir::vector::populateVectorShapeCastLoweringPatterns	(	RewritePatternSet &	patterns,
		PatternBenefit	benefit = `1`
	)

Populate the pattern set with the following patterns:

[ShapeCastOp2DDownCastRewritePattern] ShapeOp 2D -> 1D downcast serves the purpose of flattening 2-D to 1-D vectors progressively.

[ShapeCastOp2DUpCastRewritePattern] ShapeOp 1D -> 2D upcast serves the purpose of unflattening 2-D from 1-D vectors progressively.

[ShapeCastOpRewritePattern] Reference lowering to fully unrolled sequences of single element ExtractOp + InsertOp. Note that applying this pattern can almost always be considered a performance bug.

Definition at line 354 of file LowerVectorShapeCast.cpp.

References mlir::RewritePatternSet::add(), and mlir::RewritePatternSet::getContext().

◆ populateVectorToVectorCanonicalizationPatterns()

void mlir::vector::populateVectorToVectorCanonicalizationPatterns	(	RewritePatternSet &	patterns,
		PatternBenefit	benefit = `1`
	)

Collect a set of vector-to-vector canonicalization patterns.

◆ populateVectorTransferCollapseInnerMostContiguousDimsPatterns()

void mlir::vector::populateVectorTransferCollapseInnerMostContiguousDimsPatterns	(	RewritePatternSet &	patterns,
		PatternBenefit	benefit = `1`
	)

Collect a set of patterns to reduce the rank of the operands of vector transfer ops to operate on the largest contigious vector.

These patterns are useful when lowering to dialects with 1d vector type such as llvm and it will result fewer memory reads.

Definition at line 1442 of file VectorTransforms.cpp.

References mlir::RewritePatternSet::add(), and mlir::RewritePatternSet::getContext().

◆ populateVectorTransferDropUnitDimsPatterns()

void mlir::vector::populateVectorTransferDropUnitDimsPatterns	(	RewritePatternSet &	patterns,
		PatternBenefit	benefit = `1`
	)

Collect a set of one dimension removal patterns.

These patterns insert rank-reducing memref.subview ops to remove one dimensions. With them, there are more chances that we can avoid potentially expensive vector.shape_cast operations.

Definition at line 803 of file VectorTransferOpTransforms.cpp.

References mlir::RewritePatternSet::add(), mlir::RewritePatternSet::getContext(), and populateShapeCastFoldingPatterns().

◆ populateVectorTransferFullPartialPatterns()

void mlir::vector::populateVectorTransferFullPartialPatterns	(	RewritePatternSet &	patterns,
		const VectorTransformsOptions &	options
	)

Populate patterns with the following patterns.

VectorTransferFullPartialRewriter

Split a vector.transfer operation into an in-bounds (i.e., no out-of-bounds masking) fast path and a slow path.

Example (a 2-D vector.transfer_read): ``` %1 = vector.transfer_read %0[...], pad : memref<A...>, vector<...> ``` is transformed into: ``` %1:3 = scf.if (inBounds) { // fast path, direct cast memref.cast A: memref<A...> to compatibleMemRefType scf.yield view : compatibleMemRefType, index, index } else { // slow path, not in-bounds vector.transfer or linalg.copy. memref.cast alloc: memref<B...> to compatibleMemRefType scf.yield %4 : compatibleMemRefType, index, index */ // } /** %0 = vector.transfer_read %1#0[%1#1, %1#2] {in_bounds = [true ... true]} `` wherealloc` is a top of the function alloca'ed buffer of one vector.

Preconditions:

xferOp.permutation_map() must be a minor identity map
the rank of the xferOp.memref() and the rank of the xferOp.vector() must be equal. This will be relaxed in the future but requires rank-reducing subviews.

Definition at line 671 of file VectorTransferSplitRewritePatterns.cpp.

References mlir::RewritePatternSet::add(), mlir::RewritePatternSet::getContext(), and options.

◆ populateVectorTransferLoweringPatterns()

void mlir::vector::populateVectorTransferLoweringPatterns	(	RewritePatternSet &	patterns,
		std::optional< unsigned >	maxTransferRank = `std::nullopt`,
		PatternBenefit	benefit = `1`
	)

Populate the pattern set with the following patterns:

[TransferReadToVectorLoadLowering] Progressive lowering of transfer_read.This pattern supports lowering of vector.transfer_read to a combination of vector.load and vector.broadcast

[TransferWriteToVectorStoreLowering] Progressive lowering of transfer_write. This pattern supports lowering of vector.transfer_write to vector.store

[VectorLoadToMemrefLoadLowering] Replace a 0-d vector.load with a memref.load + vector.broadcast.

[VectorStoreToMemrefStoreLowering] Replace a 0-d vector.store with a vector.extractelement + memref.store.

These patterns lower transfer ops to simpler ops like vector.load, vector.store and vector.broadcast. Only transfers with a transfer rank of a most maxTransferRank are lowered. This is useful when combined with VectorToSCF, which reduces the rank of vector transfer ops.

Definition at line 630 of file LowerVectorTransfer.cpp.

References mlir::RewritePatternSet::add(), and mlir::RewritePatternSet::getContext().

◆ populateVectorTransferPermutationMapLoweringPatterns()

void mlir::vector::populateVectorTransferPermutationMapLoweringPatterns	(	RewritePatternSet &	patterns,
		PatternBenefit	benefit = `1`
	)

Collect a set of transfer read/write lowering patterns that simplify the permutation map (e.g., converting it to a minor identity map) by inserting broadcasts and transposes.

More specifically:

[TransferReadPermutationLowering] Lower transfer_read op with permutation into a transfer_read with a permutation map composed of leading zeros followed by a minor identity + vector.transpose op. Ex: vector.transfer_read ... permutation_map: (d0, d1, d2) -> (0, d1) into: v = vector.transfer_read ... permutation_map: (d0, d1, d2) -> (d1, 0) vector.transpose v, [1, 0]

vector.transfer_read ... permutation_map: (d0, d1, d2, d3) -> (0, 0, 0, d1, d3) into: v = vector.transfer_read ... permutation_map: (d0, d1, d2, d3) -> (0, 0, d1, 0, d3) vector.transpose v, [0, 1, 3, 2, 4] Note that an alternative is to transform it to linalg.transpose + vector.transfer_read to do the transpose in memory instead.

[TransferWritePermutationLowering] Lower transfer_write op with permutation into a transfer_write with a minor identity permutation map. (transfer_write ops cannot have broadcasts.) Ex: vector.transfer_write v ... permutation_map: (d0, d1, d2) -> (d2, d0, d1) into: tmp = vector.transpose v, [2, 0, 1] vector.transfer_write tmp ... permutation_map: (d0, d1, d2) -> (d0, d1, d2)

vector.transfer_write v ... permutation_map: (d0, d1, d2, d3) -> (d3, d2) into: tmp = vector.transpose v, [1, 0] v = vector.transfer_write tmp ... permutation_map: (d0, d1, d2, d3) -> (d2, d3)

[TransferOpReduceRank] Lower transfer_read op with broadcast in the leading dimensions into transfer_read of lower rank + vector.broadcast. Ex: vector.transfer_read ... permutation_map: (d0, d1, d2, d3) -> (0, d1, 0, d3) into: v = vector.transfer_read ... permutation_map: (d0, d1, d2, d3) -> (d1, 0, d3) vector.broadcast v

Definition at line 378 of file LowerVectorTransfer.cpp.

References mlir::RewritePatternSet::add(), and mlir::RewritePatternSet::getContext().

◆ populateVectorTransposeLoweringPatterns()

void mlir::vector::populateVectorTransposeLoweringPatterns	(	RewritePatternSet &	patterns,
		VectorTransformsOptions	options,
		PatternBenefit	benefit = `1`
	)

Populate the pattern set with the following patterns:

[TransposeOpLowering]

[TransposeOp2DToShuffleLowering]

Definition at line 459 of file LowerVectorTranspose.cpp.

References mlir::RewritePatternSet::add(), mlir::RewritePatternSet::getContext(), and options.

◆ populateVectorUnrollPatterns()

void mlir::vector::populateVectorUnrollPatterns	(	RewritePatternSet &	patterns,
		const UnrollVectorOptions &	options,
		PatternBenefit	benefit = `1`
	)

Collect a set of pattern to unroll vector operations to a smaller shapes.

options structure controls which operations are unrolled and the target shape. op is unrolled to the targetShape as follows, for each of its operands:

the unrolled type unrolledVectorType and number of unrolled instances numUnrolledInstances are computed from the targetShape. For now it is assumed the unrolling factors divide the vector sizes.
ExtractStridedSlice are created to break-up the vector operands.
the original op is cloned numUnrolledInstances times, once for each result.
InsertStridedSlice are inserted to re-assemble the slices into the original vectore shape.

Example:

opA(operand0, operand1) // numUnrolledInstances = 3

    operand0                   operand1
       |                          |
     fork                       fork
<----------gather all fork ops --------->
      /|\                        /|\
  f00 f01 f02                f10 f11 f12
<---------- clone op 3 times --------->
  opA0(f00, f10), opA1(f01, f11), opA2(f02, f12)
         \            |            /

<-----------------— join ----------------------—>

Other local patterns then kick in iteratively (including DCE) and compose to combine the ExtractStridedSlice/InsertStridedSlice.

Definition at line 624 of file VectorUnroll.cpp.

References mlir::RewritePatternSet::add(), mlir::RewritePatternSet::getContext(), and options.

◆ populateWarpExecuteOnLane0OpToScfForPattern()

void mlir::vector::populateWarpExecuteOnLane0OpToScfForPattern	(	RewritePatternSet &	patterns,
		const WarpExecuteOnLane0LoweringOptions &	options,
		PatternBenefit	benefit = `1`
	)

Definition at line 1655 of file VectorDistribute.cpp.

References mlir::RewritePatternSet::add(), mlir::RewritePatternSet::getContext(), and options.

◆ registerBufferizableOpInterfaceExternalModels()

void mlir::vector::registerBufferizableOpInterfaceExternalModels ( DialectRegistry & registry )

Definition at line 319 of file BufferizableOpInterfaceImpl.cpp.

References mlir::DialectRegistry::addExtension().

Referenced by mlir::registerAllDialects().

◆ registerTransformDialectExtension()

void mlir::vector::registerTransformDialectExtension ( DialectRegistry & registry )

Definition at line 208 of file VectorTransformOps.cpp.

References mlir::DialectRegistry::addExtensions().

Referenced by mlir::registerAllExtensions().

◆ rewriteBitCastOfTruncI()

FailureOr<Value> mlir::vector::rewriteBitCastOfTruncI	(	RewriterBase &	rewriter,
		vector::BitCastOp	bitCastOp,
		arith::TruncIOp	truncOp,
		vector::BroadcastOp	maybeBroadcastOp
	)

Rewrite a vector bitcast(trunci) to use a more efficient sequence of vector operations comprising shuffle and bitwise ops.

Warning: these patterns currently only work for little endian targets.

◆ rewriteExtOfBitCast()

FailureOr<Value> mlir::vector::rewriteExtOfBitCast	(	RewriterBase &	rewriter,
		Operation *	extOp,
		vector::BitCastOp	bitCastOp,
		vector::BroadcastOp	maybeBroadcastOp
	)

Rewrite a vector ext(bitcast) to use a more efficient sequence of vector operations comprising shuffle and bitwise ops.

Warning: these patterns currently only work for little endian targets.

◆ selectPassthru()

Value mlir::vector::selectPassthru	(	OpBuilder &	builder,
		Value	mask,
		Value	newValue,
		Value	passthru
	)

Creates a vector select operation that picks values from newValue or passthru for each result vector lane based on mask.

This utility is used to propagate the pass-thru value for masked-out or expeculatively executed lanes. VP intrinsics do not support pass-thru values and every mask-out lane is set to poison. LLVM backends are usually able to match op + select patterns and fold them into a native target instructions.

◆ splitFullAndPartialTransfer()

LogicalResult mlir::vector::splitFullAndPartialTransfer	(	RewriterBase &	b,
		VectorTransferOpInterface	xferOp,
		VectorTransformsOptions	options = `VectorTransformsOptions()`,
		scf::IfOp *	ifOp = `nullptr`
	)

Split a vector.transfer operation into an in-bounds (i.e., no out-of-bounds masking) fastpath and a slowpath.

If ifOp is not null and the result is success, theifOp` points to the newly created conditional upon function return. To accomodate for the fact that the original vector.transfer indexing may be arbitrary and the slow path indexes @[0...0] in the temporary buffer, the scf.if op returns a view and values of type index. At this time, only vector.transfer_read case is implemented.

Example (a 2-D vector.transfer_read): ``` %1 = vector.transfer_read %0[...], pad : memref<A...>, vector<...> ``` is transformed into: ``` %1:3 = scf.if (inBounds) { // fastpath, direct cast memref.cast A: memref<A...> to compatibleMemRefType scf.yield view : compatibleMemRefType, index, index } else { // slowpath, not in-bounds vector.transfer or linalg.copy. memref.cast alloc: memref<B...> to compatibleMemRefType scf.yield %4 : compatibleMemRefType, index, index */ // } /** %0 = vector.transfer_read %1#0[%1#1, %1#2] {in_bounds = [true ... true]} `` wherealloc` is a top of the function alloca'ed buffer of one vector.

Preconditions:

xferOp.permutation_map() must be a minor identity map
the rank of the xferOp.memref() and the rank of the xferOp.vector() must be equal. This will be relaxed in the future but requires rank-reducing subviews.

For vector.transfer_read: If ifOp is not null and the result is success, theifOp` points to the newly created conditional upon function return. To accomodate for the fact that the original vector.transfer indexing may be arbitrary and the slow path indexes @[0...0] in the temporary buffer, the scf.if op returns a view and values of type index.

Example (a 2-D vector.transfer_read): ``` %1 = vector.transfer_read %0[...], pad : memref<A...>, vector<...> ``` is transformed into: ``` %1:3 = scf.if (inBounds) { // fastpath, direct cast memref.cast A: memref<A...> to compatibleMemRefType scf.yield view : compatibleMemRefType, index, index } else { // slowpath, not in-bounds vector.transfer or linalg.copy. memref.cast alloc: memref<B...> to compatibleMemRefType scf.yield %4 : compatibleMemRefType, index, index */ // } /** %0 = vector.transfer_read %1#0[%1#1, %1#2] {in_bounds = [true ... true]} `` wherealloc` is a top of the function alloca'ed buffer of one vector.

For vector.transfer_write: There are 2 conditional blocks. First a block to decide which memref and indices to use for an unmasked, inbounds write. Then a conditional block to further copy a partial buffer into the final result in the slow path case.

Example (a 2-D vector.transfer_write): ``` vector.transfer_write arg, %0[...], pad : memref<A...>, vector<...> ``` is transformed into: ``` %1:3 = scf.if (inBounds) { memref.cast A: memref<A...> to compatibleMemRefType scf.yield view : compatibleMemRefType, index, index } else { memref.cast alloc: memref<B...> to compatibleMemRefType scf.yield %4 : compatibleMemRefType, index, index } %0 = vector.transfer_write arg, %1#0[%1#1, %1#2] {in_bounds = [true ... true]} scf.if (notInBounds) { // slowpath: not in-bounds vector.transfer or linalg.copy. } `` wherealloc` is a top of the function alloca'ed buffer of one vector.

Preconditions:

xferOp.getPermutationMap() must be a minor identity map
the rank of the xferOp.source() and the rank of the xferOp.vector() must be equal. This will be relaxed in the future but requires rank-reducing subviews.

Definition at line 519 of file VectorTransferSplitRewritePatterns.cpp.

References mlir::clone(), mlir::OpBuilder::clone(), mlir::OpBuilder::create(), createFullPartialLinalgCopy(), createFullPartialVectorTransferRead(), createFullPartialVectorTransferWrite(), createInBoundsCond(), mlir::RewriterBase::eraseOp(), mlir::failure(), mlir::Region::front(), mlir::get(), getAutomaticAllocationScope(), mlir::Builder::getBoolArrayAttr(), getCastCompatibleMemRefType(), mlir::Builder::getI64IntegerAttr(), mlir::Builder::getIndexType(), mlir::Operation::getLoc(), getLocationToWriteFullVec(), mlir::Operation::getNumRegions(), mlir::Operation::getRegion(), mlir::Value::getType(), mlir::IRMapping::map(), None, options, mlir::Operation::setAttr(), mlir::OpBuilder::setInsertionPoint(), mlir::OpBuilder::setInsertionPointToStart(), splitFullAndPartialTransferPrecondition(), mlir::succeeded(), mlir::success(), and mlir::RewriterBase::updateRootInPlace().

◆ transferOpflowOpt()

void mlir::vector::transferOpflowOpt	(	RewriterBase &	rewriter,
		Operation *	rootOp
	)

Implements transfer op write to read forwarding and dead transfer write optimizations.

Definition at line 777 of file VectorTransferOpTransforms.cpp.

References mlir::Operation::walk().

Namespaces

Classes

Typedefs

Enumerations

Functions

Typedef Documentation

◆ DistributionMapFn

Enumeration Type Documentation

◆ BroadcastableToResult

Function Documentation

◆ buildTerminatedBody()

◆ castAwayContractionLeadingOneDim()

◆ checkSameValueRAW()

◆ checkSameValueWAW()

◆ createLowerVectorMaskPass()

◆ createMaskOpRegion()

◆ createOrFoldDimOp()

◆ createVectorBufferizePass()

◆ getAsConstantIndexOps()

◆ getAsIntegers() [1/2]

◆ getAsIntegers() [2/2]

◆ getAsValues()

◆ getTransferMinorIdentityMap()

◆ getVectorReductionOp()

◆ getVectorSubscriptAttr()

◆ getVectorSubscriptType()

◆ isBroadcastableTo()

◆ isDisjointTransferIndices()

◆ isDisjointTransferSet()

◆ isParallelIterator()

◆ isReductionIterator()

◆ isTranspose2DSlice()

◆ makeArithReduction()

◆ maskOperation()

◆ populateBreakDownVectorBitCastOpPatterns()

◆ populateBubbleVectorBitCastOpPatterns()

◆ populateCastAwayVectorLeadingOneDimPatterns()

◆ populateFlattenVectorTransferPatterns()

◆ populateFoldArithExtensionPatterns()

◆ populateScalarVectorTransferLoweringPatterns()

◆ populateShapeCastFoldingPatterns()

◆ populateSinkVectorBroadcastPatterns()

◆ populateVectorBroadcastLoweringPatterns()

◆ populateVectorContractCanonicalizeMatmulToMMT()

◆ populateVectorContractLoweringPatterns()

◆ populateVectorExtractStridedSliceToExtractInsertChainPatterns()

◆ populateVectorGatherLoweringPatterns()

◆ populateVectorInsertExtractStridedSliceDecompositionPatterns()

[DecomposeDifferentRankInsertStridedSlice]

[DecomposeNDExtractStridedSlice]

◆ populateVectorInsertExtractStridedSliceTransforms()

[ConvertSameRankInsertStridedSliceIntoShuffle]

[Convert1DExtractStridedSliceIntoShuffle]

◆ populateVectorMaskLoweringPatternsForSideEffectingOps()

◆ populateVectorMaskMaterializationPatterns()

◆ populateVectorMaskOpLoweringPatterns()

◆ populateVectorMultiReductionLoweringPatterns()

◆ populateVectorNarrowTypeEmulationPatterns()

◆ populateVectorNarrowTypeRewritePatterns()

◆ populateVectorOuterProductLoweringPatterns()

◆ populateVectorReductionToContractPatterns()

◆ populateVectorScanLoweringPatterns()

◆ populateVectorShapeCastLoweringPatterns()

◆ populateVectorToVectorCanonicalizationPatterns()

◆ populateVectorTransferCollapseInnerMostContiguousDimsPatterns()

◆ populateVectorTransferDropUnitDimsPatterns()

◆ populateVectorTransferFullPartialPatterns()

◆ populateVectorTransferLoweringPatterns()

◆ populateVectorTransferPermutationMapLoweringPatterns()

◆ populateVectorTransposeLoweringPatterns()

◆ populateVectorUnrollPatterns()

◆ populateWarpExecuteOnLane0OpToScfForPattern()

◆ registerBufferizableOpInterfaceExternalModels()

◆ registerTransformDialectExtension()

◆ rewriteBitCastOfTruncI()

◆ rewriteExtOfBitCast()

◆ selectPassthru()

◆ splitFullAndPartialTransfer()

◆ transferOpflowOpt()