9 #ifndef MLIR_DIALECT_LINALG_TRANSFORMS_TRANSFORMS_H 10 #define MLIR_DIALECT_LINALG_TRANSFORMS_TRANSFORMS_H 24 #include "llvm/ADT/SmallBitVector.h" 25 #include "llvm/ADT/SmallSet.h" 28 namespace bufferization {
29 class BufferizeTypeConverter;
32 class FrozenRewritePatternSet;
36 struct LinalgElementwiseFusionOptions;
37 struct LinalgFusionOptions;
38 struct LinalgTilingOptions;
65 std::function<bool(const OpResult &producer, OpOperand &consumer)>;
233 std::function<LogicalResult(OpBuilder &b, Value buffer)>;
240 std::function<LogicalResult(OpBuilder &b, Value src, Value dst)>;
247 operandsToPromote->insert(operands.begin(), operands.end());
258 unsigned size = useFullTiles.size();
259 llvm::SmallBitVector tmp(size,
false);
260 for (
unsigned i = 0; i < size; ++i)
261 tmp[i] = useFullTiles[i];
262 useFullTileBuffers = tmp;
267 bool useFullTileBuffersDefault =
false;
269 useFullTileBuffersDefault = use;
273 bool dynamicBuffers =
false;
275 dynamicBuffers = dynamic;
285 bool useAlloca =
false;
297 DeallocBufferCallbackFn
const &deallocFn) {
298 allocationFn = allocFn;
299 deallocationFn = deallocFn;
398 bool hasReplacementFilter(
Operation *op)
const;
402 filters.push_back(f);
406 template <
typename... OpTypes>
413 return addFilter([opName](
Operation *op) {
419 matchByDefault =
true;
508 paddingValues.assign(pv.begin(), pv.end());
514 paddingDimensions.assign(pd.begin(), pd.end());
521 packPaddings.assign(pp.begin(), pp.end());
527 hoistPaddings.assign(hp.begin(), hp.end());
535 transposePaddings.assign(tp.begin(), tp.end());
544 tileSizes.assign(ts.begin(), ts.end());
554 tileDistribution = std::move(distributionOptions);
567 tileSizeComputationFunction = std::move(fun);
574 tileSizeComputationFunction = [=](OpBuilder &,
Operation *) {
return ts; };
590 interchangeVector.assign(interchange.begin(), interchange.end());
608 distribution = std::move(distributionOptions);
616 distributionTypes.assign(types.begin(), types.end());
625 peeledLoops.append(loops.begin(), loops.end());
659 returningMatchAndRewrite(LinalgOp op,
PatternRewriter &rewriter)
const;
663 return returningMatchAndRewrite(op, rewriter);
701 return returningMatchAndRewrite(op, rewriter);
713 llvm::SmallSet<unsigned, 1> indicesToFuse = {};
715 indicesToFuse.insert(operands.begin(), operands.end());
752 template <
typename OpTy>
763 OpTy::getOperationName(), context, dependenceGraph, tilingOptions,
764 fusionOptions, f, fusedOpMarker, originalOpMarker, benefit) {}
793 return returningMatchAndRewrite(op, rewriter);
821 returningMatchAndRewrite(GenericOp op,
PatternRewriter &rewriter)
const;
825 return returningMatchAndRewrite(op, rewriter);
858 returningMatchAndRewrite(LinalgOp op,
PatternRewriter &rewriter)
const;
862 return returningMatchAndRewrite(op, rewriter);
899 template <
typename OpTy>
903 template <
typename ConcreateOpTy = OpTy>
977 hoistRedundantVectorTransfers = val;
984 hoistRedundantVectorTransfersOnTensor = val;
994 bool contractionLowering =
false;
996 contractionLowering = val;
1001 bool multiReductionLowering =
false;
1003 multiReductionLowering = val;
1008 bool transferPartialRewrite =
false;
1010 transferPartialRewrite = val;
1015 bool transferToSCFConversion =
false;
1017 transferToSCFConversion = val;
1021 int64_t maxTransferRank = 1;
1023 maxTransferRank = val;
1029 bool transferLowering =
true;
1031 transferLowering = val;
1036 bool shapeCastLowering =
true;
1038 shapeCastLowering = val;
1043 bool transposeLowering =
false;
1045 transposeLowering = val;
1049 bool avx2Lowering =
false;
1060 vectorTransferToSCFOptions =
options;
1067 vectorTransformOptions =
options;
1074 avx2LoweringOptions =
options;
1095 template <
typename OpTy>
1102 filter(std::move(f)), loweringType(loweringType) {}
1107 LinalgOp linalgOp = dyn_cast<LinalgOp>(op);
1110 if (
failed(filter.checkAndNotify(rewriter, linalgOp)))
1113 switch (loweringType) {
1114 case LinalgLoweringType::LibraryCall:
1121 case LinalgLoweringType::AffineLoops:
1125 case LinalgLoweringType::ParallelLoops:
1187 std::function<LogicalResult(PatternRewriter &, tensor::PadOp, Value)>;
1197 optimizeCopyFn(std::move(optimizeCopyFn)) {}
1244 LogicalResult matchAndRewrite(vector::TransferReadOp xferOp,
1271 LogicalResult matchAndRewrite(vector::TransferWriteOp xferOp,
1305 using ControlFn = std::function<llvm::Optional<bool>(tensor::ExtractSliceOp)>;
1312 LogicalResult matchAndRewrite(tensor::ExtractSliceOp sliceOp,
1322 template <
typename... OpTypes>
1333 template <
typename OpTy,
typename... OpTypes>
1345 template <
typename... OpTypes>
1356 template <
typename OpTy,
typename... OpTypes>
1374 std::function<std::pair<int64_t, unsigned>(LinalgOp op)>;
1429 #endif // MLIR_DIALECT_LINALG_TRANSFORMS_TRANSFORMS_H Include the generated interface declarations.
LogicalResult matchAndRewrite(Operation *op, PatternRewriter &rewriter) const override
Attempt to match against code rooted at the specified operation, which is the same operation code as ...
SmallVector< SmallVector< int64_t > > transposePaddings
A permutation vector for every operand used to transpose the packed PadOp results.
LinalgVectorLoweringOptions & enableContractionLowering(bool val=true)
LinalgPaddingOptions & setHoistPaddings(ArrayRef< int64_t > hp)
LogicalResult matchAndRewrite(Operation *op, PatternRewriter &rewriter) const override
Attempt to match against code rooted at the specified operation, which is the same operation code as ...
LinalgLoweringType
Linalg lowering patterns.
Options for controlling specialized AVX2 lowerings.
A special type of RewriterBase that coordinates the application of a rewrite pattern on the current I...
Operation is a basic unit of execution within MLIR.
SmallVector< Value, 4 > tensorResults
void populateDecomposeConvolutionPatterns(RewritePatternSet &patterns, const LinalgTransformationFilter &filter=LinalgTransformationFilter(), PatternBenefit benefit=1)
Linalg decompose convolutions patterns.
SmallVector< LinalgOp, 1 > fusedProducers
The generated fused operations (created within the fused loops).
static void insert(RewritePatternSet &patterns, const LinalgTilingOptions &options, const LinalgTransformationFilter &f)
This class represents a frozen set of patterns that can be processed by a pattern applicator...
std::function< LogicalResult(OpBuilder &b, Value buffer)> DeallocBufferCallbackFn
Callback function type used to deallocate the buffers used to hold the promoted subview.
void populateFoldReshapeOpsByCollapsingPatterns(RewritePatternSet &patterns, const ControlFusionFn &controlFoldingReshapes)
Patterns to fold an expanding tensor.expand_shape operation with its producer generic operation by co...
LogicalResult matchAndRewrite(GenericOp op, PatternRewriter &rewriter) const override
Options to control the application of enabling transformations.
Rewrite a tensor::PadOp into a sequence of InitTensorOp, FillOp and InsertSliceOp.
virtual void eraseOp(Operation *op)
This method erases an operation that is known to have no uses.
FailureOr< GenericOp > generalizeNamedOp(RewriterBase &rewriter, LinalgOp namedOp)
Create a GenericOp from the given named operation namedOp and replace namedOp.
bool failed(LogicalResult result)
Utility function that returns true if the provided LogicalResult corresponds to a failure value...
void populateFoldReshapeOpsByExpansionPatterns(RewritePatternSet &patterns, const ControlFusionFn &controlFoldingReshapes)
Patterns to fold an expanding (collapsing) tensor_reshape operation with its producer (consumer) gene...
LinalgPaddingOptions & setPaddingDimensions(ArrayRef< int64_t > pd)
LinalgLoweringPattern(MLIRContext *context, LinalgLoweringType loweringType, LinalgTransformationFilter f=LinalgTransformationFilter(), PatternBenefit benefit=1)
SmallVector< int64_t > tileInterchange
Tile interchange used to permute the tile loops.
FailureOr< LinalgLoops > linalgOpToAffineLoops(PatternRewriter &rewriter, LinalgOp linalgOp)
Emit a loop nest of affine.for with the proper body for linalgOp.
LinalgTilingOptions & setInterchange(ArrayRef< unsigned > interchange)
LinalgVectorLoweringOptions & enableMultiReductionLowering(bool val=true)
LinalgTilingOptions & setDistributionOptions(LinalgLoopDistributionOptions distributionOptions)
LinalgVectorLoweringOptions & enableTransferToSCFConversion(bool val=true)
LinalgPaddingOptions & setPackPaddings(ArrayRef< bool > pp)
SmallVector< int64_t > paddingDimensions
A list of iterator dimensions to pad.
OptimizeCopyFn optimizeCopyFn
std::function< SmallVector< Value, 4 >(OpBuilder &, Operation *)> TileSizeComputationFunction
LogicalResult matchAndRewrite(LinalgOp op, PatternRewriter &rewriter) const override
std::tuple< SmallVector< Range, 4 >, LoopIndexToRangeIndexMap > makeTiledLoopRanges(RewriterBase &b, Location loc, AffineMap map, ValueRange allShapeSizes, ValueRange allTileSizes)
void populatePadTensorTilingPatterns(RewritePatternSet &patterns, const LinalgTilingOptions &options)
LinalgVectorLoweringOptions & setMaxTransferRank(int64_t val)
std::set< unsigned > fusedLoopDims
The dimension of the loops that are fused.
LinalgEnablingOptions & enableLICM(bool val=true)
SmallVector< bool > packPaddings
A flag for every operand to mark the PadOp as nofold which enables packing for statically shaped oper...
This class defines the main interface for locations in MLIR and acts as a non-nullable wrapper around...
FailureOr< PromotionInfo > promoteSubviewAsNewBuffer(OpBuilder &b, Location loc, memref::SubViewOp subView, const AllocBufferCallbackFn &allocationFn, DataLayout &layout)
LinalgVectorLoweringOptions & enableAVX2Lowering(bool val=true)
Match and rewrite for the pattern: ``` alloc = ...
FailureOr< GenericOp > interchangeGenericOp(RewriterBase &rewriter, GenericOp genericOp, ArrayRef< unsigned > interchangeVector)
Interchange the iterator_types and iterator_maps dimensions and adapts the index accesses of op...
void populateElementwiseToLinalgConversionPatterns(RewritePatternSet &patterns)
Populate patterns that convert ElementwiseMappable ops to linalg parallel loops.
void populateSparseTensorRewriting(RewritePatternSet &patterns)
Populate patterns that are only useful in the context of sparse tensors.
void populateFuseTensorPadWithProducerLinalgOpPatterns(RewritePatternSet &patterns)
Pattern to fuse a tensor.pad operation with the producer of its source, if the producer is a linalg o...
std::function< Optional< Value >(OpBuilder &b, memref::SubViewOp subView, ArrayRef< Value > boundingSubViewSize, DataLayout &layout)> AllocBufferCallbackFn
Callback function type used to perform the allocation for the promoted subView.
SmallVector< scf::ForOp, 8 > Loops
Tile a nest of standard for loops rooted at rootForOp by finding such parametric tile sizes that the ...
filter controls LinalgTransformMarker matching and update when specified.
RewritePattern is the common base class for all DAG to DAG replacements.
SmallVector< Operation *, 4 > fusedLoops
The fused loop generated.
FailureOr< SmallVector< Value > > rewriteAsPaddedOp(OpBuilder &b, LinalgOp opToPad, ArrayRef< int64_t > paddingDimensions, ArrayRef< Attribute > paddingValues, ArrayRef< bool > packPaddings, LinalgOp &paddedOp)
Pad the iterator dimensions paddingDimensions of all opToPad operands to a static bounding box...
Match and rewrite for the pattern: ``` alloc = ...
StringRef getStringRef() const
Return the name of this operation. This always succeeds.
LogicalResult success(bool isSuccess=true)
Utility function to generate a LogicalResult.
This class represents an efficient way to signal success or failure.
static void insert(RewritePatternSet &patterns, const LinalgTilingOptions &options, const LinalgTransformationFilter &f)
LogicalResult failure(bool isFailure=true)
Utility function to generate a LogicalResult.
This class provides support for representing a failure result, or a valid value of type T...
FailureOr< TiledAndFusedLinalgOps > tileAndFuseLinalgOps(OpBuilder &builder, ArrayRef< LinalgOp > ops, const LinalgDependenceGraph &dependenceGraph, const LinalgTilingOptions &tilingOptions)
llvm::Optional< vector::CombiningKind > getCombinerOpKind(Operation *combinerOp)
Return vector::CombiningKind for the given op.
void populateConvolutionVectorizationPatterns(RewritePatternSet &patterns, PatternBenefit benefit=1)
Populate patterns for vectorizing low-D convolution ops.
LinalgVectorLoweringOptions & setAVX2LoweringOptions(x86vector::avx2::LoweringOptions options)
LinalgTileAndFusePattern(MLIRContext *context, const LinalgDependenceGraph &dependenceGraph, LinalgTilingOptions tilingOptions, LinalgFusionOptions fusionOptions, LinalgTransformationFilter f=LinalgTransformationFilter(), LinalgTransformationFilter fusedOpMarker=LinalgTransformationFilter(), LinalgTransformationFilter originalOpMarker=LinalgTransformationFilter(), PatternBenefit benefit=1)
LinalgPaddingOptions & setPaddingValues(ArrayRef< Attribute > pv)
LogicalResult promoteSubviewsPrecondition(Operation *op, LinalgPromotionOptions options)
Promote memref.subviews feeding linalg-on-buffers operations.
This class represents the benefit of a pattern match in a unitless scheme that ranges from 0 (very li...
void populateInlineConstantOperandsPatterns(RewritePatternSet &patterns)
Patterns that are used to inline constant operands into linalg generic ops.
LinalgFusionOptions & setIndicesToFuse(ArrayRef< int64_t > operands)
LinalgVectorLoweringOptions & enableVectorTransposeLowering(bool val=true)
x86vector::avx2::LoweringOptions avx2LoweringOptions
Configure specialized vector lowerings.
SmallVector< int64_t > tileSizes
Tile sizes used to tile the root operation.
Data structure for holding a dependence graph that operates on LinalgOp and views as SSA values...
LinalgOp op
Operation obtained by tiling the last operation in sequence of ops passed to tileAndFuseLinalgOps.
Linalg generic interchange pattern.
std::function< LogicalResult(PatternRewriter &, tensor::PadOp, Value)> OptimizeCopyFn
filter controls LinalgTransformMarker matching and update when specified.
FailureOr< LinalgOp > promoteSubViews(OpBuilder &b, LinalgOp op, const LinalgPromotionOptions &options)
Promote the subViews into a new buffer allocated at the insertion point b.
LogicalResult matchAndRewrite(LinalgOp op, PatternRewriter &rewriter) const override
LogicalResult vectorizeCopy(RewriterBase &builder, memref::CopyOp copyOp)
Emit a suitable vector form for a Copy op with fully static shape.
void populateFoldUnitExtentDimsPatterns(RewritePatternSet &patterns)
Patterns to fold unit-extent dimensions in operands/results of linalg ops on tensors.
SmallVector< int64_t > hoistPaddings
A number of loops to hoist the PadOp out for every operand.
A multi-dimensional affine map Affine map's are immutable like Type's, and they are uniqued...
static void insert(RewritePatternSet &patterns, const LinalgVectorizationOptions &options, const LinalgTransformationFilter &f)
LinalgVectorLoweringOptions & enableTransferPartialRewrite(bool val=true)
FailureOr< TiledLinalgOp > tileLinalgOp(RewriterBase &b, LinalgOp op, const LinalgTilingOptions &options)
SmallVector< int64_t > peeledLoops
Peel the specified loops.
LinalgTilingOptions & setDistributionTypes(ArrayRef< StringRef > types)
void populateConstantFoldLinalgOperations(RewritePatternSet &patterns, const ControlFusionFn &controlFn)
Patterns to constant fold Linalg operations.
LinalgEnablingOptions & enableHoistRedundantVectorTransfers(bool val=true)
LogicalResult matchAndRewrite(LinalgOp op, PatternRewriter &rewriter) const override
LinalgVectorLoweringOptions & setVectorTransformsOptions(vector::VectorTransformsOptions options)
void populateLinalgTilingCanonicalizationPatterns(RewritePatternSet &patterns)
void populateLinalgNamedOpConversionPatterns(RewritePatternSet &patterns)
Patterns to convert from one named op to another.
SmallVector< Attribute > paddingValues
A padding value for every operand.
std::function< LogicalResult(OpBuilder &b, Value src, Value dst)> CopyCallbackFn
Callback function type used to insert copy from original subview to subview of the promoted region fo...
This class represents an instance of an SSA value in the MLIR system, representing a computable value...
FailureOr< LinalgLoops > linalgOpToLoops(PatternRewriter &rewriter, LinalgOp linalgOp)
Emit a loop nest of scf.for with the proper body for linalgOp.
void transformIndexOps(RewriterBase &b, LinalgOp op, SmallVectorImpl< Value > &ivs, const LoopIndexToRangeIndexMap &loopIndexToRangeIndex)
All indices returned by IndexOp should be invariant with respect to tiling.
static llvm::ManagedStatic< PassManagerOptions > options
LinalgVectorLoweringOptions & enableShapeCastLowering(bool val=true)
OpRewritePattern is a wrapper around RewritePattern that allows for matching and rewriting against an...
void hoistRedundantVectorTransfers(func::FuncOp func)
Hoist vector.transfer_read/vector.transfer_write on buffers pairs out of immediately enclosing scf::F...
LinalgTilingLoopType
The type of loops to be generated during tiling.
RewritePatternSet & add(ConstructorArg &&arg, ConstructorArgs &&... args)
Add an instance of each of the pattern types 'Ts' to the pattern list with the given arguments...
LinalgVectorLoweringOptions & enableTransferLowering(bool val=true)
VectorTransferToSCFOptions vectorTransferToSCFOptions
Configure the post staged-patterns late vector.transfer to scf conversion.
Fuse a sequence of linalg operations (ops) using tile-and-fuse.
LinalgTilingOptions & setLoopType(LinalgTilingLoopType lt)
LinalgTilingAndFusionOptions & setDistributionOptions(LinalgLoopDistributionOptions distributionOptions)
Linalg generalization pattern.
LinalgVectorLoweringOptions & setVectorTransferToSCFOptions(VectorTransferToSCFOptions options)
Linalg tile and fuse tensor ops pattern.
void populateBubbleUpExtractSliceOpPatterns(RewritePatternSet &patterns)
Patterns that are used to bubble up extract slice op above linalg op.
Perform standalone tiling of a single LinalgOp by tileSizes.
LinalgTilingOptions & setPeeledLoops(ArrayRef< int64_t > loops)
GeneralizePadOpPattern(MLIRContext *context, OptimizeCopyFn optimizeCopyFn=nullptr, PatternBenefit benefit=1)
MLIRContext is the top-level object for a collection of MLIR operations.
void populateLinalgNamedOpsGeneralizationPatterns(RewritePatternSet &patterns, const LinalgTransformationFilter &filter=LinalgTransformationFilter())
Linalg generalization patterns.
Vector lowering options control how ops are lowered down to 1-D and scf.for form. ...
std::function< bool(const OpResult &producer, OpOperand &consumer)> ControlFusionFn
Function type which is used to control when to stop fusion.
LogicalResult vectorize(RewriterBase &builder, LinalgOp linalgOp)
Emit a suitable vector form for a Linalg op with fully static shape.
void populateSplitReductionPattern(RewritePatternSet &patterns, const ControlSplitReductionFn &controlSplitReductionFn, const LinalgTransformationFilter &f=LinalgTransformationFilter())
Patterns to apply splitReduction below.
void hoistRedundantVectorTransfersOnTensor(func::FuncOp func)
Same behavior as hoistRedundantVectorTransfers but works on tensors instead of buffers.
FailureOr< LinalgOp > splitReduction(PatternRewriter &b, LinalgOp op, const ControlSplitReductionFn &controlSplitReductionFn, const LinalgTransformationFilter &f)
Apply transformation to split the single linalg op reduction into a parallel and reduction dimension...
void populateElementwiseOpsFusionPatterns(RewritePatternSet &patterns, const ControlFusionFn &controlElementwiseOpFusion)
Patterns for fusing linalg operation on tensors.
Linalg vectorization patterns.
LogicalResult applyStagedPatterns(Operation *op, ArrayRef< FrozenRewritePatternSet > stage1Patterns, const FrozenRewritePatternSet &stage2Patterns, function_ref< LogicalResult(Operation *)> stage3Lambda=nullptr)
Helper function to allow applying rewrite patterns, interleaved with more global transformations, in a staged fashion:
void peelTiledLinalgOp(RewriterBase &rewriter, TiledLinalgOp &res, ArrayRef< int64_t > peeledLoops, LinalgTilingLoopType loopType)
Peel the loops of a TiledLinalgOp.
OperationName getName()
The name of an operation is the key identifier for it.
void populatePadOpVectorizationPatterns(RewritePatternSet &patterns, PatternBenefit baseBenefit=1)
Populates patterns with patterns that vectorize linalg.pad_tensor.
std::function< std::pair< int64_t, unsigned >(LinalgOp op)> ControlSplitReductionFn
Function signature to control reduction splitting.
SmallVector< Operation *, 8 > loops
LinalgTilingOptions & setTileSizeComputationFunction(TileSizeComputationFunction fun)
FailureOr< LinalgLoops > linalgOpToParallelLoops(PatternRewriter &rewriter, LinalgOp linalgOp)
Emit a loop nest of scf.parallel with the proper body for linalgOp.
RewritePatternSet getLinalgTilingCanonicalizationPatterns(MLIRContext *ctx)
Canonicalization patterns relevant to apply after tiling patterns.
Options that allow distribution of loops generated in Linalg transforms to processors while generatin...
This class helps build Operations.
This class provides an abstraction over the different types of ranges over Values.
LinalgEnablingOptions & enableHoistRedundantVectorTransfersOnTensor(bool val=true)
LinalgTilingAndFusionOptions & setTileSizes(ArrayRef< int64_t > ts)
MLIRContext * getContext() const
static void insert(RewritePatternSet &patterns, const LinalgVectorizationOptions &options, const LinalgTransformationFilter &f)
When lowering an N-d vector transfer op to an (N-1)-d vector transfer op, a temporary buffer is creat...
This class coordinates the application of a rewrite on a set of IR, providing a way for clients to tr...
LinalgTilingOptions & setTileSizes(const SmallVector< Value, 4 > &ts)
Set the tileSizeComputationFunction to return the values ts.
OpInterfaceRewritePattern is a wrapper around RewritePattern that allows for matching and rewriting a...
The main mechanism for performing data layout queries.
vector::VectorTransformsOptions vectorTransformOptions
Configure late vector transformations.
LinalgPaddingOptions & setTransposePaddings(ArrayRef< SmallVector< int64_t >> tp)