MLIR
21.0.0git
|
#include "mlir/Dialect/Linalg/Passes.h"
#include "mlir/Dialect/Affine/IR/AffineOps.h"
#include "mlir/Dialect/Arith/IR/Arith.h"
#include "mlir/Dialect/Arith/Utils/Utils.h"
#include "mlir/Dialect/Linalg/IR/Linalg.h"
#include "mlir/Dialect/Linalg/Transforms/Transforms.h"
#include "mlir/Dialect/SparseTensor/IR/SparseTensor.h"
#include "mlir/Dialect/Tensor/Transforms/Transforms.h"
#include "mlir/IR/AffineExpr.h"
#include "mlir/IR/AffineMap.h"
#include "mlir/IR/Matchers.h"
#include "mlir/IR/PatternMatch.h"
#include "mlir/Support/LLVM.h"
#include "mlir/Transforms/GreedyPatternRewriteDriver.h"
#include "mlir/Transforms/RegionUtils.h"
#include <optional>
#include <utility>
#include "mlir/Dialect/Linalg/Passes.h.inc"
Go to the source code of this file.
Namespaces | |
mlir | |
Include the generated interface declarations. | |
Macros | |
#define | GEN_PASS_DEF_LINALGELEMENTWISEOPFUSIONPASS |
Functions | |
static AffineMap | getIndexingMapOfProducerOperandsInCoordinatesOfFusedOp (OpOperand *producerOpOperand, AffineMap producerResultIndexMap, AffineMap fusedConsumerArgIndexMap) |
Append to fusedOpIndexingMapAttrs the indexing maps for the operands of the producer to use in the fused operation given the indexing map of the result of the producer in the consumer. More... | |
static bool | isOpOperandCanBeDroppedAfterFusedLinalgs (GenericOp producer, GenericOp consumer, ArrayRef< OpOperand * > opOperandsToIgnore) |
static void | generateFusedElementwiseOpRegion (RewriterBase &rewriter, GenericOp fusedOp, AffineMap consumerToProducerLoopsMap, OpOperand *fusedOperand, unsigned nloops, llvm::SmallDenseSet< int > &preservedProducerResults) |
Generate the region of the fused tensor operation. More... | |
static bool | isFusableWithReshapeByDimExpansion (LinalgOp linalgOp, OpOperand *fusableOpOperand) |
Conditions for folding a structured linalg operation with a reshape op by expanding the iteration space dimensionality for tensor operations. More... | |
static AffineMap | getIndexingMapInExpandedOp (OpBuilder &builder, AffineMap indexingMap, const ExpansionInfo &expansionInfo) |
Return the indexing map to use in the expanded op for a given the indexingMap of the original operation. More... | |
static std::tuple< SmallVector< OpFoldResult >, RankedTensorType > | getExpandedShapeAndType (RankedTensorType originalType, AffineMap indexingMap, const ExpansionInfo &expansionInfo) |
Return the shape and type of the operand/result to use in the expanded op given the type in the original op. More... | |
static SmallVector< ReassociationIndices > | getReassociationForExpansion (AffineMap indexingMap, const ExpansionInfo &expansionInfo) |
Returns the reassociation maps to use in the tensor.expand_shape operation to convert the operands of the original operation to operands of the expanded operation. More... | |
static void | updateExpandedGenericOpRegion (PatternRewriter &rewriter, Location loc, Region &fusedRegion, const ExpansionInfo &expansionInfo) |
Update the body of an expanded linalg operation having index semantics. More... | |
static Operation * | createExpandedTransposeOp (PatternRewriter &rewriter, TransposeOp transposeOp, Value expandedInput, Value output, ExpansionInfo &expansionInfo) |
static Operation * | createExpandedGenericOp (PatternRewriter &rewriter, LinalgOp linalgOp, TypeRange resultTypes, ArrayRef< Value > &expandedOpOperands, ArrayRef< Value > outputs, ExpansionInfo &expansionInfo, ArrayRef< AffineMap > expandedOpIndexingMaps) |
static Operation * | createExpandedOp (PatternRewriter &rewriter, LinalgOp linalgOp, TypeRange resultTypes, ArrayRef< Value > expandedOpOperands, ArrayRef< Value > outputs, ArrayRef< AffineMap > expandedOpIndexingMaps, ExpansionInfo &expansionInfo) |
static std::optional< SmallVector< Value > > | fuseWithReshapeByExpansion (LinalgOp linalgOp, Operation *reshapeOp, OpOperand *fusableOpOperand, PatternRewriter &rewriter) |
Implements the fusion of a tensor.collapse_shape or a tensor.expand_shape op and a generic op as explained in isFusableWithReshapeByExpansion . More... | |
static ReassociationIndices | getDomainReassociation (AffineMap indexingMap, ReassociationIndicesRef rangeReassociation) |
For a given list of indices in the range of the indexingMap that are folded, return the indices of the corresponding domain. More... | |
static SmallVector< ReassociationIndices > | getCollapsableIterationSpaceDims (GenericOp genericOp, OpOperand *fusableOperand, ArrayRef< ReassociationIndices > reassociation) |
static SmallVector< utils::IteratorType > | getCollapsedOpIteratorTypes (ArrayRef< utils::IteratorType > iteratorTypes, const CollapsingInfo &collapsingInfo) |
Get the iterator types for the collapsed operation given the original iterator types and collapsed dimensions. More... | |
static AffineMap | getCollapsedOpIndexingMap (AffineMap indexingMap, const CollapsingInfo &collapsingInfo) |
Compute the indexing map in the collapsed op that corresponds to the given indexingMap of the original operation. More... | |
static SmallVector< ReassociationIndices > | getOperandReassociation (AffineMap indexingMap, const CollapsingInfo &collapsingInfo) |
Return the reassociation indices to use to collapse the operand when the iteration space of a generic op is collapsed. More... | |
static Value | getCollapsedOpOperand (Location loc, LinalgOp op, OpOperand *opOperand, const CollapsingInfo &collapsingInfo, OpBuilder &builder) |
Get the new value to use for a given OpOperand in the collapsed operation. More... | |
static void | generateCollapsedIndexingRegion (Location loc, Block *block, const CollapsingInfo &collapsingInfo, ArrayRef< OpFoldResult > loopRange, RewriterBase &rewriter) |
Modify the linalg.index operations in the original generic op, to its value in the collapsed operation. More... | |
void | collapseOperandsAndResults (LinalgOp op, const CollapsingInfo &collapsingInfo, RewriterBase &rewriter, SmallVectorImpl< Value > &inputOperands, SmallVectorImpl< Value > &outputOperands, SmallVectorImpl< Type > &resultTypes) |
template<typename OpTy > | |
OpTy | cloneToCollapsedOp (RewriterBase &rewriter, OpTy origOp, const CollapsingInfo &collapsingInfo) |
Clone a LinalgOp to a collapsed version of same name. More... | |
template<> | |
LinalgOp | cloneToCollapsedOp< LinalgOp > (RewriterBase &rewriter, LinalgOp origOp, const CollapsingInfo &collapsingInfo) |
Collapse any LinalgOp that does not require any specialization such as indexing_maps, iterator_types, etc. More... | |
template<> | |
GenericOp | cloneToCollapsedOp< GenericOp > (RewriterBase &rewriter, GenericOp origOp, const CollapsingInfo &collapsingInfo) |
Collapse a GenericOp More... | |
LinalgOp | createCollapsedOp (LinalgOp op, const CollapsingInfo &collapsingInfo, RewriterBase &rewriter) |
#define GEN_PASS_DEF_LINALGELEMENTWISEOPFUSIONPASS |
Definition at line 33 of file ElementwiseOpFusion.cpp.
OpTy cloneToCollapsedOp | ( | RewriterBase & | rewriter, |
OpTy | origOp, | ||
const CollapsingInfo & | collapsingInfo | ||
) |
Clone a LinalgOp
to a collapsed version of same name.
Definition at line 1655 of file ElementwiseOpFusion.cpp.
Referenced by createCollapsedOp().
GenericOp cloneToCollapsedOp< GenericOp > | ( | RewriterBase & | rewriter, |
GenericOp | origOp, | ||
const CollapsingInfo & | collapsingInfo | ||
) |
Collapse a GenericOp
Definition at line 1677 of file ElementwiseOpFusion.cpp.
References collapseOperandsAndResults(), mlir::OpBuilder::create(), mlir::Block::front(), mlir::Block::getArguments(), getCollapsedOpIteratorTypes(), and mlir::RewriterBase::mergeBlocks().
LinalgOp cloneToCollapsedOp< LinalgOp > | ( | RewriterBase & | rewriter, |
LinalgOp | origOp, | ||
const CollapsingInfo & | collapsingInfo | ||
) |
Collapse any LinalgOp
that does not require any specialization such as indexing_maps, iterator_types, etc.
Definition at line 1663 of file ElementwiseOpFusion.cpp.
References mlir::clone(), and collapseOperandsAndResults().
void collapseOperandsAndResults | ( | LinalgOp | op, |
const CollapsingInfo & | collapsingInfo, | ||
RewriterBase & | rewriter, | ||
SmallVectorImpl< Value > & | inputOperands, | ||
SmallVectorImpl< Value > & | outputOperands, | ||
SmallVectorImpl< Type > & | resultTypes | ||
) |
Definition at line 1626 of file ElementwiseOpFusion.cpp.
References getCollapsedOpOperand(), and mlir::Value::getType().
Referenced by cloneToCollapsedOp< GenericOp >(), and cloneToCollapsedOp< LinalgOp >().
LinalgOp createCollapsedOp | ( | LinalgOp | op, |
const CollapsingInfo & | collapsingInfo, | ||
RewriterBase & | rewriter | ||
) |
Definition at line 1702 of file ElementwiseOpFusion.cpp.
References cloneToCollapsedOp().
Referenced by mlir::linalg::collapseOpIterationDims().
|
static |
Definition at line 805 of file ElementwiseOpFusion.cpp.
References mlir::Region::begin(), mlir::OpBuilder::cloneRegionBefore(), mlir::OpBuilder::create(), mlir::detail::enumerate(), mlir::Operation::getRegion(), and updateExpandedGenericOpRegion().
Referenced by createExpandedOp().
|
static |
Definition at line 835 of file ElementwiseOpFusion.cpp.
References mlir::clone(), createExpandedGenericOp(), and createExpandedTransposeOp().
|
static |
Definition at line 789 of file ElementwiseOpFusion.cpp.
References mlir::OpBuilder::create(), and mlir::invertPermutationVector().
Referenced by createExpandedOp().
|
static |
Implements the fusion of a tensor.collapse_shape or a tensor.expand_shape op and a generic op as explained in isFusableWithReshapeByExpansion
.
Assumes that those conditions have been satisfied.
Definition at line 865 of file ElementwiseOpFusion.cpp.
References mlir::tensor::getMixedSizes(), isFusableWithReshapeByDimExpansion(), and mlir::moveValueDefinitions().
|
static |
Modify the linalg.index
operations in the original generic op, to its value in the collapsed operation.
Definition at line 1587 of file ElementwiseOpFusion.cpp.
References mlir::OpBuilder::create(), mlir::OpBuilder::createOrFold(), mlir::detail::enumerate(), mlir::Block::getOps(), mlir::getValueOrCreateConstantIndexOp(), mlir::RewriterBase::replaceOp(), and mlir::OpBuilder::setInsertionPointToStart().
Referenced by mlir::linalg::collapseOpIterationDims().
|
static |
Generate the region of the fused tensor operation.
The region of the fused op must be empty.
Definition at line 217 of file ElementwiseOpFusion.cpp.
References mlir::Block::addArgument(), mlir::OpBuilder::clone(), mlir::OpBuilder::create(), mlir::OpBuilder::createBlock(), mlir::detail::enumerate(), mlir::Block::front(), mlir::IROperand< DerivedT, IRValueT >::get(), mlir::Block::getArgument(), mlir::Block::getArguments(), mlir::Value::getDefiningOp(), mlir::Block::getNumArguments(), mlir::OpOperand::getOperandNumber(), mlir::Block::getOps(), mlir::detail::IROperandBase::getOwner(), mlir::AffineMap::getSubMap(), mlir::Block::getTerminator(), mlir::IRMapping::lookupOrDefault(), mlir::IRMapping::map(), max(), and mlir::Block::without_terminator().
Referenced by mlir::linalg::fuseElementwiseOps().
|
static |
Definition at line 1297 of file ElementwiseOpFusion.cpp.
References mlir::detail::enumerate(), getDomainReassociation(), mlir::linalg::isParallelIterator(), and mlir::linalg::isReductionIterator().
|
static |
Compute the indexing map in the collapsed op that corresponds to the given indexingMap
of the original operation.
Definition at line 1507 of file ElementwiseOpFusion.cpp.
References mlir::AffineMap::get(), mlir::getAffineDimExpr(), mlir::AffineMap::getContext(), mlir::AffineMap::getResults(), and mlir::AffineMap::isProjectedPermutation().
|
static |
Get the iterator types for the collapsed operation given the original iterator types and collapsed dimensions.
Definition at line 1489 of file ElementwiseOpFusion.cpp.
Referenced by cloneToCollapsedOp< GenericOp >().
|
static |
Get the new value to use for a given OpOperand
in the collapsed operation.
Definition at line 1559 of file ElementwiseOpFusion.cpp.
References mlir::OpBuilder::create(), mlir::IROperand< DerivedT, IRValueT >::get(), mlir::AffineMap::getNumResults(), getOperandReassociation(), and mlir::Value::getType().
Referenced by collapseOperandsAndResults().
|
static |
For a given list of indices in the range of the indexingMap
that are folded, return the indices of the corresponding domain.
Return std::nullopt
on failure. Ensures that all the elements of the returned reassociation are distinct.
Definition at line 1183 of file ElementwiseOpFusion.cpp.
References mlir::AffineMap::getResults(), and mlir::AffineMap::isProjectedPermutation().
Referenced by getCollapsableIterationSpaceDims().
|
static |
Return the shape and type of the operand/result to use in the expanded op given the type in the original op.
Definition at line 687 of file ElementwiseOpFusion.cpp.
References mlir::decomposeMixedValues(), mlir::get(), and mlir::AffineMap::getResults().
|
static |
Return the indexing map to use in the expanded op for a given the indexingMap
of the original operation.
Definition at line 668 of file ElementwiseOpFusion.cpp.
References mlir::AffineMap::get(), mlir::Builder::getContext(), mlir::AffineMap::getNumSymbols(), and mlir::AffineMap::getResults().
|
static |
Append to fusedOpIndexingMapAttrs
the indexing maps for the operands of the producer
to use in the fused operation given the indexing map of the result of the producer in the consumer.
Definition at line 47 of file ElementwiseOpFusion.cpp.
References mlir::AffineMap::compose(), mlir::detail::IROperandBase::getOwner(), and mlir::inversePermutation().
Referenced by mlir::linalg::areElementwiseOpsFusable(), and mlir::linalg::fuseElementwiseOps().
|
static |
Return the reassociation
indices to use to collapse the operand when the iteration space of a generic op is collapsed.
Definition at line 1532 of file ElementwiseOpFusion.cpp.
References mlir::AffineMap::getNumResults(), and mlir::AffineMap::getResult().
Referenced by mlir::linalg::collapseOpIterationDims(), and getCollapsedOpOperand().
|
static |
Returns the reassociation maps to use in the tensor.expand_shape
operation to convert the operands of the original operation to operands of the expanded operation.
The same method is used to compute the tensor.collapse_shape
used to collapse the result of the expanded op to get the value that can replace all uses of the results of the original op.
Definition at line 710 of file ElementwiseOpFusion.cpp.
References mlir::AffineMap::getResults().
|
static |
Conditions for folding a structured linalg operation with a reshape op by expanding the iteration space dimensionality for tensor operations.
These are preconditions assumed by foldReshapeByDimExpansion
which implements the following fusion pattern.
Consider
c = linalg.generic ins(a, b : memref<?x?x?xf32>, memref<?x?xf32>) indexing_maps = [affine_map<(d0, d1, d2) -> (d1, d0, d2)>, affine_map<(d0, d1, d2) -> (d1, d2)>, affine_map<(d0, d1, d2) -> (d0, d2, d1)>] d = tensor.expand_shape c [[0, 1], [2], [3, 4, 5]] : tensor<?x?x?xf32> into tensor<?x?x?x?x?x?xf32>
The reshape can be folded into the linalgOp
if its loop dimensionality is increased to match the result (operand) of the tensor.expand_shape. The indexing_map of the fused tensor in the linalgOp
and the reassociation map helps compute the indexing maps of the modified op. For the above example, based on the reassociation map it can be concluded that
i.e. (e0, e1, e2, e3, e4) is the domain of the indexing map of the modified op, then
d0 -> e0, e1 d1 -> e2, e3, e4 d2 -> e5
substituting this, the structured op can be rewritten as
d = linalg.generic ins(%0, %1 : ) indexing_maps = [affine_map<(e0, e1, e2, e3, e4, e5) -> (e2, e3, e4, e0, e1, e5)>, affine_map<(e0, e1, e2, e3, e4, e5) -> (e2, e3, e4, e5)>, affine_map<(e0, e1, e2, e3, e4, e5) -> (e0, e1, e5, e2, e3, e4)>]
Since operands to the linalg generic are now 5D, reshapes can be introduced to make it consistent
%0 = tensor.expand_shape a [[0, 1, 2], [3, 4], [5]] : tensor<?x?x?xf32> into tensor<?x?x?x?x?x?xf32> %1 = tensor.expand_shape b [[0, 1, 2], [3]] : tensor<?x?x?xf32> into tensor<?x?x?x?xf32>
The added reshapes are again expanding patterns, so they will get fused with its producers if possible.
Definition at line 564 of file ElementwiseOpFusion.cpp.
References mlir::AffineMap::getNumResults().
Referenced by fuseWithReshapeByExpansion().
|
static |
Definition at line 79 of file ElementwiseOpFusion.cpp.
References mlir::concatAffineMaps(), and mlir::inversePermutation().
Referenced by mlir::linalg::getPreservedProducerResults().
|
static |
Update the body of an expanded linalg operation having index semantics.
The indices of the original operation need to be recovered by linearizing the indices of the correspoding dimensions of the expanded operation. For now it is assumed that the shapes of the expanded operation needed for linearization are static.
Definition at line 730 of file ElementwiseOpFusion.cpp.
References mlir::bindDims(), mlir::bindSymbols(), mlir::OpBuilder::create(), mlir::Region::front(), mlir::Builder::getContext(), mlir::Block::getOps(), mlir::getValueOrCreateConstantIndexOp(), mlir::affine::makeComposedFoldedAffineApply(), mlir::RewriterBase::replaceOp(), and mlir::OpBuilder::setInsertionPointAfter().
Referenced by createExpandedGenericOp().