30 #include "llvm/ADT/DenseSet.h"
31 #include "llvm/ADT/SetVector.h"
39 if (isa<UnrankedMemRefType, MemRefType>(source.
getType()))
41 if (isa<UnrankedTensorType, RankedTensorType>(source.
getType()))
43 llvm_unreachable(
"Expected MemRefType or TensorType");
69 for (int64_t permDim : transp) {
76 llvm_unreachable(
"Ill-formed transpose pattern");
81 VectorType srcType = op.getSourceVectorType();
85 srcGtOneDims.push_back(index);
87 if (srcGtOneDims.size() != 2)
97 return std::pair<int, int>(srcGtOneDims[0], srcGtOneDims[1]);
125 if (enclosingLoopToVectorDim.empty())
128 enclosingLoopToVectorDim.begin()->getFirst()->getContext();
132 for (
auto kvp : enclosingLoopToVectorDim) {
133 assert(kvp.second < perm.size());
135 cast<affine::AffineForOp>(kvp.first).getInductionVar(), indices);
136 unsigned numIndices = indices.size();
137 unsigned countInvariantIndices = 0;
138 for (
unsigned dim = 0; dim < numIndices; ++dim) {
139 if (!invariants.count(indices[dim])) {
141 "permutationMap already has an entry along dim");
144 ++countInvariantIndices;
147 assert((countInvariantIndices == numIndices ||
148 countInvariantIndices == numIndices - 1) &&
149 "Vectorization prerequisite violated: at most 1 index may be "
150 "invariant wrt a vectorized loop");
151 (void)countInvariantIndices;
160 template <
typename T>
165 if ([[maybe_unused]]
auto typedParent = dyn_cast<T>(current)) {
166 assert(res.count(current) == 0 &&
"Already inserted");
169 current = current->getParentOp();
176 return getParentsOfType<affine::AffineForOp>(block);
184 for (
auto *forInst : enclosingLoops) {
185 auto it = loopToVectorDim.find(forInst);
186 if (it != loopToVectorDim.end()) {
187 enclosingLoopToVectorDim.insert(*it);
199 bool matcher::operatesOnSuperVectorsOf(
Operation &op,
200 VectorType subVectorType) {
210 bool mustDivide =
false;
212 VectorType superVectorType;
213 if (
auto transfer = dyn_cast<VectorTransferOpInterface>(op)) {
214 superVectorType = transfer.getVectorType();
217 if (!isa<func::ReturnOp>(op)) {
218 op.
emitError(
"NYI: assuming only return operations can have 0 "
219 " results at this point");
232 op.
emitError(
"NYI: operation has more than 1 result");
241 assert((ratio || !mustDivide) &&
242 "vector.transfer operation in which super-vector size is not an"
243 " integer multiple of sub-vector size");
250 return ratio.has_value();
static bool areDimsTransposedIn2DSlice(int64_t dim0, int64_t dim1, ArrayRef< int64_t > transp)
Given the n-D transpose pattern 'transp', return true if 'dim0' and 'dim1' should be transposed with ...
static SetVector< Operation * > getEnclosingforOps(Block *block)
Returns the enclosing AffineForOp, from closest to farthest.
static AffineMap makePermutationMap(ArrayRef< Value > indices, const DenseMap< Operation *, unsigned > &enclosingLoopToVectorDim)
Constructs a permutation map from memref indices to vector dimension.
static SetVector< Operation * > getParentsOfType(Block *block)
Implementation detail that walks up the parents and records the ones with the specified type.
A multi-dimensional affine map Affine map's are immutable like Type's, and they are uniqued.
static AffineMap get(MLIRContext *context)
Returns a zero result affine map with no dimensions or symbols: () -> ().
Block represents an ordered list of Operations.
Operation * getParentOp()
Returns the closest surrounding operation that contains this block.
This class provides support for representing a failure result, or a valid value of type T.
This class defines the main interface for locations in MLIR and acts as a non-nullable wrapper around...
MLIRContext is the top-level object for a collection of MLIR operations.
This class helps build Operations.
void createOrFold(SmallVectorImpl< Value > &results, Location location, Args &&...args)
Create an operation of specific op type at the current insertion point, and immediately try to fold i...
Operation is the basic unit of execution within MLIR.
OpResult getResult(unsigned idx)
Get the 'idx'th result of this operation.
InFlightDiagnostic emitError(const Twine &message={})
Emit an error about fatal conditions with this operation, reporting up to any diagnostic handlers tha...
Block * getBlock()
Returns the operation block that contains this operation.
unsigned getNumResults()
Return the number of results held by this operation.
This class represents an instance of an SSA value in the MLIR system, representing a computable value...
Type getType() const
Return the type of this value.
DenseSet< Value, DenseMapInfo< Value > > getInvariantAccesses(Value iv, ArrayRef< Value > indices)
Given an induction variable iv of type AffineForOp and indices of type IndexType, returns the set of ...
constexpr void enumerate(std::tuple< Tys... > &tuple, CallbackT &&callback)
FailureOr< std::pair< int, int > > isTranspose2DSlice(vector::TransposeOp op)
Returns two dims that are greater than one if the transposition is applied on a 2D slice.
Value createOrFoldDimOp(OpBuilder &b, Location loc, Value source, int64_t dim)
Helper function that creates a memref::DimOp or tensor::DimOp depending on the type of source.
Include the generated interface declarations.
LogicalResult failure(bool isFailure=true)
Utility function to generate a LogicalResult.
AffineExpr getAffineConstantExpr(int64_t constant, MLIRContext *context)
std::optional< SmallVector< int64_t > > computeShapeRatio(ArrayRef< int64_t > shape, ArrayRef< int64_t > subShape)
Return the multi-dimensional integral ratio of subShape to the trailing dimensions of shape.
AffineExpr getAffineDimExpr(unsigned position, MLIRContext *context)
These free functions allow clients of the API to not use classes in detail.