MLIR
17.0.0git
|
#include "mlir/Dialect/Affine/Utils.h"
#include "mlir/Analysis/SliceAnalysis.h"
#include "mlir/Dialect/Affine/IR/AffineOps.h"
#include "mlir/Dialect/Arith/IR/Arith.h"
#include "mlir/Dialect/Func/IR/FuncOps.h"
#include "mlir/Dialect/Linalg/IR/Linalg.h"
#include "mlir/Dialect/Linalg/Transforms/Transforms.h"
#include "mlir/Dialect/Linalg/Utils/Utils.h"
#include "mlir/Dialect/Tensor/IR/Tensor.h"
#include "mlir/Dialect/Utils/StructuredOpsUtils.h"
#include "mlir/Dialect/Vector/IR/VectorOps.h"
#include "mlir/Dialect/Vector/Interfaces/MaskableOpInterface.h"
#include "mlir/IR/AffineExpr.h"
#include "mlir/IR/PatternMatch.h"
#include "mlir/Support/LLVM.h"
#include "mlir/Transforms/RegionUtils.h"
#include "llvm/ADT/Sequence.h"
#include "llvm/ADT/SmallVector.h"
#include "llvm/ADT/TypeSwitch.h"
#include "llvm/Support/Debug.h"
#include "llvm/Support/raw_ostream.h"
#include <optional>
#include <type_traits>
Go to the source code of this file.
Classes | |
struct | VectorizationState |
Contains the vectorization state and related methods used across the vectorization process of a given operation. More... | |
struct | VectorizationResult |
struct | GenericPadOpVectorizationPattern |
Rewrite a tensor::PadOp into a sequence of EmptyOp, FillOp and InsertSliceOp. More... | |
struct | VectorizePadOpUserPattern< OpTy > |
Base pattern for rewriting tensor::PadOps whose result is consumed by a given operation type OpTy. More... | |
struct | PadOpVectorizationWithTransferReadPattern |
Rewrite use of tensor::PadOp result in TransferReadOp. More... | |
struct | PadOpVectorizationWithTransferWritePattern |
Rewrite use of tensor::PadOp result in TransferWriteOp. More... | |
struct | PadOpVectorizationWithInsertSlicePattern |
Rewrite use of tensor::PadOp result in InsertSliceOp. More... | |
struct | VectorizeConvolution |
Macros | |
#define | DEBUG_TYPE "linalg-vectorization" |
#define | DBGS() (llvm::dbgs() << '[' << DEBUG_TYPE << "] ") |
#define | LDBG(X) LLVM_DEBUG(DBGS() << X << "\n") |
Typedefs | |
using | CustomVectorizationPrecondition = std::function< LogicalResult(Operation *, bool)> |
using | CustomVectorizationHook = std::function< VectorizationResult(Operation *, const IRMapping &)> |
Enumerations | |
enum class | Conv1DOpOrder { W , Ncw , Nwc } |
Helper enum to represent conv1d input traversal order. More... | |
enum | VectorizationStatus { Failure = 0 , NoReplace , NewOp } |
Helper data structure to represent the result of vectorization. More... | |
enum | VectorMemoryAccessKind { Contiguous , Gather } |
Functions | |
static FailureOr< Operation * > | vectorizeConvolution (RewriterBase &rewriter, LinalgOp convOp) |
Try to vectorize convOp as a convolution. More... | |
template<typename OpType > | |
static OpType | getSingleOpOfType (Block &block) |
Return the unique instance of OpType in block if it is indeed unique. More... | |
static SmallVector< Value > | extractConvInputSlices (RewriterBase &rewriter, Location loc, Value input, int64_t nSize, int64_t wSize, int64_t cSize, int64_t kwSize, int strideW, int dilationW, int64_t wSizeStep, bool isSingleChanneled) |
Helper function to extract the input slices after filter is unrolled along kw. More... | |
static SmallVector< Value > | extractConvFilterSlices (RewriterBase &rewriter, Location loc, Value filter, int64_t kwSize) |
Helper function to extract the filter slices after filter is unrolled along kw. More... | |
static SmallVector< Value > | extractConvResultSlices (RewriterBase &rewriter, Location loc, Value res, int64_t nSize, int64_t wSize, int64_t fSize, int64_t wSizeStep, bool isSingleChanneled) |
Helper function to extract the result slices after filter is unrolled along kw. More... | |
static Value | insertConvResultSlices (RewriterBase &rewriter, Location loc, Value res, int64_t wSize, int64_t wSizeStep, SmallVectorImpl< Value > &resVals, bool isSingleChanneled) |
Helper function to insert the computed result slices. More... | |
static AffineMap | reindexIndexingMap (AffineMap map) |
Given an indexing map coming from a LinalgOp indexing, restricted to a projectedPermutation, compress the unused dimensions to serve as a permutation_map for a vector transfer operation. More... | |
static Operation * | matchLinalgReduction (OpOperand *outputOperand) |
Check whether outputOperand is a reduction with a single combiner operation. More... | |
static Value | broadcastIfNeeded (OpBuilder &b, Value value, ArrayRef< int64_t > shape) |
Broadcast value to a vector of shape if possible. More... | |
static Operation * | buildMultiDimReduce (OpBuilder &b, Operation *reduceOp, Value valueToReduce, Value acc, ArrayRef< bool > dimsToMask) |
Create MultiDimReductionOp to compute the reduction for reductionOp . More... | |
static SmallVector< bool > | getDimsToReduce (LinalgOp linalgOp) |
static Value | buildVectorWrite (RewriterBase &rewriter, Value value, OpOperand *outputOperand, VectorizationState &state) |
Build a vector.transfer_write of value into outputOperand at indices set to all 0 ; where outputOperand is an output operand of the LinalgOp currently being vectorized. More... | |
static VectorizationResult | vectorizeLinalgYield (RewriterBase &rewriter, Operation *op, const IRMapping &bvm, VectorizationState &state, LinalgOp linalgOp, SmallVectorImpl< Value > &newResults) |
Helper function to vectorize the terminator of a linalgOp . More... | |
static VectorizationResult | vectorizeLinalgIndex (RewriterBase &rewriter, VectorizationState &state, Operation *op, LinalgOp linalgOp) |
Helper function to vectorize the index operations of a linalgOp . More... | |
static LogicalResult | tensorExtractVectorizationPrecondition (Operation *op, bool vectorizeNDExtract) |
Helper function to check if the tensor.extract can be vectorized by the custom hook vectorizeTensorExtract. More... | |
static Value | calculateGatherOffset (RewriterBase &rewriter, tensor::ExtractOp extractOp, const IRMapping &bvm, const ArrayRef< int64_t > targetShape) |
Calculates the offsets ($index_vec ) for vector.gather operations generated from tensor.extract . More... | |
static bool | isLoopInvariantIdx (LinalgOp &linalgOp, Value &val) |
Checks whether /p val can be used for calculating a loop invariant index. More... | |
static bool | isContiguousLoadIdx (LinalgOp &linalgOp, Value &val, bool &foundIndexOp) |
Check whether val could be used for calculating the trailing index for a contiguous load operation. More... | |
static VectorMemoryAccessKind | getTensorExtractMemoryAccessPattern (tensor::ExtractOp extractOp, LinalgOp &linalgOp) |
Check whether extractOp would be a gather or a contiguous load Op after vectorising linalgOp . More... | |
static VectorizationResult | vectorizeTensorExtract (RewriterBase &rewriter, VectorizationState &state, Operation *op, LinalgOp linalgOp, const IRMapping &bvm) |
Helper function to vectorize the tensor.extract operations. More... | |
static Operation * | reduceIfNeeded (OpBuilder &b, LinalgOp linalgOp, Operation *op, Value reduceValue, Value initialValue, const IRMapping &bvm) |
Emit reduction operations if the shapes of the value to reduce is different that the result shape. More... | |
static VectorizationResult | vectorizeOneOp (RewriterBase &rewriter, LinalgOp linalgOp, Operation *op, const IRMapping &bvm, ArrayRef< CustomVectorizationHook > customVectorizationHooks) |
Generic vectorization for a single operation op , given already vectorized operands carried by bvm . More... | |
static LogicalResult | vectorizeAsLinalgGeneric (RewriterBase &rewriter, VectorizationState &state, LinalgOp linalgOp, SmallVectorImpl< Value > &newResults) |
Generic vectorization function that rewrites the body of a linalgOp into vector form. More... | |
static LogicalResult | reductionPreconditions (LinalgOp op) |
static LogicalResult | vectorizeDynamicLinalgOpPrecondition (linalg::LinalgOp op) |
static void | convertAffineApply (RewriterBase &rewriter, LinalgOp linalgOp) |
Converts affine.apply Ops to arithmetic operations. More... | |
static int64_t | getIntFromAttr (Attribute attr) |
Helper function that retrieves the value of an IntegerAttr. More... | |
static SmallVector< Value > | ofrToIndexValues (RewriterBase &rewriter, Location loc, ArrayRef< OpFoldResult > ofrs) |
Given an ArrayRef of OpFoldResults, return a vector of Values. More... | |
static bool | mayExistInterleavedUses (Operation *firstOp, Operation *secondOp, ValueRange values) |
Check whether there is any interleaved use of any values between firstOp and secondOp . More... | |
static memref::SubViewOp | getSubViewUseIfUnique (Value v) |
Return the unique subview use of v if it is indeed unique, null otherwise. More... | |
template<int N> | |
static void | bindShapeDims (ShapedType shapedType) |
template<int N, typename IntTy , typename... IntTy2> | |
static void | bindShapeDims (ShapedType shapedType, IntTy &val, IntTy2 &...vals) |
template<typename... IntTy> | |
static void | bindShapeDims (ShapedType shapedType, IntTy &...vals) |
Bind a pack of int& to the leading dimensions of shapedType.getShape(). More... | |
#define DBGS | ( | ) | (llvm::dbgs() << '[' << DEBUG_TYPE << "] ") |
Definition at line 42 of file Vectorization.cpp.
#define DEBUG_TYPE "linalg-vectorization" |
Definition at line 40 of file Vectorization.cpp.
#define LDBG | ( | X | ) | LLVM_DEBUG(DBGS() << X << "\n") |
Definition at line 43 of file Vectorization.cpp.
using CustomVectorizationHook = std::function<VectorizationResult(Operation *, const IRMapping &)> |
Definition at line 606 of file Vectorization.cpp.
using CustomVectorizationPrecondition = std::function<LogicalResult(Operation *, bool)> |
Definition at line 600 of file Vectorization.cpp.
|
strong |
Helper enum to represent conv1d input traversal order.
Enumerator | |
---|---|
W | |
Ncw | |
Nwc |
Definition at line 442 of file Vectorization.cpp.
enum VectorizationStatus |
Helper data structure to represent the result of vectorization.
In certain specific cases, like terminators, we do not want to propagate/
Definition at line 450 of file Vectorization.cpp.
Enumerator | |
---|---|
Contiguous | |
Gather |
Definition at line 740 of file Vectorization.cpp.
|
static |
Definition at line 2128 of file Vectorization.cpp.
Referenced by bindShapeDims().
|
static |
Bind a pack of int& to the leading dimensions of shapedType.getShape().
Definition at line 2138 of file Vectorization.cpp.
|
static |
Definition at line 2131 of file Vectorization.cpp.
References bindShapeDims().
Broadcast value
to a vector of shape
if possible.
Return value otherwise.
Definition at line 515 of file Vectorization.cpp.
References mlir::OpBuilder::createOrFold(), mlir::getElementTypeOrSelf(), mlir::OpBuilder::getInsertionPoint(), mlir::Value::getType(), mlir::vector::isBroadcastableTo(), and mlir::vector::Success.
Referenced by buildVectorWrite(), and calculateGatherOffset().
|
static |
Create MultiDimReductionOp to compute the reduction for reductionOp
.
This assumes that reductionOp
has two operands and one of them is the reduction initial value.buildMultiDimReduce
Definition at line 534 of file Vectorization.cpp.
References mlir::OpBuilder::create(), mlir::linalg::getCombinerOpKind(), and mlir::Operation::getLoc().
Referenced by reduceIfNeeded().
|
static |
Build a vector.transfer_write of value
into outputOperand
at indices set to all 0
; where outputOperand
is an output operand of the LinalgOp currently being vectorized.
If dest
has null rank, build an memref.store. Return the produced value or null if no value is produced.
Definition at line 554 of file Vectorization.cpp.
References broadcastIfNeeded(), mlir::AffineMap::compose(), mlir::OpBuilder::create(), mlir::IROperand< DerivedT, IRValueT >::get(), mlir::Builder::getBoolArrayAttr(), VectorizationState::getCanonicalVecShape(), mlir::getElementTypeOrSelf(), mlir::Value::getLoc(), mlir::detail::IROperandBase::getOwner(), mlir::Operation::getResult(), mlir::Operation::getResults(), mlir::Value::getType(), mlir::inversePermutation(), mlir::Type::isa(), LDBG, VectorizationState::maskOperation(), and reindexIndexingMap().
Referenced by vectorizeLinalgYield().
|
static |
Calculates the offsets ($index_vec
) for vector.gather
operations generated from tensor.extract
.
The offset is calculated as follows (example using scalar values):
offset = extractOp.indices[0] for (i = 1; i < numIndices; i++) offset = extractOp.dimSize[i] * offset + extractOp.indices[i];
For tensor<45 x 80 x 15 x f32> and index [1, 2, 3], this leads to: offset = ( ( 1 ) * 80 + 2 ) * 15 + 3
Definition at line 708 of file Vectorization.cpp.
References broadcastIfNeeded(), mlir::OpBuilder::create(), mlir::Builder::getIndexType(), and mlir::IRMapping::lookup().
Referenced by vectorizeTensorExtract().
|
static |
Converts affine.apply Ops to arithmetic operations.
Definition at line 1393 of file Vectorization.cpp.
References mlir::expandAffineExpr(), mlir::RewriterBase::replaceOp(), and mlir::OpBuilder::setInsertionPoint().
Referenced by mlir::linalg::vectorize().
|
static |
Helper function to extract the filter slices after filter is unrolled along kw.
Definition at line 103 of file Vectorization.cpp.
References mlir::OpBuilder::create().
|
static |
Helper function to extract the input slices after filter is unrolled along kw.
Definition at line 68 of file Vectorization.cpp.
References mlir::OpBuilder::create().
|
static |
Helper function to extract the result slices after filter is unrolled along kw.
Definition at line 119 of file Vectorization.cpp.
References mlir::OpBuilder::create().
|
static |
Definition at line 543 of file Vectorization.cpp.
References mlir::linalg::isReductionIterator().
Referenced by reduceIfNeeded().
|
static |
Helper function that retrieves the value of an IntegerAttr.
Definition at line 1506 of file Vectorization.cpp.
References mlir::Attribute::cast().
Referenced by ofrToIndexValues().
|
static |
Return the unique instance of OpType in block
if it is indeed unique.
Return null if none or more than 1 instances exist.
Definition at line 52 of file Vectorization.cpp.
References mlir::WalkResult::advance(), mlir::WalkResult::interrupt(), and mlir::Block::walk().
|
static |
Return the unique subview use of v
if it is indeed unique, null otherwise.
Definition at line 1978 of file Vectorization.cpp.
References mlir::Value::getUses().
Referenced by mlir::linalg::LinalgCopyVTRForwardingPattern::matchAndRewrite(), and mlir::linalg::LinalgCopyVTWForwardingPattern::matchAndRewrite().
|
static |
Check whether extractOp
would be a gather or a contiguous load Op after vectorising linalgOp
.
Note that it is always safe to use gather load operations for contiguous loads (albeit slow), but not vice-versa. When in doubt, bail out and assume that extractOp
is a gather load.
Definition at line 861 of file Vectorization.cpp.
References Contiguous, mlir::detail::enumerate(), Gather, isContiguousLoadIdx(), isLoopInvariantIdx(), and LDBG.
Referenced by vectorizeTensorExtract().
|
static |
Helper function to insert the computed result slices.
Definition at line 145 of file Vectorization.cpp.
References mlir::OpBuilder::create().
|
static |
Check whether val
could be used for calculating the trailing index for a contiguous load operation.
There are currently 3 types of values that are allowed here:
val
.Additionally, the trailing index for a contiguous load operation should increment by 1 with every loop iteration, i.e. be based on:
linalg.index <dim>
, where <dim> is the trailing dim of the iteration space. foundIndexOp
is updated to true
when such an op is found. Definition at line 808 of file Vectorization.cpp.
References mlir::Value::getDefiningOp().
Referenced by getTensorExtractMemoryAccessPattern().
|
static |
Checks whether /p val can be used for calculating a loop invariant index.
Definition at line 747 of file Vectorization.cpp.
References mlir::Value::getDefiningOp().
Referenced by getTensorExtractMemoryAccessPattern().
Check whether outputOperand
is a reduction with a single combiner operation.
Return the combiner operation of the reduction. Return nullptr otherwise. Multiple reduction operations would impose an ordering between reduction dimensions and is currently unsupported in Linalg. This limitation is motivated by the fact that e.g. min(max(X)) != max(min(X))
Definition at line 499 of file Vectorization.cpp.
References mlir::OpOperand::getOperandNumber(), mlir::detail::IROperandBase::getOwner(), and mlir::matchReduction().
Referenced by reductionPreconditions().
|
static |
Check whether there is any interleaved use of any values
between firstOp
and secondOp
.
Conservatively return true
if any op or value is in a different block.
Definition at line 1951 of file Vectorization.cpp.
References mlir::Operation::getBlock(), mlir::Operation::isBeforeInBlock(), and LDBG.
Referenced by mlir::linalg::LinalgCopyVTRForwardingPattern::matchAndRewrite(), and mlir::linalg::LinalgCopyVTWForwardingPattern::matchAndRewrite().
|
static |
Given an ArrayRef of OpFoldResults, return a vector of Values.
IntegerAttrs are converted to ConstantIndexOps. Other attribute types are not supported.
Definition at line 1513 of file Vectorization.cpp.
References mlir::OpBuilder::create(), and getIntFromAttr().
Referenced by PadOpVectorizationWithInsertSlicePattern::rewriteUser(), and GenericPadOpVectorizationPattern::tryVectorizeCopy().
|
static |
Emit reduction operations if the shapes of the value to reduce is different that the result shape.
Definition at line 1010 of file Vectorization.cpp.
References buildMultiDimReduce(), mlir::Type::dyn_cast(), getDimsToReduce(), mlir::Value::getType(), and mlir::IRMapping::lookup().
Referenced by vectorizeOneOp().
|
static |
Definition at line 1273 of file Vectorization.cpp.
References mlir::failure(), mlir::linalg::getCombinerOpKind(), mlir::AffineMap::isPermutation(), mlir::linalg::isReductionIterator(), LDBG, matchLinalgReduction(), and mlir::success().
Referenced by mlir::linalg::vectorizeLinalgOpPrecondition().
Given an indexing map
coming from a LinalgOp indexing, restricted to a projectedPermutation, compress the unused dimensions to serve as a permutation_map for a vector transfer operation.
For example, given a linalg op such as:
the iteration domain size of the linalg op is 3x5x4x6x2. The first affine map is reindexed to affine_map<(d0, d1, d2) -> (d2, d0, d1)>
, the second affine map is reindexed to affine_map<(d0, d1) -> (d0, d1)>
.
Definition at line 432 of file Vectorization.cpp.
References mlir::compressUnusedDims(), and mlir::AffineMap::isProjectedPermutation().
Referenced by buildVectorWrite(), and vectorizeAsLinalgGeneric().
|
static |
Helper function to check if the tensor.extract can be vectorized by the custom hook vectorizeTensorExtract.
Definition at line 678 of file Vectorization.cpp.
References mlir::failure(), and mlir::success().
Referenced by mlir::linalg::vectorizeLinalgOpPrecondition().
|
static |
Generic vectorization function that rewrites the body of a linalgOp
into vector form.
Generic vectorization proceeds as follows:
linalgOp
has one non-empty region.When broadcastToMaximalCommonShape
is set to true, eager broadcasting is performed to the maximal common vector size implied by the linalgOp
iteration space. This eager broadcasting is introduced in the permutation_map of the vector.transfer_read operations. The eager broadcasting makes it trivial to detrmine where broadcast, transposes and reductions should occur, without any bookkeeping. The tradeoff is that, in the absence of good canonicalizations, the amount of work increases. This is not deemed a problem as we expect canonicalizations and foldings to aggressively clean up the useless work.
Definition at line 1147 of file Vectorization.cpp.
References mlir::AffineMap::compose(), mlir::OpBuilder::create(), mlir::AffineMap::dropResults(), mlir::detail::enumerate(), mlir::failure(), Failure, mlir::BlockArgument::getArgNumber(), mlir::Builder::getBoolArrayAttr(), VectorizationState::getCanonicalVecShape(), mlir::getElementTypeOrSelf(), mlir::Block::getOperations(), mlir::Operation::getResult(), mlir::Operation::getResults(), mlir::AffineMap::getResults(), mlir::getUsedValuesDefinedAbove(), mlir::inverseAndBroadcastProjectedPermutation(), mlir::inversePermutation(), LDBG, mlir::IRMapping::map(), VectorizationState::maskOperation(), NewOp, VectorizationResult::newOp, mlir::sparse_tensor::detail::readValue(), reindexIndexingMap(), VectorizationResult::status, mlir::success(), vectorizeLinalgIndex(), vectorizeLinalgYield(), vectorizeOneOp(), and vectorizeTensorExtract().
Referenced by mlir::linalg::vectorize().
|
static |
Try to vectorize convOp
as a convolution.
Helper function to vectorize a LinalgOp with convolution semantics.
Definition at line 2838 of file Vectorization.cpp.
References mlir::succeeded().
Referenced by VectorizeConvolution::matchAndRewrite(), and mlir::linalg::vectorize().
|
static |
Definition at line 1292 of file Vectorization.cpp.
References mlir::failure(), LDBG, and mlir::success().
Referenced by mlir::linalg::vectorizeLinalgOpPrecondition().
|
static |
Helper function to vectorize the index operations of a linalgOp
.
Return VectorizationStatus::NewOp to signal the vectorization algorithm that it should map the produced operations. This function is meant to be used as a CustomVectorizationHook.
Definition at line 641 of file Vectorization.cpp.
References mlir::OpBuilder::create(), Failure, VectorizationState::getCanonicalVecShape(), mlir::Builder::getIndexType(), mlir::Builder::getIndexVectorAttr(), and NewOp.
Referenced by vectorizeAsLinalgGeneric().
|
static |
Helper function to vectorize the terminator of a linalgOp
.
New result vector values are appended to newResults
. Return VectorizationStatus::NoReplace to signal the vectorization algorithm that it should not try to map produced operations and instead return the results using the newResults
vector making them available to the vectorization algorithm for RAUW. This function is meant to be used as a CustomVectorizationHook.
Definition at line 617 of file Vectorization.cpp.
References buildVectorWrite(), mlir::detail::enumerate(), Failure, mlir::IRMapping::lookup(), and NoReplace.
Referenced by vectorizeAsLinalgGeneric().
|
static |
Generic vectorization for a single operation op
, given already vectorized operands carried by bvm
.
Vectorization occurs as follows:
customVectorizationHooks
and return its result on success.ElementwiseMappable
op. It is the purpose of the customVectorizationHooks
to cover such cases.op
in vector form to a vector of shape prescribed by the first operand of maximal rank. Other operands have smaller rank and are broadcast accordingly. It is assumed this broadcast is always legal, otherwise, it means one of the customVectorizationHooks
is incorrect.This function assumes all operands of op
have been vectorized and are in the bvm
mapping. As a consequence, this function is meant to be called on a topologically-sorted list of ops. This function does not update bvm
but returns a VectorizationStatus that instructs the caller what bvm
update needs to occur.
Definition at line 1046 of file Vectorization.cpp.
References mlir::OpBuilder::clone(), mlir::OpBuilder::create(), Failure, mlir::Operation::getAttrs(), mlir::OperationName::getIdentifier(), mlir::Operation::getLoc(), mlir::Operation::getName(), mlir::Operation::getOperands(), mlir::Operation::getResultTypes(), mlir::OpTrait::hasElementwiseMappableTraits(), LDBG, mlir::IRMapping::lookup(), mlir::matchReduction(), NewOp, reduceIfNeeded(), and VectorizationResult::status.
Referenced by vectorizeAsLinalgGeneric().
|
static |
Helper function to vectorize the tensor.extract operations.
Returns VectorizationStatus::NewOp to signal the vectorization algorithm that it should map the produced operations. This function is meant to be used as a CustomVectorizationHook.
Definition at line 921 of file Vectorization.cpp.
References calculateGatherOffset(), mlir::OpBuilder::create(), Failure, Gather, mlir::DenseIntElementsAttr::get(), VectorizationState::getCanonicalVecShape(), mlir::Builder::getI1Type(), mlir::Builder::getI32Type(), mlir::Builder::getIndexType(), getTensorExtractMemoryAccessPattern(), mlir::Builder::getZeroAttr(), LDBG, mlir::IRMapping::lookup(), VectorizationState::maskOperation(), and NewOp.
Referenced by vectorizeAsLinalgGeneric().