MLIR 24.0.0git
MoveAccumulatorForContractLoop.cpp
Go to the documentation of this file.
1//===- MoveAccumulatorForContractLoop.cpp ---------------------------------===//
2//
3// Part of the LLVM Project, under the Apache License v2.0 with LLVM Exceptions.
4// See https://llvm.org/LICENSE.txt for license information.
5// SPDX-License-Identifier: Apache-2.0 WITH LLVM-exception
6//
7//===----------------------------------------------------------------------===//
8
15
17#include "mlir/IR/Dominance.h"
19#include "llvm/Support/Casting.h"
20
21#include "mlir/Pass/Pass.h"
23
24using namespace mlir;
25using namespace mlir::vector;
26using namespace mlir::x86;
27
28namespace {
29// Transforms vector.contract(A, B, Acc) into vector.contract(A, B, 0) + Acc
30// to decouple the contraction computation from the accumulator update.
31struct MoveAccumulatorForContractLoop : public AddContractionOpRewritePattern {
32 using AddContractionOpRewritePattern::AddContractionOpRewritePattern;
33
34 LogicalResult
35 matchAndRewriteAddContractOp(vector::ContractionOp contractOp,
36 PatternRewriter &rewriter) const override {
37
38 Operation *accReadOp =
39 traceToVectorReadLikeParentOperation(contractOp.getAcc());
40
41 Value contractValue = contractionUsersAfterYield(contractOp.getResult());
42
43 if (!contractValue)
44 return rewriter.notifyMatchFailure(
45 contractOp, "Final acc write might have multiple users.");
46
47 Operation *resultUserOp = *contractValue.getUsers().begin();
48
49 if (!accReadOp || !resultUserOp)
50 return rewriter.notifyMatchFailure(
51 contractOp, "Read from acc matrix is not by "
52 "transfer_read/load/constant_zero or multiple users of "
53 "contract operation.");
54
55 if (isa<arith::ConstantOp>(accReadOp))
56 return rewriter.notifyMatchFailure(
57 contractOp,
58 "The input acc to contract is already a constant vector.");
59
60 if ((accReadOp->getBlock() == contractOp->getBlock()) ||
61 (resultUserOp->getBlock() == contractOp->getBlock()))
62 return rewriter.notifyMatchFailure(
63 contractOp, "Acc read/write should be in a separate block.");
64
65 // Replace acc of a contraction operation with vector constant.
66 Value accValue = accReadOp->getResult(0);
67 if (!accValue.hasOneUse())
68 return rewriter.notifyMatchFailure(
69 contractOp, "The input accumulator has multiple users.");
70
71 Operation *firstUser = *accValue.getUsers().begin();
72 rewriter.setInsertionPoint(firstUser);
73
74 auto vecTy = dyn_cast<VectorType>(accValue.getType());
75 if (!vecTy)
76 return rewriter.notifyMatchFailure(contractOp, "Expects vector type.");
77
78 Location loc = accReadOp->getLoc();
79 Type elemTy = vecTy.getElementType();
80
81 Value zeroVec = arith::ConstantOp::create(
82 rewriter, loc,
83 DenseElementsAttr::get(vecTy, rewriter.getZeroAttr(elemTy)));
84
85 rewriter.replaceAllUsesWith(accValue, zeroVec);
86
87 // Adds the initial acc value with contract results before storing to acc
88 // matrix.
89 rewriter.setInsertionPoint(resultUserOp);
90 Location locUser = resultUserOp->getLoc();
91
92 Value addition;
93
94 if (isa<FloatType>(elemTy)) {
95 addition =
96 arith::AddFOp::create(rewriter, locUser, contractValue, accValue);
97 } else if (isa<IntegerType>(elemTy)) {
98 addition =
99 arith::AddIOp::create(rewriter, locUser, contractValue, accValue);
100 } else {
101 llvm_unreachable("expected floating-point or integer element type");
102 }
103
104 rewriter.modifyOpInPlace(resultUserOp, [&]() {
105 resultUserOp->replaceUsesOfWith(contractValue, addition);
106 });
107 return success();
108 }
109};
110} // namespace
111
113 RewritePatternSet &patterns) {
114 patterns.add<MoveAccumulatorForContractLoop>(patterns.getContext());
115}
return success()
TypedAttr getZeroAttr(Type type)
Definition Builders.cpp:333
static DenseElementsAttr get(ShapedType type, ArrayRef< Attribute > values)
Constructs a dense elements attribute from an array of element values.
void setInsertionPoint(Block *block, Block::iterator insertPoint)
Set the insertion point to the specified location.
Definition Builders.h:401
void replaceUsesOfWith(Value from, Value to)
Replace any uses of 'from' with 'to' within this operation.
Block * getBlock()
Returns the operation block that contains this operation.
Definition Operation.h:230
OpResult getResult(unsigned idx)
Get the 'idx'th result of this operation.
Definition Operation.h:434
Location getLoc()
The source location the operation was defined or derived from.
Definition Operation.h:240
MLIRContext * getContext() const
RewritePatternSet & add(ConstructorArg &&arg, ConstructorArgs &&...args)
Add an instance of each of the pattern types 'Ts' to the pattern list with the given arguments.
std::enable_if_t<!std::is_convertible< CallbackT, Twine >::value, LogicalResult > notifyMatchFailure(Location loc, CallbackT &&reasonCallback)
Used to notify the listener that the IR failed to be rewritten because of a match failure,...
void modifyOpInPlace(Operation *root, CallableT &&callable)
This method is a utility wrapper around an in-place modification of an operation.
virtual void replaceAllUsesWith(Value from, Value to)
Find uses of from and replace them with to.
Type getType() const
Return the type of this value.
Definition Value.h:105
user_range getUsers() const
Definition Value.h:218
bool hasOneUse() const
Returns true if this value has exactly one use.
Definition Value.h:197
Value contractionUsersAfterYield(Value v)
Definition X86Utils.cpp:149
Operation * traceToVectorReadLikeParentOperation(Value v)
Definition X86Utils.cpp:173
void populateMoveAccumulatorForContractLoopPatterns(RewritePatternSet &patterns)
Include the generated interface declarations.
Convenience wrapper for patterns that are only intended for default contractions, i....