56 #include "llvm/ADT/DenseSet.h" 57 #include "llvm/ADT/SetVector.h" 83 llvm::to_vector<4>(attr.getAsValueRange<StringAttr>()));
93 OpBuilder(op).getStrArrayAttr(inPlaceVector));
106 for (
Block &b : r.getBlocks())
107 for (
auto bbArg : b.getArguments())
117 equivalentInfo.insert(v);
124 aliasInfo.unionSets(newValue, alias);
132 equivalentInfo.unionSets(newValue, alias);
137 return inplaceBufferized.contains(&operand);
145 aliasInfo.unionSets(result, operand.
get());
150 assert(!inplaceBufferized.contains(&operand) &&
151 "OpOperand was already decided to bufferize inplace");
157 auto leaderIt = equivalentInfo.findLeader(v);
158 for (
auto mit = leaderIt, meit = equivalentInfo.member_end(); mit != meit;
167 auto leaderIt = aliasInfo.findLeader(v);
168 for (
auto mit = leaderIt, meit = aliasInfo.member_end(); mit != meit; ++mit) {
174 BufferizationAliasInfo::getAliases(
Value v)
const {
176 auto it = aliasInfo.findValue(aliasInfo.getLeaderValue(v));
177 for (
auto mit = aliasInfo.member_begin(it), meit = aliasInfo.member_end();
178 mit != meit; ++mit) {
179 res.insert(static_cast<Value>(*mit));
182 aliasInfo.member_begin(it), aliasInfo.member_end());
194 op->
walk([&](BufferizableOpInterface bufferizableOp) {
197 for (
OpOperand &opOperand : bufferizableOp->getOpOperands()) {
198 if (opOperand.get().getType().isa<
TensorType>())
199 if (bufferizableOp.mustBufferizeInPlace(opOperand, *
this)) {
201 bufferizableOp.getAliasingOpResult(opOperand, *
this))
229 Value returnVal = returnValOperand.get();
238 if (bbArg.getOwner()->getParentOp() == returnOp->getParentOp())
239 yieldedTensors.insert(bbArg);
243 if (definingOp->getParentOp() == returnOp->
getParentOp())
244 yieldedTensors.insert(v);
270 bool isUndefined = llvm::none_of(lastWrites, [&](
Value lastWrite) {
271 if (
auto bufferizableOp =
getOptions().dynCastBufferizableOp(lastWrite))
272 return bufferizableOp.isMemoryWrite(lastWrite.
cast<
OpResult>(),
277 for (
OpOperand &use : opResult.getUses())
278 undefinedTensorUses.insert(&use);
286 return undefinedTensorUses.contains(opOperand);
290 return yieldedTensors.contains(tensor);
313 bool foundNonWritableBuffer =
false;
318 if (bufferizableOp && bufferizableOp.isWritable(v, state))
324 bbArg.getOwner()->getParentOp()))
325 if (bufferizableOp.isWritable(bbArg, state))
328 foundNonWritableBuffer =
true;
331 return foundNonWritableBuffer;
339 bool foundInplaceWrite =
false;
341 for (
auto &use : v.
getUses()) {
343 foundInplaceWrite =
true;
348 return foundInplaceWrite;
390 return bufferizableOp.isMemoryWrite(opResult, state);
396 static uint64_t counter = 0;
401 std::string
id =
"C_" + std::to_string(counter++);
403 std::string conflictingWriteAttr =
407 conflictingWritingOp->
setAttr(conflictingWriteAttr, b.getUnitAttr());
409 std::string readAttr =
411 readingOp->
setAttr(readAttr, b.getUnitAttr());
414 std::string lastWriteAttr =
id +
"[LAST-WRITE: result " +
415 std::to_string(opResult.getResultNumber()) +
417 opResult.getDefiningOp()->setAttr(lastWriteAttr, b.getUnitAttr());
420 std::string lastWriteAttr =
421 id +
"[LAST-WRITE: bbArg " + std::to_string(bbArg.getArgNumber()) +
"]";
443 writtenAliases.push_back(uWrite->get());
450 Operation *readingOp = uRead->getOwner();
466 for (
OpOperand *uConflictingWrite : usesWrite) {
469 Operation *conflictingWritingOp = uConflictingWrite->getOwner();
509 bool canUseOpDominance =
510 writtenAliases.empty() ||
511 repetitiveRegionOfWrites ==
519 if (canUseOpDominance &&
529 if (canUseOpDominance && uConflictingWrite == uRead)
534 if (bufferizableOp.isNotConflicting(uRead, uConflictingWrite, state))
537 if (conflictingWritingOp != readingOp)
538 if (
auto bufferizableOp =
540 if (bufferizableOp.isNotConflicting(uRead, uConflictingWrite, state))
548 if (canUseOpDominance &&
553 for (
Value lastWrite : lastWrites) {
556 if (
Operation *writingOp = lastWrite.getDefiningOp()) {
561 if (writingOp->isProperAncestor(conflictingWritingOp))
565 Block *block = bbArg.getOwner();
566 if (!block->findAncestorOpInBlock(*conflictingWritingOp))
576 if (aliasingOpResult.size() == 1 && aliasingOpResult[0] == lastWrite)
624 bool checkConsistencyOnly =
false) {
628 for (
auto &use : alias.
getUses())
638 for (
auto &use : alias.
getUses())
647 getAliasingReads(usesRead, operand.
get());
648 getAliasingInplaceWrites(usesWrite, operand.
get());
650 getAliasingReads(usesRead, result);
651 getAliasingInplaceWrites(usesWrite, result);
654 usesWrite.insert(&operand);
692 bool foundInterference =
696 if (foundInterference)
726 unsigned analysisFuzzerSeed = 0) {
727 if (analysisFuzzerSeed) {
731 std::mt19937 g(analysisFuzzerSeed);
732 llvm::shuffle(ops.begin(), ops.end(), g);
737 for (
OpOperand &opOperand : op->getOpOperands())
738 if (opOperand.get().getType().isa<
TensorType>())
751 return hasTensorResult || hasTensorOperand;
759 unsigned analysisFuzzerSeed = 0) {
769 return inPlaceAnalysis(ops, aliasInfo, state, domInfo, analysisFuzzerSeed);
778 for (
OpResult opResult : op->getOpResults())
781 bufferizableOp.getAliasingOpOperand(opResult, state))
783 if (bufferizableOp.bufferRelation(opResult, state) ==
815 if (opOperand.get().getType().isa<
TensorType>()) {
817 opOperand, domInfo, state, aliasInfo,
830 return inconsistentOp->
emitError(
"input IR has RaW conflict");
842 if (opOperand.get().getType().isa<
TensorType>())
882 Value returnVal = returnValOperand.get();
887 bool foundEquivValue =
false;
890 Operation *definingOp = bbArg.getOwner()->getParentOp();
891 if (definingOp->isProperAncestor(returnOp))
892 foundEquivValue = true;
897 if (definingOp->getBlock()->findAncestorOpInBlock(
901 foundEquivValue =
true;
904 if (!foundEquivValue)
907 <<
"operand #" << returnValOperand.getOperandNumber()
908 <<
" of ReturnLike op does not satisfy destination passing style";
921 const auto &options =
925 assert((state.
hasDialectState(func::FuncDialect::getDialectNamespace()) ||
926 !options.bufferizeFunctionBoundaries) &&
927 "must use ModuleBufferize to bufferize function boundaries");
934 options.analysisFuzzerSeed)))
940 if (
failed(fn(op, state, aliasInfo, newOps)))
948 bool failedAnalysis =
false;
949 if (!options.allowReturnAllocs) {
963 if (BufferizableOpInterface bufferizableOp =
964 options.dynCastBufferizableOp(op))
965 failedAnalysis |=
failed(bufferizableOp.verifyAnalysis(state));
969 if (options.testAnalysisOnly)
972 return success(!failedAnalysis);
bool properlyDominates(Operation *a, Operation *b, bool enclosingOpOk=true) const
Return true if operation A properly dominates operation B, i.e.
Include the generated interface declarations.
This class contains a list of basic blocks and a link to the parent operation it is attached to...
SmallVector< OpResult > getAliasingOpResult(OpOperand &opOperand) const
Determine which OpResult will alias with opOperand if the op is bufferized in place.
LogicalResult analyzeOp(Operation *op, OneShotAnalysisState &state)
Analyze op and its nested ops.
LogicalResult bufferizeOp(Operation *op, const AnalysisState &analysisState)
Bufferize op and its nested ops that implement BufferizableOpInterface.
U dyn_cast_or_null() const
Operation is a basic unit of execution within MLIR.
MutableArrayRef< Region > getRegions()
Returns the regions held by this operation.
static Optional< Region * > getCommonEnclosingRepetitiveRegion(ArrayRef< Value > values)
For each given value, find the closest enclosing repetitive region.
Operation * getParentOp()
Returns the closest surrounding operation that contains this block.
OneShotAnalysisState(Operation *op, const OneShotBufferizationOptions &options)
static bool hasTensorSemantics(Operation *op)
Return true if the given op has a tensor result or a tensor operand.
This is a value defined by a result of an operation.
Block represents an ordered list of Operations.
bool wasInterrupted() const
Returns true if the walk was interrupted.
const BufferizationOptions & getOptions() const
Return a reference to the BufferizationOptions.
bool failed(LogicalResult result)
Utility function that returns true if the provided LogicalResult corresponds to a failure value...
bool areEquivalentBufferizedValues(Value v1, Value v2) const override
Return true if v1 and v2 bufferize to equivalent buffers.
unsigned getNumOperands()
virtual bool isInPlace(OpOperand &opOperand) const =0
Return true if the given OpResult has been decided to bufferize inplace.
bool testAnalysisOnly
If set to true, does not modify the IR apart from adding attributes (for checking the results of the ...
static void annotateOpsWithBufferizationMarkers(Operation *op, const BufferizationAliasInfo &aliasInfo, AnalysisState &state)
Annotate the IR with the result of the analysis. For testing/debugging only.
operand_type_range getOperandTypes()
static bool isInplaceMemoryWrite(OpOperand &opOperand, const BufferizationAliasInfo &aliasInfo, AnalysisState &state)
Return true if opOperand has been decided to bufferize in-place.
bool areEquivalentBufferizedValues(Value v1, Value v2) const
Return true if v1 and v2 bufferize to equivalent buffers.
static bool aliasesInPlaceWrite(Value value, const BufferizationAliasInfo &aliasInfo, AnalysisState &state)
Return true if the buffer to which operand would bufferize is equivalent to some buffer write...
A class for computing basic dominance information.
bool hasDialectState(StringRef name) const
Return true if the given dialect state exists.
static LogicalResult bufferizableInPlaceAnalysisImpl(OpOperand &operand, BufferizationAliasInfo &aliasInfo, AnalysisState &state, const DominanceInfo &domInfo)
Determine if operand can be bufferized in-place.
bool isRegionReturnLike(Operation *operation)
Returns true if the given operation is either annotated with the ReturnLike trait or implements the R...
static void equivalenceAnalysis(SmallVector< Operation *> &ops, BufferizationAliasInfo &aliasInfo, AnalysisState &state)
Analyze equivalence of tied OpResult/OpOperand pairs of the given ops.
void unionEquivalenceClasses(Value v1, Value v2)
Union the equivalence classes of v1 and v2.
bool bufferizesToMemoryWrite(OpOperand &opOperand) const
Return true if opOperand bufferizes to a memory write.
static bool wouldCreateReadAfterWriteInterference(OpOperand &operand, const DominanceInfo &domInfo, AnalysisState &state, const BufferizationAliasInfo &aliasInfo, bool checkConsistencyOnly=false)
Return true if bufferizing operand inplace would create a conflict.
The BufferizationAliasInfo class maintains a list of buffer aliases and equivalence classes to suppor...
bool isInPlace(OpOperand &opOperand) const
Return true if a value was marked as in-place bufferized.
void insertNewBufferAlias(Value newValue, Value alias)
Insert an info entry for newValue and merge its alias set with that of alias.
static constexpr const bool value
MLIRContext * getContext()
Return the context this operation is associated with.
Block * getOwner() const
Returns the block that owns this argument.
void createAliasInfoEntry(Value v)
Add a new entry for v in the aliasInfo and equivalentInfo.
void gatherYieldedTensors(Operation *op)
Find all tensors that are yielded/returned from a block and store them in yieldedTensors.
MutableArrayRef< OpOperand > getOpOperands()
LogicalResult success(bool isSuccess=true)
Utility function to generate a LogicalResult.
std::enable_if< llvm::function_traits< std::decay_t< FnT > >::num_args==1, RetT >::type walk(FnT &&callback)
Walk the operation by calling the callback for each nested operation (including this one)...
unsigned getOperandNumber()
Return which operand this is in the OpOperand list of the Operation.
AnalysisState provides a variety of helper functions for dealing with tensor values.
This class represents an efficient way to signal success or failure.
LogicalResult failure(bool isFailure=true)
Utility function to generate a LogicalResult.
void bufferizeOutOfPlace(OpOperand &operand)
Set the inPlace bufferization spec to false.
static void setInPlaceOpOperand(OpOperand &opOperand, bool inPlace)
Mark whether OpOperand will be bufferized inplace.
void applyOnEquivalenceClass(Value v, function_ref< void(Value)> fun) const
Apply fun to all the members of the equivalence class of v.
static bool hasReadAfterWriteInterference(const DenseSet< OpOperand *> &usesRead, const DenseSet< OpOperand *> &usesWrite, const DominanceInfo &domInfo, AnalysisState &state, const BufferizationAliasInfo &aliasInfo)
Given sets of uses and writes, return true if there is a RaW conflict under the assumption that all g...
static LogicalResult checkAliasInfoConsistency(Operation *op, const DominanceInfo &domInfo, AnalysisState &state, const BufferizationAliasInfo &aliasInfo)
Assert that the current bufferization decisions are consistent.
Operation * getParentOp()
Returns the closest surrounding operation that contains this operation or nullptr if this is a top-le...
BufferizableOpInterface dynCastBufferizableOp(Operation *op) const
Try to cast the given op to BufferizableOpInterface if the op is allow listed.
std::function< LogicalResult(Operation *, AnalysisState &, BufferizationAliasInfo &, SmallVector< Operation * > &)> PostAnalysisStepFn
PostAnalysisStepFns can be registered with BufferizationOptions and are executed after the analysis...
static WalkResult advance()
void bufferizeInPlace(OpOperand &operand, AnalysisState &state)
Set the inPlace bufferization spec to true.
IRValueT get() const
Return the current value being used by this operand.
static WalkResult interrupt()
static bool wouldCreateWriteToNonWritableBuffer(OpOperand &opOperand, const BufferizationAliasInfo &aliasInfo, AnalysisState &state)
Return true if bufferizing opOperand inplace would create a write to a non-writable buffer...
A utility result that is used to signal how to proceed with an ongoing walk:
This class represents an argument of a Block.
void applyOnAliases(Value v, function_ref< void(Value)> fun) const
Apply fun to all aliases of v.
Tensor types represent multi-dimensional arrays, and have two variants: RankedTensorType and Unranked...
void insertNewBufferEquivalence(Value newValue, Value alias)
Insert an info entry for newValue and merge its alias set with that of alias.
Options for BufferizableOpInterface-based bufferization.
result_range getOpResults()
Instances of the Type class are uniqued, have an immutable identifier and an optional mutable compone...
void markInPlace(OpOperand &o)
Mark a value as in-place bufferized.
BufferizationAliasInfo(Operation *rootOp)
This class represents an instance of an SSA value in the MLIR system, representing a computable value...
bool hasUndefinedContents(OpOperand *opOperand) const override
Return true if the given tensor has undefined contents.
bool printConflicts
If set to true, the IR is annotated with details about RaW conflicts.
static llvm::ManagedStatic< PassManagerOptions > options
LogicalResult runOneShotBufferize(Operation *op, const OneShotBufferizationOptions &options)
Run One-Shot Bufferize on the given op: Analysis + Bufferization.
void setAttr(StringAttr name, Attribute value)
If the an attribute exists with the specified name, change it to the new value.
Type getType() const
Return the type of this value.
State for analysis-enabled bufferization.
Operation * getOwner() const
Return the owner of this operand.
Operation * getDefiningOp() const
If this value is the result of an operation, return the operation that defines it.
static bool aliasesNonWritableBuffer(Value value, const BufferizationAliasInfo &aliasInfo, AnalysisState &state)
Return true if, under current bufferization decisions, the buffer of value is not writable...
void unionAliasSets(Value v1, Value v2)
Union the alias sets of v1 and v2.
This class represents an operand of an operation.
Region * getEnclosingRepetitiveRegion(Operation *op)
Return the first enclosing region of the given op that may be executed repetitively as per RegionBran...
static bool happensBefore(Operation *a, Operation *b, const DominanceInfo &domInfo)
Return true if a happens before b, i.e., a or one of its ancestors properly dominates b and b is not ...
constexpr StringLiteral kInPlaceResultsAttrName
Attribute marker to specify op results that can be bufferized inPlace.
bool insideMutuallyExclusiveRegions(Operation *a, Operation *b)
Return true if a and b are in mutually exclusive regions as per RegionBranchOpInterface.
BufferizationAliasInfo & getAliasInfo()
Return a reference to the BufferizationAliasInfo.
void gatherUndefinedTensorUses(Operation *op)
Find all tensor values in the given operation that have undefined contents and store them in undefine...
bool isOpAllowed(Operation *op) const
Return whether the op should be bufferized or not.
static bool isaTensor(Type t)
InFlightDiagnostic emitError(const Twine &message={})
Emit an error about fatal conditions with this operation, reporting up to any diagnostic handlers tha...
static void annotateConflict(OpOperand *uRead, OpOperand *uConflictingWrite, Value lastWrite)
Annotate IR with details about the detected RaW conflict.
bool isInPlace(OpOperand &opOperand) const override
Return true if the given OpResult has been decided to bufferize inplace.
bool bufferizesToMemoryRead(OpOperand &opOperand) const
Return true if opOperand bufferizes to a memory read.
result_range getResults()
This class helps build Operations.
static bool isMemoryWrite(Value value, const AnalysisState &state)
Return true if the given tensor value is a memory write.
use_range getUses() const
Returns a range of all uses, which is useful for iterating over all uses.
Attribute getAttr(StringAttr name)
Return the specified attribute if present, null otherwise.
static LogicalResult assertDestinationPassingStyle(Operation *op, AnalysisState &state, BufferizationAliasInfo &aliasInfo, SmallVector< Operation *> &newOps)
Assert that IR is in destination-passing style.
result_type_range getResultTypes()
bool isProperAncestor(Operation *other)
Return true if this operation is a proper ancestor of the other operation.
SetVector< Value > findLastPrecedingWrite(Value value) const
Find the Values of the last preceding write of a given Value.
static LogicalResult inPlaceAnalysis(SmallVector< Operation *> &ops, BufferizationAliasInfo &aliasInfo, AnalysisState &state, const DominanceInfo &domInfo, unsigned analysisFuzzerSeed=0)
Analyze the ops to determine which OpOperands are inplaceable.
bool isTensorYielded(Value tensor) const override
Return true if the given tensor (or an aliasing tensor) is yielded from the containing block...
Options for analysis-enabled bufferization.