32 namespace bufferization {
33 #define GEN_PASS_DEF_OWNERSHIPBASEDBUFFERDEALLOCATION
34 #include "mlir/Dialect/Bufferization/Transforms/Passes.h.inc"
54 if (isa<MemoryEffectOpInterface>(op))
55 return !hasEffect<MemoryEffects::Allocate>(op) &&
56 !hasEffect<MemoryEffects::Free>(op);
72 if (llvm::any_of(region.front().getArguments(),
isMemref))
95 size_t size()
const {
return edgeSet.size(); }
98 BackedgeSetT::const_iterator begin()
const {
return edgeSet.begin(); }
101 BackedgeSetT::const_iterator end()
const {
return edgeSet.end(); }
107 bool enter(
Block ¤t,
Block *predecessor) {
108 bool inserted = visited.insert(¤t).second;
110 edgeSet.insert(std::make_pair(predecessor, ¤t));
115 void exit(
Block ¤t) { visited.erase(¤t); }
123 if (isa<BranchOpInterface>(op)) {
125 recurse(*succ, current);
131 recurse(region.front(), current);
137 void recurse(
Block &block,
Block *predecessor) {
140 if (!enter(block, predecessor))
155 BackedgeSetT edgeSet;
167 class BufferDeallocation {
173 LogicalResult deallocate(FunctionOpInterface op);
177 template <
typename... T>
178 typename std::enable_if<
sizeof...(T) == 0, FailureOr<Operation *>>::type
211 template <
typename InterfaceT,
typename... InterfacesU>
212 FailureOr<Operation *> handleOp(
Operation *op) {
214 if (
auto concreteOp = dyn_cast<InterfaceT>(op)) {
215 FailureOr<Operation *> result = handleInterface(concreteOp);
221 return FailureOr<Operation *>(
nullptr);
222 return handleOp<InterfacesU...>(next);
226 FailureOr<Operation *> handleAllInterfaces(
Operation *op) {
227 if (
auto deallocOpInterface = dyn_cast<BufferDeallocationOpInterface>(op))
228 return deallocOpInterface.process(state,
options);
230 if (failed(verifyOperationPreconditions(op)))
233 return handleOp<MemoryEffectOpInterface, RegionBranchOpInterface,
234 CallOpInterface, BranchOpInterface,
235 RegionBranchTerminatorOpInterface>(op);
258 FailureOr<Operation *> handleInterface(BranchOpInterface op);
290 FailureOr<Operation *> handleInterface(RegionBranchOpInterface op);
319 FailureOr<Operation *> handleInterface(CallOpInterface op);
339 FailureOr<Operation *> handleInterface(MemoryEffectOpInterface op);
362 FailureOr<Operation *> handleInterface(RegionBranchTerminatorOpInterface op);
375 template <
typename OpTy>
377 return cast<OpTy>(appendOpResults(op.getOperation(), types));
383 LogicalResult deallocate(
Block *block);
396 void populateRemainingOwnerships(
Operation *op);
401 Value materializeMemrefWithGuaranteedOwnership(
OpBuilder &builder,
407 bool isFunctionWithoutDynamicOwnership(
Operation *op);
416 std::pair<Value, Value>
423 static LogicalResult verifyFunctionPreconditions(FunctionOpInterface op);
439 static LogicalResult verifyOperationPreconditions(
Operation *op);
446 static LogicalResult updateFunctionSignature(FunctionOpInterface op);
464 std::pair<Value, Value>
465 BufferDeallocation::materializeUniqueOwnership(
OpBuilder &builder,
Value memref,
470 return state.getMemrefWithUniqueOwnership(builder, memref, block);
478 if (
auto deallocOpInterface = dyn_cast<BufferDeallocationOpInterface>(owner))
479 return deallocOpInterface.materializeUniqueOwnershipForMemref(
480 state,
options, builder, memref);
483 return state.getMemrefWithUniqueOwnership(builder, memref, block);
487 BufferDeallocation::verifyFunctionPreconditions(FunctionOpInterface op) {
490 Backedges backedges(op);
491 if (backedges.size()) {
492 op->
emitError(
"Only structured control-flow loops are supported.");
499 LogicalResult BufferDeallocation::verifyOperationPreconditions(
Operation *op) {
507 if (isa<DeallocOp>(op))
509 "No deallocation operations must be present when running this pass!");
519 if (!isa<MemoryEffectOpInterface>(op) &&
521 !isa<CallOpInterface>(op))
523 "ops with unknown memory side effects are not supported");
533 size_t size = regions.size();
534 if (((size == 1 && !op->
getResults().empty()) || size > 1) &&
535 !dyn_cast<RegionBranchOpInterface>(op)) {
536 return op->
emitError(
"All operations with attached regions need to "
537 "implement the RegionBranchOpInterface.");
544 return op->
emitError(
"NoTerminator trait is not supported");
549 if (!isa<BranchOpInterface, RegionBranchTerminatorOpInterface>(op) ||
550 (isa<BranchOpInterface>(op) &&
551 isa<RegionBranchTerminatorOpInterface>(op)))
554 "Terminators must implement either BranchOpInterface or "
555 "RegionBranchTerminatorOpInterface (but not both)!");
561 return op->
emitError(
"Terminators with more than one successor "
562 "are not supported!");
569 BufferDeallocation::updateFunctionSignature(FunctionOpInterface op) {
571 op.getFunctionBody().getOps<RegionBranchTerminatorOpInterface>(),
572 [](RegionBranchTerminatorOpInterface op) {
573 return op.getSuccessorOperands(RegionBranchPoint::parent()).getTypes();
575 if (!llvm::all_equal(returnOperandTypes))
577 "there are multiple return operations with different operand types");
584 if (!returnOperandTypes.empty())
585 resultTypes = returnOperandTypes[0];
588 op->
getContext(), op.getFunctionBody().front().getArgumentTypes(),
594 LogicalResult BufferDeallocation::deallocate(FunctionOpInterface op) {
596 if (failed(verifyFunctionPreconditions(op)))
602 if (failed(deallocate(block)))
606 if (result.wasInterrupted())
611 return updateFunctionSignature(op);
614 LogicalResult BufferDeallocation::deallocate(
Block *block) {
619 state.getLiveMemrefsIn(block, liveMemrefs);
620 for (
auto li : liveMemrefs) {
624 if (li.getParentRegion() == block->
getParent()) {
625 state.updateOwnership(li, state.getOwnership(li, li.getParentBlock()),
627 state.addMemrefToDeallocate(li, block);
631 if (li.getParentRegion()->isProperAncestor(block->
getParent())) {
633 state.updateOwnership(li, falseVal, block);
644 if (isa<FunctionOpInterface>(block->
getParentOp()) &&
647 state.updateOwnership(arg, newArg);
648 state.addMemrefToDeallocate(arg, block);
654 state.updateOwnership(arg, newArg);
655 state.addMemrefToDeallocate(arg, block);
660 for (
Operation &op : llvm::make_early_inc_range(*block)) {
661 FailureOr<Operation *> result = handleAllInterfaces(&op);
667 populateRemainingOwnerships(*result);
677 newTypes.append(types.begin(), types.end());
682 for (
auto [oldRegion, newRegion] :
683 llvm::zip(op->
getRegions(), newOp->getRegions()))
684 newRegion.takeBody(oldRegion);
693 FailureOr<Operation *>
694 BufferDeallocation::handleInterface(RegionBranchOpInterface op) {
712 assert(!regions.empty() &&
"Must have at least one successor region");
714 op.getEntrySuccessorOperands(regions.front()));
715 unsigned numMemrefOperands = llvm::count_if(entryOperands,
isMemref);
726 RegionBranchOpInterface newOp = appendOpResults(op, ownershipResults);
728 for (
auto result : llvm::make_filter_range(newOp->getResults(),
isMemref)) {
729 state.updateOwnership(result, newOp->getResult(counter++));
730 state.addMemrefToDeallocate(result, newOp->getBlock());
733 return newOp.getOperation();
736 Value BufferDeallocation::materializeMemrefWithGuaranteedOwnership(
739 std::pair<Value, Value> newMemrefAndOnwership =
740 materializeUniqueOwnership(builder, memref, block);
741 Value newMemref = newMemrefAndOnwership.first;
742 Value condition = newMemrefAndOnwership.second;
755 memref.
getLoc(), condition,
757 builder.create<scf::YieldOp>(loc, newMemref);
761 builder.
create<bufferization::CloneOp>(loc, newMemref);
766 state.updateOwnership(maybeClone, trueVal);
767 state.addMemrefToDeallocate(maybeClone, maybeClone.
getParentBlock());
771 FailureOr<Operation *>
772 BufferDeallocation::handleInterface(BranchOpInterface op) {
774 return op->
emitError(
"BranchOpInterface operations with multiple "
775 "successors are not supported yet");
779 "only BranchOpInterface operations with exactly "
780 "one successor are supported yet");
782 if (op.getSuccessorOperands(0).getProducedOperandCount() > 0)
783 return op.
emitError(
"produced operands are not supported");
790 if (failed(state.getMemrefsAndConditionsToDeallocate(
791 builder, op.
getLoc(), block, memrefs, conditions)))
795 op.getSuccessorOperands(0).getForwardedOperands();
796 state.getMemrefsToRetain(block, op->
getSuccessor(0), forwardedOperands,
799 auto deallocOp = builder.create<bufferization::DeallocOp>(
800 op.
getLoc(), memrefs, conditions, toRetain);
804 state.resetOwnerships(deallocOp.getRetained(), block);
805 for (
auto [retained, ownership] :
806 llvm::zip(deallocOp.getRetained(), deallocOp.getUpdatedConditions())) {
807 state.updateOwnership(retained, ownership, block);
810 unsigned numAdditionalReturns = llvm::count_if(forwardedOperands,
isMemref);
812 auto additionalConditions =
813 deallocOp.getUpdatedConditions().take_front(numAdditionalReturns);
814 newOperands.append(additionalConditions.begin(), additionalConditions.end());
815 op.getSuccessorOperands(0).getMutableForwardedOperands().assign(newOperands);
817 return op.getOperation();
820 FailureOr<Operation *> BufferDeallocation::handleInterface(CallOpInterface op) {
827 Operation *funcOp = op.resolveCallableInTable(state.getSymbolTable());
828 bool isPrivate =
false;
829 if (
auto symbol = dyn_cast_or_null<SymbolOpInterface>(funcOp))
830 isPrivate = symbol.isPrivate() && !symbol.isDeclaration();
836 if (
options.privateFuncDynamicOwnership && isPrivate) {
840 op = appendOpResults(op, ownershipTypesToAppend);
843 state.updateOwnership(result, op->
getResult(ownershipCounter++));
844 state.addMemrefToDeallocate(result, result.getParentBlock());
847 return op.getOperation();
855 state.updateOwnership(result, trueVal);
856 state.addMemrefToDeallocate(result, result.getParentBlock());
859 return op.getOperation();
862 FailureOr<Operation *>
863 BufferDeallocation::handleInterface(MemoryEffectOpInterface op) {
875 if (!op->
hasAttr(BufferizationDialect::kManualDeallocation))
877 "memory free side-effect on MemRef value not supported!");
885 Ownership ownership = state.getOwnership(operand, block);
887 Value ownershipInverted = builder.
create<arith::XOrIOp>(
890 builder.
create<cf::AssertOp>(
891 op.
getLoc(), ownershipInverted,
892 "expected that the block does not have ownership");
899 if (allocEffect.has_value()) {
900 if (isa<SideEffects::AutomaticAllocationScopeResource>(
901 allocEffect->getResource())) {
909 state.resetOwnerships(res, block);
914 if (op->
hasAttr(BufferizationDialect::kManualDeallocation)) {
918 state.resetOwnerships(res, block);
924 state.addMemrefToDeallocate(res, block);
928 return op.getOperation();
931 FailureOr<Operation *>
932 BufferDeallocation::handleInterface(RegionBranchTerminatorOpInterface op) {
939 bool funcWithoutDynamicOwnership =
940 isFunctionWithoutDynamicOwnership(op->
getParentOp());
941 if (funcWithoutDynamicOwnership) {
946 val.set(materializeMemrefWithGuaranteedOwnership(builder, val.get(),
961 if (failed(result) || !*result)
965 if (!funcWithoutDynamicOwnership) {
967 newOperands.append(updatedOwnerships.begin(), updatedOwnerships.end());
968 operands.
assign(newOperands);
971 return op.getOperation();
974 bool BufferDeallocation::isFunctionWithoutDynamicOwnership(
Operation *op) {
975 auto funcOp = dyn_cast<FunctionOpInterface>(op);
976 return funcOp && (!
options.privateFuncDynamicOwnership ||
977 !funcOp.isPrivate() || funcOp.isExternal());
980 void BufferDeallocation::populateRemainingOwnerships(
Operation *op) {
984 if (!state.getOwnership(res, op->
getBlock()).isUninitialized())
994 state.updateOwnership(
995 res, state.getOwnership(operand, operand.getParentBlock()),
1005 if (state.getOwnership(res, op->
getBlock()).isUninitialized()) {
1021 struct OwnershipBasedBufferDeallocationPass
1022 :
public bufferization::impl::OwnershipBasedBufferDeallocationBase<
1023 OwnershipBasedBufferDeallocationPass> {
1024 OwnershipBasedBufferDeallocationPass() =
default;
1026 : OwnershipBasedBufferDeallocationPass() {
1027 this->privateFuncDynamicOwnership.setValue(
1028 options.privateFuncDynamicOwnership);
1030 void runOnOperation()
override {
1032 options.privateFuncDynamicOwnership = privateFuncDynamicOwnership;
1034 auto status = getOperation()->walk([&](func::FuncOp func) {
1035 if (func.isExternal())
1043 if (status.wasInterrupted())
1044 signalPassFailure();
1058 BufferDeallocation deallocation(op,
options);
1061 return deallocation.deallocate(op);
1068 std::unique_ptr<Pass>
1071 return std::make_unique<OwnershipBasedBufferDeallocationPass>(
options);
static bool isMemref(Value v)
static Value buildBoolValue(OpBuilder &builder, Location loc, bool value)
static bool hasBufferSemantics(Operation *op)
Return "true" if the given op has buffer semantics.
static bool hasNeitherAllocateNorFreeSideEffect(Operation *op)
Return "true" if the given op is guaranteed to have neither "Allocate" nor "Free" side effects.
static llvm::ManagedStatic< PassManagerOptions > options
This class represents an argument of a Block.
Location getLoc() const
Return the location for this argument.
Block represents an ordered list of Operations.
BlockArgument getArgument(unsigned i)
unsigned getNumArguments()
Region * getParent() const
Provide a 'getParent' method for ilist_node_with_parent methods.
SuccessorRange getSuccessors()
BlockArgument addArgument(Type type, Location loc)
Add one value to the argument list.
OpListType & getOperations()
bool isEntryBlock()
Return if this block is the entry block in the parent region.
Operation * getParentOp()
Returns the closest surrounding operation that contains this block.
BoolAttr getBoolAttr(bool value)
This class defines the main interface for locations in MLIR and acts as a non-nullable wrapper around...
This class provides a mutable adaptor for a range of operands.
OperandRange getAsOperandRange() const
Explicit conversion to an OperandRange.
void assign(ValueRange values)
Assign this range to the given values.
RAII guard to reset the insertion point of the builder when destroyed.
This class helps build Operations.
static OpBuilder atBlockBegin(Block *block, Listener *listener=nullptr)
Create a builder and set the insertion point to before the first operation in the block but still ins...
void setInsertionPoint(Block *block, Block::iterator insertPoint)
Set the insertion point to the specified location.
Operation * create(const OperationState &state)
Creates an operation given the fields represented as an OperationState.
Operation * insert(Operation *op)
Insert the given operation at the current insertion point and return it.
This class represents an operand of an operation.
This trait indicates that the memory effects of an operation includes the effects of operations neste...
This class provides the API for ops that are known to be terminators.
This class indicates that the regions associated with this op don't have terminators.
This class implements the operand iterators for the Operation class.
Operation is the basic unit of execution within MLIR.
DictionaryAttr getAttrDictionary()
Return all of the attributes on this operation as a DictionaryAttr.
bool hasTrait()
Returns true if the operation was registered with a particular trait, e.g.
void insertOperands(unsigned index, ValueRange operands)
Insert the given operands into the operand list at the given 'index'.
Block * getSuccessor(unsigned index)
bool hasAttr(StringAttr name)
Return true if the operation has an attribute with the provided name, false otherwise.
unsigned getNumSuccessors()
OpResult getResult(unsigned idx)
Get the 'idx'th result of this operation.
std::enable_if_t< llvm::function_traits< std::decay_t< FnT > >::num_args==1, RetT > walk(FnT &&callback)
Walk the operation by calling the callback for each nested operation (including this one),...
MLIRContext * getContext()
Return the context this operation is associated with.
unsigned getNumRegions()
Returns the number of regions held by this operation.
Location getLoc()
The source location the operation was defined or derived from.
unsigned getNumOperands()
static Operation * create(Location location, OperationName name, TypeRange resultTypes, ValueRange operands, NamedAttrList &&attributes, OpaqueProperties properties, BlockRange successors, unsigned numRegions)
Create a new Operation with the specific fields.
Operation * getParentOp()
Returns the closest surrounding operation that contains this operation or nullptr if this is a top-le...
InFlightDiagnostic emitError(const Twine &message={})
Emit an error about fatal conditions with this operation, reporting up to any diagnostic handlers tha...
Block * getBlock()
Returns the operation block that contains this operation.
MutableArrayRef< Region > getRegions()
Returns the regions held by this operation.
OperationName getName()
The name of an operation is the key identifier for it.
MutableArrayRef< OpOperand > getOpOperands()
result_type_range getResultTypes()
operand_range getOperands()
Returns an iterator on the underlying Value's.
void replaceAllUsesWith(ValuesT &&values)
Replace all uses of results of this operation with the provided 'values'.
SuccessorRange getSuccessors()
result_range getResults()
OpaqueProperties getPropertiesStorage()
Returns the properties storage.
void erase()
Remove this operation from its parent block and delete it.
unsigned getNumResults()
Return the number of results held by this operation.
static constexpr RegionBranchPoint parent()
Returns an instance of RegionBranchPoint representing the parent operation.
This class contains a list of basic blocks and a link to the parent operation it is attached to.
This class provides an abstraction over the various different ranges of value types.
This class represents an instance of an SSA value in the MLIR system, representing a computable value...
Type getType() const
Return the type of this value.
Block * getParentBlock()
Return the Block in which this Value is defined.
Location getLoc() const
Return the location of this value.
Operation * getDefiningOp() const
If this value is the result of an operation, return the operation that defines it.
static WalkResult advance()
static WalkResult interrupt()
This class collects all the state that we need to perform the buffer deallocation pass with associate...
This class is used to track the ownership of values.
bool isUnique() const
Check if this ownership value is in the 'Unique' state.
Value getIndicator() const
If this ownership value is in 'Unique' state, this function can be used to get the indicator paramete...
FailureOr< Operation * > insertDeallocOpForReturnLike(DeallocationState &state, Operation *op, ValueRange operands, SmallVectorImpl< Value > &updatedOperandOwnerships)
Insert a bufferization.dealloc operation right before op which has to be a terminator without any suc...
std::unique_ptr< Pass > createOwnershipBasedBufferDeallocationPass(DeallocationOptions options=DeallocationOptions())
Creates an instance of the OwnershipBasedBufferDeallocation pass to free all allocated buffers.
LogicalResult deallocateBuffersOwnershipBased(FunctionOpInterface op, DeallocationOptions options)
Run the ownership-based buffer deallocation.
Include the generated interface declarations.
bool matchPattern(Value value, const Pattern &pattern)
Entry point for matching a pattern over a Value.
InFlightDiagnostic emitError(Location loc)
Utility method to emit an error message using this location.
detail::constant_int_predicate_matcher m_One()
Matches a constant scalar / vector splat / tensor splat integer one.
Operation * clone(OpBuilder &b, Operation *op, TypeRange newResultTypes, ValueRange newOperands)
auto get(MLIRContext *context, Ts &&...params)
Helper method that injects context only if needed, this helps unify some of the attribute constructio...
This iterator enumerates elements according to their dominance relationship.
The following effect indicates that the operation allocates from some resource.
The following effect indicates that the operation frees some resource that has been allocated.
Options for BufferDeallocationOpInterface-based buffer deallocation.