32 namespace bufferization {
33 #define GEN_PASS_DEF_OWNERSHIPBASEDBUFFERDEALLOCATION
34 #include "mlir/Dialect/Bufferization/Transforms/Passes.h.inc"
54 if (isa<MemoryEffectOpInterface>(op))
55 return hasEffect<MemoryEffects::Allocate>(op) ||
56 hasEffect<MemoryEffects::Free>(op);
72 if (llvm::any_of(region.front().getArguments(),
isMemref))
95 size_t size()
const {
return edgeSet.size(); }
98 BackedgeSetT::const_iterator begin()
const {
return edgeSet.begin(); }
101 BackedgeSetT::const_iterator end()
const {
return edgeSet.end(); }
107 bool enter(
Block ¤t,
Block *predecessor) {
108 bool inserted = visited.insert(¤t).second;
110 edgeSet.insert(std::make_pair(predecessor, ¤t));
115 void exit(
Block ¤t) { visited.erase(¤t); }
123 if (isa<BranchOpInterface>(op)) {
125 recurse(*succ, current);
131 recurse(region.front(), current);
137 void recurse(
Block &block,
Block *predecessor) {
140 if (!enter(block, predecessor))
155 BackedgeSetT edgeSet;
167 class BufferDeallocation {
177 template <
typename... T>
211 template <
typename InterfaceT,
typename... InterfacesU>
214 if (
auto concreteOp = dyn_cast<InterfaceT>(op)) {
222 return handleOp<InterfacesU...>(next);
227 if (
auto deallocOpInterface = dyn_cast<BufferDeallocationOpInterface>(op))
228 return deallocOpInterface.process(state,
options);
230 if (
failed(verifyOperationPreconditions(op)))
233 return handleOp<MemoryEffectOpInterface, RegionBranchOpInterface,
234 CallOpInterface, BranchOpInterface,
235 RegionBranchTerminatorOpInterface>(op);
375 template <
typename OpTy>
377 return cast<OpTy>(appendOpResults(op.getOperation(), types));
396 void populateRemainingOwnerships(
Operation *op);
401 Value materializeMemrefWithGuaranteedOwnership(
OpBuilder &builder,
407 bool isFunctionWithoutDynamicOwnership(
Operation *op);
416 std::pair<Value, Value>
423 static LogicalResult verifyFunctionPreconditions(FunctionOpInterface op);
446 static LogicalResult updateFunctionSignature(FunctionOpInterface op);
464 std::pair<Value, Value>
465 BufferDeallocation::materializeUniqueOwnership(
OpBuilder &builder,
Value memref,
470 return state.getMemrefWithUniqueOwnership(builder, memref, block);
478 if (
auto deallocOpInterface = dyn_cast<BufferDeallocationOpInterface>(owner))
479 return deallocOpInterface.materializeUniqueOwnershipForMemref(
480 state,
options, builder, memref);
483 return state.getMemrefWithUniqueOwnership(builder, memref, block);
487 BufferDeallocation::verifyFunctionPreconditions(FunctionOpInterface op) {
490 Backedges backedges(op);
491 if (backedges.size()) {
492 op->
emitError(
"Only structured control-flow loops are supported.");
502 if (isa<DeallocOp>(op))
504 "No deallocation operations must be present when running this pass!");
514 if (!isa<MemoryEffectOpInterface>(op) &&
516 !isa<CallOpInterface>(op))
518 "ops with unknown memory side effects are not supported");
533 size_t size = regions.size();
534 if (((size == 1 && !op->
getResults().empty()) || size > 1) &&
535 !dyn_cast<RegionBranchOpInterface>(op)) {
536 return op->
emitError(
"All operations with attached regions need to "
537 "implement the RegionBranchOpInterface.");
544 return op->
emitError(
"NoTerminator trait is not supported");
549 if (!isa<BranchOpInterface, RegionBranchTerminatorOpInterface>(op) ||
550 (isa<BranchOpInterface>(op) &&
551 isa<RegionBranchTerminatorOpInterface>(op)))
554 "Terminators must implement either BranchOpInterface or "
555 "RegionBranchTerminatorOpInterface (but not both)!");
561 return op->
emitError(
"Terminators with more than one successor "
562 "are not supported!");
569 BufferDeallocation::updateFunctionSignature(FunctionOpInterface op) {
571 op.getFunctionBody().getOps<RegionBranchTerminatorOpInterface>(),
572 [](RegionBranchTerminatorOpInterface op) {
573 return op.getSuccessorOperands(RegionBranchPoint::parent()).getTypes();
575 if (!llvm::all_equal(returnOperandTypes))
577 "there are multiple return operations with different operand types");
584 if (!returnOperandTypes.empty())
585 resultTypes = returnOperandTypes[0];
588 op->
getContext(), op.getFunctionBody().front().getArgumentTypes(),
594 LogicalResult BufferDeallocation::deallocate(FunctionOpInterface op) {
596 if (
failed(verifyFunctionPreconditions(op)))
602 if (
failed(deallocate(block)))
606 if (result.wasInterrupted())
611 return updateFunctionSignature(op);
619 state.getLiveMemrefsIn(block, liveMemrefs);
620 for (
auto li : liveMemrefs) {
624 if (li.getParentRegion() == block->
getParent()) {
625 state.updateOwnership(li, state.getOwnership(li, li.getParentBlock()),
627 state.addMemrefToDeallocate(li, block);
631 if (li.getParentRegion()->isProperAncestor(block->
getParent())) {
633 state.updateOwnership(li, falseVal, block);
644 if (isa<FunctionOpInterface>(block->
getParentOp()) &&
647 state.updateOwnership(arg, newArg);
648 state.addMemrefToDeallocate(arg, block);
654 state.updateOwnership(arg, newArg);
655 state.addMemrefToDeallocate(arg, block);
660 for (
Operation &op : llvm::make_early_inc_range(*block)) {
667 populateRemainingOwnerships(*result);
677 newTypes.append(types.begin(), types.end());
682 for (
auto [oldRegion, newRegion] :
683 llvm::zip(op->
getRegions(), newOp->getRegions()))
684 newRegion.takeBody(oldRegion);
694 BufferDeallocation::handleInterface(RegionBranchOpInterface op) {
712 assert(!regions.empty() &&
"Must have at least one successor region");
714 op.getEntrySuccessorOperands(regions.front()));
715 unsigned numMemrefOperands = llvm::count_if(entryOperands,
isMemref);
726 RegionBranchOpInterface newOp = appendOpResults(op, ownershipResults);
728 for (
auto result : llvm::make_filter_range(newOp->getResults(),
isMemref)) {
729 state.updateOwnership(result, newOp->getResult(counter++));
730 state.addMemrefToDeallocate(result, newOp->getBlock());
733 return newOp.getOperation();
736 Value BufferDeallocation::materializeMemrefWithGuaranteedOwnership(
739 std::pair<Value, Value> newMemrefAndOnwership =
740 materializeUniqueOwnership(builder, memref, block);
741 Value newMemref = newMemrefAndOnwership.first;
742 Value condition = newMemrefAndOnwership.second;
755 memref.
getLoc(), condition,
757 builder.create<scf::YieldOp>(loc, newMemref);
761 builder.
create<bufferization::CloneOp>(loc, newMemref);
766 state.updateOwnership(maybeClone, trueVal);
767 state.addMemrefToDeallocate(maybeClone, maybeClone.
getParentBlock());
772 BufferDeallocation::handleInterface(BranchOpInterface op) {
774 return op->
emitError(
"BranchOpInterface operations with multiple "
775 "successors are not supported yet");
779 "only BranchOpInterface operations with exactly "
780 "one successor are supported yet");
782 if (op.getSuccessorOperands(0).getProducedOperandCount() > 0)
783 return op.
emitError(
"produced operands are not supported");
790 if (
failed(state.getMemrefsAndConditionsToDeallocate(
791 builder, op.
getLoc(), block, memrefs, conditions)))
795 op.getSuccessorOperands(0).getForwardedOperands();
796 state.getMemrefsToRetain(block, op->
getSuccessor(0), forwardedOperands,
799 auto deallocOp = builder.create<bufferization::DeallocOp>(
800 op.
getLoc(), memrefs, conditions, toRetain);
804 state.resetOwnerships(deallocOp.getRetained(), block);
805 for (
auto [retained, ownership] :
806 llvm::zip(deallocOp.getRetained(), deallocOp.getUpdatedConditions())) {
807 state.updateOwnership(retained, ownership, block);
810 unsigned numAdditionalReturns = llvm::count_if(forwardedOperands,
isMemref);
812 auto additionalConditions =
813 deallocOp.getUpdatedConditions().take_front(numAdditionalReturns);
814 newOperands.append(additionalConditions.begin(), additionalConditions.end());
815 op.getSuccessorOperands(0).getMutableForwardedOperands().assign(newOperands);
817 return op.getOperation();
826 Operation *funcOp = op.resolveCallable(state.getSymbolTable());
827 bool isPrivate =
true;
828 if (
auto symbol = dyn_cast<SymbolOpInterface>(funcOp))
829 isPrivate = symbol.isPrivate() && !symbol.isDeclaration();
835 if (
options.privateFuncDynamicOwnership && isPrivate) {
839 op = appendOpResults(op, ownershipTypesToAppend);
842 state.updateOwnership(result, op->
getResult(ownershipCounter++));
843 state.addMemrefToDeallocate(result, result.getParentBlock());
846 return op.getOperation();
854 state.updateOwnership(result, trueVal);
855 state.addMemrefToDeallocate(result, result.getParentBlock());
858 return op.getOperation();
862 BufferDeallocation::handleInterface(MemoryEffectOpInterface op) {
874 if (!op->
hasAttr(BufferizationDialect::kManualDeallocation))
876 "memory free side-effect on MemRef value not supported!");
884 Ownership ownership = state.getOwnership(operand, block);
886 Value ownershipInverted = builder.
create<arith::XOrIOp>(
889 builder.
create<cf::AssertOp>(
890 op.
getLoc(), ownershipInverted,
891 "expected that the block does not have ownership");
898 if (allocEffect.has_value()) {
899 if (isa<SideEffects::AutomaticAllocationScopeResource>(
900 allocEffect->getResource())) {
908 state.resetOwnerships(res, block);
913 if (op->
hasAttr(BufferizationDialect::kManualDeallocation)) {
917 state.resetOwnerships(res, block);
923 state.addMemrefToDeallocate(res, block);
927 return op.getOperation();
931 BufferDeallocation::handleInterface(RegionBranchTerminatorOpInterface op) {
938 bool funcWithoutDynamicOwnership =
939 isFunctionWithoutDynamicOwnership(op->
getParentOp());
940 if (funcWithoutDynamicOwnership) {
945 val.set(materializeMemrefWithGuaranteedOwnership(builder, val.get(),
960 if (
failed(result) || !*result)
964 if (!funcWithoutDynamicOwnership) {
966 newOperands.append(updatedOwnerships.begin(), updatedOwnerships.end());
967 operands.
assign(newOperands);
970 return op.getOperation();
973 bool BufferDeallocation::isFunctionWithoutDynamicOwnership(
Operation *op) {
974 auto funcOp = dyn_cast<FunctionOpInterface>(op);
975 return funcOp && (!
options.privateFuncDynamicOwnership ||
976 !funcOp.isPrivate() || funcOp.isExternal());
979 void BufferDeallocation::populateRemainingOwnerships(
Operation *op) {
983 if (!state.getOwnership(res, op->
getBlock()).isUninitialized())
993 state.updateOwnership(
994 res, state.getOwnership(operand, operand.getParentBlock()),
1004 if (state.getOwnership(res, op->
getBlock()).isUninitialized()) {
1020 struct OwnershipBasedBufferDeallocationPass
1021 :
public bufferization::impl::OwnershipBasedBufferDeallocationBase<
1022 OwnershipBasedBufferDeallocationPass> {
1023 OwnershipBasedBufferDeallocationPass() =
default;
1025 : OwnershipBasedBufferDeallocationPass() {
1026 this->privateFuncDynamicOwnership.setValue(
1027 options.privateFuncDynamicOwnership);
1029 void runOnOperation()
override {
1031 options.privateFuncDynamicOwnership = privateFuncDynamicOwnership;
1033 auto status = getOperation()->walk([&](func::FuncOp func) {
1034 if (func.isExternal())
1042 if (status.wasInterrupted())
1043 signalPassFailure();
1057 BufferDeallocation deallocation(op,
options);
1060 return deallocation.deallocate(op);
1067 std::unique_ptr<Pass>
1070 return std::make_unique<OwnershipBasedBufferDeallocationPass>(
options);
static bool isMemref(Value v)
static Value buildBoolValue(OpBuilder &builder, Location loc, bool value)
static bool hasBufferSemantics(Operation *op)
Return "true" if the given op has buffer semantics.
static bool hasNeitherAllocateNorFreeSideEffect(Operation *op)
Return "true" if the given op is guaranteed to have neither "Allocate" nor "Free" side effects.
static llvm::ManagedStatic< PassManagerOptions > options
This class represents an argument of a Block.
Location getLoc() const
Return the location for this argument.
Block represents an ordered list of Operations.
BlockArgument getArgument(unsigned i)
unsigned getNumArguments()
Region * getParent() const
Provide a 'getParent' method for ilist_node_with_parent methods.
SuccessorRange getSuccessors()
BlockArgument addArgument(Type type, Location loc)
Add one value to the argument list.
OpListType & getOperations()
bool isEntryBlock()
Return if this block is the entry block in the parent region.
Operation * getParentOp()
Returns the closest surrounding operation that contains this block.
BoolAttr getBoolAttr(bool value)
This class provides support for representing a failure result, or a valid value of type T.
This class defines the main interface for locations in MLIR and acts as a non-nullable wrapper around...
This class provides a mutable adaptor for a range of operands.
OperandRange getAsOperandRange() const
Explicit conversion to an OperandRange.
void assign(ValueRange values)
Assign this range to the given values.
RAII guard to reset the insertion point of the builder when destroyed.
This class helps build Operations.
static OpBuilder atBlockBegin(Block *block, Listener *listener=nullptr)
Create a builder and set the insertion point to before the first operation in the block but still ins...
void setInsertionPoint(Block *block, Block::iterator insertPoint)
Set the insertion point to the specified location.
Operation * create(const OperationState &state)
Creates an operation given the fields represented as an OperationState.
Operation * insert(Operation *op)
Insert the given operation at the current insertion point and return it.
This class represents an operand of an operation.
This trait indicates that the memory effects of an operation includes the effects of operations neste...
This class provides the API for ops that are known to be terminators.
This class indicates that the regions associated with this op don't have terminators.
This class implements the operand iterators for the Operation class.
Operation is the basic unit of execution within MLIR.
DictionaryAttr getAttrDictionary()
Return all of the attributes on this operation as a DictionaryAttr.
bool hasTrait()
Returns true if the operation was registered with a particular trait, e.g.
void insertOperands(unsigned index, ValueRange operands)
Insert the given operands into the operand list at the given 'index'.
Block * getSuccessor(unsigned index)
bool hasAttr(StringAttr name)
Return true if the operation has an attribute with the provided name, false otherwise.
unsigned getNumSuccessors()
OpResult getResult(unsigned idx)
Get the 'idx'th result of this operation.
std::enable_if_t< llvm::function_traits< std::decay_t< FnT > >::num_args==1, RetT > walk(FnT &&callback)
Walk the operation by calling the callback for each nested operation (including this one),...
MLIRContext * getContext()
Return the context this operation is associated with.
unsigned getNumRegions()
Returns the number of regions held by this operation.
Location getLoc()
The source location the operation was defined or derived from.
unsigned getNumOperands()
static Operation * create(Location location, OperationName name, TypeRange resultTypes, ValueRange operands, NamedAttrList &&attributes, OpaqueProperties properties, BlockRange successors, unsigned numRegions)
Create a new Operation with the specific fields.
Operation * getParentOp()
Returns the closest surrounding operation that contains this operation or nullptr if this is a top-le...
InFlightDiagnostic emitError(const Twine &message={})
Emit an error about fatal conditions with this operation, reporting up to any diagnostic handlers tha...
Block * getBlock()
Returns the operation block that contains this operation.
MutableArrayRef< Region > getRegions()
Returns the regions held by this operation.
OperationName getName()
The name of an operation is the key identifier for it.
MutableArrayRef< OpOperand > getOpOperands()
result_type_range getResultTypes()
operand_range getOperands()
Returns an iterator on the underlying Value's.
void replaceAllUsesWith(ValuesT &&values)
Replace all uses of results of this operation with the provided 'values'.
SuccessorRange getSuccessors()
result_range getResults()
OpaqueProperties getPropertiesStorage()
Returns the properties storage.
void erase()
Remove this operation from its parent block and delete it.
unsigned getNumResults()
Return the number of results held by this operation.
static constexpr RegionBranchPoint parent()
Returns an instance of RegionBranchPoint representing the parent operation.
This class contains a list of basic blocks and a link to the parent operation it is attached to.
This class provides an abstraction over the various different ranges of value types.
This class represents an instance of an SSA value in the MLIR system, representing a computable value...
Type getType() const
Return the type of this value.
Block * getParentBlock()
Return the Block in which this Value is defined.
Location getLoc() const
Return the location of this value.
Operation * getDefiningOp() const
If this value is the result of an operation, return the operation that defines it.
static WalkResult advance()
static WalkResult interrupt()
This class collects all the state that we need to perform the buffer deallocation pass with associate...
This class is used to track the ownership of values.
bool isUnique() const
Check if this ownership value is in the 'Unique' state.
Value getIndicator() const
If this ownership value is in 'Unique' state, this function can be used to get the indicator paramete...
FailureOr< Operation * > insertDeallocOpForReturnLike(DeallocationState &state, Operation *op, ValueRange operands, SmallVectorImpl< Value > &updatedOperandOwnerships)
Insert a bufferization.dealloc operation right before op which has to be a terminator without any suc...
std::unique_ptr< Pass > createOwnershipBasedBufferDeallocationPass(DeallocationOptions options=DeallocationOptions())
Creates an instance of the OwnershipBasedBufferDeallocation pass to free all allocated buffers.
LogicalResult deallocateBuffersOwnershipBased(FunctionOpInterface op, DeallocationOptions options)
Run the ownership-based buffer deallocation.
Include the generated interface declarations.
bool matchPattern(Value value, const Pattern &pattern)
Entry point for matching a pattern over a Value.
LogicalResult failure(bool isFailure=true)
Utility function to generate a LogicalResult.
InFlightDiagnostic emitError(Location loc)
Utility method to emit an error message using this location.
LogicalResult success(bool isSuccess=true)
Utility function to generate a LogicalResult.
detail::constant_int_predicate_matcher m_One()
Matches a constant scalar / vector splat / tensor splat integer one.
Operation * clone(OpBuilder &b, Operation *op, TypeRange newResultTypes, ValueRange newOperands)
auto get(MLIRContext *context, Ts &&...params)
Helper method that injects context only if needed, this helps unify some of the attribute constructio...
bool failed(LogicalResult result)
Utility function that returns true if the provided LogicalResult corresponds to a failure value.
This iterator enumerates elements according to their dominance relationship.
This class represents an efficient way to signal success or failure.
The following effect indicates that the operation allocates from some resource.
The following effect indicates that the operation frees some resource that has been allocated.
Options for BufferDeallocationOpInterface-based buffer deallocation.