32 namespace bufferization {
33 #define GEN_PASS_DEF_OWNERSHIPBASEDBUFFERDEALLOCATIONPASS
34 #include "mlir/Dialect/Bufferization/Transforms/Passes.h.inc"
54 if (isa<MemoryEffectOpInterface>(op))
55 return !hasEffect<MemoryEffects::Allocate>(op) &&
56 !hasEffect<MemoryEffects::Free>(op);
72 if (llvm::any_of(region.front().getArguments(),
isMemref))
95 size_t size()
const {
return edgeSet.size(); }
98 BackedgeSetT::const_iterator begin()
const {
return edgeSet.begin(); }
101 BackedgeSetT::const_iterator end()
const {
return edgeSet.end(); }
107 bool enter(
Block ¤t,
Block *predecessor) {
108 bool inserted = visited.insert(¤t).second;
110 edgeSet.insert(std::make_pair(predecessor, ¤t));
115 void exit(
Block ¤t) { visited.erase(¤t); }
123 if (isa<BranchOpInterface>(op)) {
125 recurse(*succ, current);
131 recurse(region.front(), current);
137 void recurse(
Block &block,
Block *predecessor) {
140 if (!enter(block, predecessor))
155 BackedgeSetT edgeSet;
167 class BufferDeallocation {
174 LogicalResult deallocate(FunctionOpInterface op);
178 template <
typename... T>
179 typename std::enable_if<
sizeof...(T) == 0, FailureOr<Operation *>>::type
212 template <
typename InterfaceT,
typename... InterfacesU>
213 FailureOr<Operation *> handleOp(
Operation *op) {
215 if (
auto concreteOp = dyn_cast<InterfaceT>(op)) {
216 FailureOr<Operation *> result = handleInterface(concreteOp);
222 return FailureOr<Operation *>(
nullptr);
223 return handleOp<InterfacesU...>(next);
227 FailureOr<Operation *> handleAllInterfaces(
Operation *op) {
228 if (
auto deallocOpInterface = dyn_cast<BufferDeallocationOpInterface>(op))
229 return deallocOpInterface.process(state,
options);
231 if (failed(verifyOperationPreconditions(op)))
234 return handleOp<MemoryEffectOpInterface, RegionBranchOpInterface,
235 CallOpInterface, BranchOpInterface,
236 RegionBranchTerminatorOpInterface>(op);
259 FailureOr<Operation *> handleInterface(BranchOpInterface op);
291 FailureOr<Operation *> handleInterface(RegionBranchOpInterface op);
320 FailureOr<Operation *> handleInterface(CallOpInterface op);
340 FailureOr<Operation *> handleInterface(MemoryEffectOpInterface op);
363 FailureOr<Operation *> handleInterface(RegionBranchTerminatorOpInterface op);
376 template <
typename OpTy>
378 return cast<OpTy>(appendOpResults(op.getOperation(), types));
384 LogicalResult deallocate(
Block *block);
397 void populateRemainingOwnerships(
Operation *op);
402 Value materializeMemrefWithGuaranteedOwnership(
OpBuilder &builder,
408 bool isFunctionWithoutDynamicOwnership(
Operation *op);
417 std::pair<Value, Value>
424 static LogicalResult verifyFunctionPreconditions(FunctionOpInterface op);
440 static LogicalResult verifyOperationPreconditions(
Operation *op);
447 static LogicalResult updateFunctionSignature(FunctionOpInterface op);
465 std::pair<Value, Value>
466 BufferDeallocation::materializeUniqueOwnership(
OpBuilder &builder,
Value memref,
471 return state.getMemrefWithUniqueOwnership(builder, memref, block);
479 if (
auto deallocOpInterface = dyn_cast<BufferDeallocationOpInterface>(owner))
480 return deallocOpInterface.materializeUniqueOwnershipForMemref(
481 state,
options, builder, memref);
484 return state.getMemrefWithUniqueOwnership(builder, memref, block);
488 BufferDeallocation::verifyFunctionPreconditions(FunctionOpInterface op) {
491 Backedges backedges(op);
492 if (backedges.size()) {
493 op->emitError(
"Only structured control-flow loops are supported.");
500 LogicalResult BufferDeallocation::verifyOperationPreconditions(
Operation *op) {
508 if (isa<DeallocOp>(op))
510 "No deallocation operations must be present when running this pass!");
520 if (!isa<MemoryEffectOpInterface>(op) &&
522 !isa<CallOpInterface>(op))
524 "ops with unknown memory side effects are not supported");
534 size_t size = regions.size();
535 if (((size == 1 && !op->
getResults().empty()) || size > 1) &&
536 !dyn_cast<RegionBranchOpInterface>(op)) {
537 return op->
emitError(
"All operations with attached regions need to "
538 "implement the RegionBranchOpInterface.");
545 return op->
emitError(
"NoTerminator trait is not supported");
550 if (!isa<BranchOpInterface, RegionBranchTerminatorOpInterface>(op) ||
551 (isa<BranchOpInterface>(op) &&
552 isa<RegionBranchTerminatorOpInterface>(op)))
555 "Terminators must implement either BranchOpInterface or "
556 "RegionBranchTerminatorOpInterface (but not both)!");
562 return op->
emitError(
"Terminators with more than one successor "
563 "are not supported!");
570 BufferDeallocation::updateFunctionSignature(FunctionOpInterface op) {
572 op.getFunctionBody().getOps<RegionBranchTerminatorOpInterface>(),
573 [](RegionBranchTerminatorOpInterface op) {
574 return op.getSuccessorOperands(RegionBranchPoint::parent()).getTypes();
576 if (!llvm::all_equal(returnOperandTypes))
577 return op->emitError(
578 "there are multiple return operations with different operand types");
580 TypeRange resultTypes = op.getResultTypes();
585 if (!returnOperandTypes.empty())
586 resultTypes = returnOperandTypes[0];
589 op->getContext(), op.getFunctionBody().front().getArgumentTypes(),
595 LogicalResult BufferDeallocation::deallocate(FunctionOpInterface op) {
597 if (failed(verifyFunctionPreconditions(op)))
603 if (failed(deallocate(block)))
607 if (result.wasInterrupted())
612 return updateFunctionSignature(op);
615 LogicalResult BufferDeallocation::deallocate(
Block *block) {
620 state.getLiveMemrefsIn(block, liveMemrefs);
621 for (
auto li : liveMemrefs) {
625 if (li.getParentRegion() == block->
getParent()) {
626 state.updateOwnership(li, state.getOwnership(li, li.getParentBlock()),
628 state.addMemrefToDeallocate(li, block);
632 if (li.getParentRegion()->isProperAncestor(block->
getParent())) {
634 state.updateOwnership(li, falseVal, block);
645 if (isa<FunctionOpInterface>(block->
getParentOp()) &&
648 state.updateOwnership(arg, newArg);
649 state.addMemrefToDeallocate(arg, block);
655 state.updateOwnership(arg, newArg);
656 state.addMemrefToDeallocate(arg, block);
661 for (
Operation &op : llvm::make_early_inc_range(*block)) {
662 FailureOr<Operation *> result = handleAllInterfaces(&op);
668 populateRemainingOwnerships(*result);
678 newTypes.append(types.begin(), types.end());
683 for (
auto [oldRegion, newRegion] :
684 llvm::zip(op->
getRegions(), newOp->getRegions()))
685 newRegion.takeBody(oldRegion);
694 FailureOr<Operation *>
695 BufferDeallocation::handleInterface(RegionBranchOpInterface op) {
713 assert(!regions.empty() &&
"Must have at least one successor region");
715 op.getEntrySuccessorOperands(regions.front()));
716 unsigned numMemrefOperands = llvm::count_if(entryOperands,
isMemref);
721 op->insertOperands(op->getNumOperands(),
724 int counter = op->getNumResults();
725 unsigned numMemrefResults = llvm::count_if(op->getResults(),
isMemref);
727 RegionBranchOpInterface newOp = appendOpResults(op, ownershipResults);
729 for (
auto result : llvm::make_filter_range(newOp->getResults(),
isMemref)) {
730 state.updateOwnership(result, newOp->getResult(counter++));
731 state.addMemrefToDeallocate(result, newOp->getBlock());
734 return newOp.getOperation();
737 Value BufferDeallocation::materializeMemrefWithGuaranteedOwnership(
740 std::pair<Value, Value> newMemrefAndOnwership =
741 materializeUniqueOwnership(builder, memref, block);
742 Value newMemref = newMemrefAndOnwership.first;
743 Value condition = newMemrefAndOnwership.second;
756 memref.
getLoc(), condition,
758 builder.create<scf::YieldOp>(loc, newMemref);
762 builder.
create<bufferization::CloneOp>(loc, newMemref);
767 state.updateOwnership(maybeClone, trueVal);
768 state.addMemrefToDeallocate(maybeClone, maybeClone.
getParentBlock());
772 FailureOr<Operation *>
773 BufferDeallocation::handleInterface(BranchOpInterface op) {
774 if (op->getNumSuccessors() > 1)
775 return op->emitError(
"BranchOpInterface operations with multiple "
776 "successors are not supported yet");
778 if (op->getNumSuccessors() != 1)
780 "only BranchOpInterface operations with exactly "
781 "one successor are supported yet");
783 if (op.getSuccessorOperands(0).getProducedOperandCount() > 0)
784 return op.emitError(
"produced operands are not supported");
788 Block *block = op->getBlock();
791 if (failed(state.getMemrefsAndConditionsToDeallocate(
792 builder, op.getLoc(), block, memrefs, conditions)))
796 op.getSuccessorOperands(0).getForwardedOperands();
797 state.getMemrefsToRetain(block, op->getSuccessor(0), forwardedOperands,
800 auto deallocOp = builder.create<bufferization::DeallocOp>(
801 op.getLoc(), memrefs, conditions, toRetain);
805 state.resetOwnerships(deallocOp.getRetained(), block);
806 for (
auto [retained, ownership] :
807 llvm::zip(deallocOp.getRetained(), deallocOp.getUpdatedConditions())) {
808 state.updateOwnership(retained, ownership, block);
811 unsigned numAdditionalReturns = llvm::count_if(forwardedOperands,
isMemref);
813 auto additionalConditions =
814 deallocOp.getUpdatedConditions().take_front(numAdditionalReturns);
815 newOperands.append(additionalConditions.begin(), additionalConditions.end());
816 op.getSuccessorOperands(0).getMutableForwardedOperands().assign(newOperands);
818 return op.getOperation();
821 FailureOr<Operation *> BufferDeallocation::handleInterface(CallOpInterface op) {
828 Operation *funcOp = op.resolveCallableInTable(state.getSymbolTable());
829 bool isPrivate =
false;
830 if (
auto symbol = dyn_cast_or_null<SymbolOpInterface>(funcOp))
831 isPrivate = symbol.isPrivate() && !symbol.isDeclaration();
837 if (
options.privateFuncDynamicOwnership && isPrivate) {
838 unsigned numMemrefs = llvm::count_if(op->getResults(),
isMemref);
840 unsigned ownershipCounter = op->getNumResults();
841 op = appendOpResults(op, ownershipTypesToAppend);
843 for (
auto result : llvm::make_filter_range(op->getResults(),
isMemref)) {
844 state.updateOwnership(result, op->getResult(ownershipCounter++));
845 state.addMemrefToDeallocate(result, result.getParentBlock());
848 return op.getOperation();
855 for (
auto result : llvm::make_filter_range(op->getResults(),
isMemref)) {
856 state.updateOwnership(result, trueVal);
857 state.addMemrefToDeallocate(result, result.getParentBlock());
860 return op.getOperation();
863 FailureOr<Operation *>
864 BufferDeallocation::handleInterface(MemoryEffectOpInterface op) {
865 auto *block = op->getBlock();
868 for (
auto operand : llvm::make_filter_range(op->getOperands(),
isMemref)) {
876 if (!op->hasAttr(BufferizationDialect::kManualDeallocation))
877 return op->emitError(
878 "memory free side-effect on MemRef value not supported!");
886 Ownership ownership = state.getOwnership(operand, block);
888 Value ownershipInverted = builder.
create<arith::XOrIOp>(
891 builder.
create<cf::AssertOp>(
892 op.getLoc(), ownershipInverted,
893 "expected that the block does not have ownership");
898 for (
auto res : llvm::make_filter_range(op->getResults(),
isMemref)) {
900 if (allocEffect.has_value()) {
901 if (isa<SideEffects::AutomaticAllocationScopeResource>(
902 allocEffect->getResource())) {
910 state.resetOwnerships(res, block);
911 state.updateOwnership(res,
buildBoolValue(builder, op.getLoc(),
false));
915 if (op->hasAttr(BufferizationDialect::kManualDeallocation)) {
919 state.resetOwnerships(res, block);
920 state.updateOwnership(res,
buildBoolValue(builder, op.getLoc(),
false));
924 state.updateOwnership(res,
buildBoolValue(builder, op.getLoc(),
true));
925 state.addMemrefToDeallocate(res, block);
929 return op.getOperation();
932 FailureOr<Operation *>
933 BufferDeallocation::handleInterface(RegionBranchTerminatorOpInterface op) {
940 bool funcWithoutDynamicOwnership =
941 isFunctionWithoutDynamicOwnership(op->getParentOp());
942 if (funcWithoutDynamicOwnership) {
943 for (
OpOperand &val : op->getOpOperands()) {
947 val.set(materializeMemrefWithGuaranteedOwnership(builder, val.get(),
962 if (failed(result) || !*result)
966 if (!funcWithoutDynamicOwnership) {
968 newOperands.append(updatedOwnerships.begin(), updatedOwnerships.end());
969 operands.
assign(newOperands);
972 return op.getOperation();
975 bool BufferDeallocation::isFunctionWithoutDynamicOwnership(
Operation *op) {
976 auto funcOp = dyn_cast<FunctionOpInterface>(op);
977 return funcOp && (!
options.privateFuncDynamicOwnership ||
978 !funcOp.isPrivate() || funcOp.isExternal());
981 void BufferDeallocation::populateRemainingOwnerships(
Operation *op) {
985 if (!state.getOwnership(res, op->
getBlock()).isUninitialized())
995 state.updateOwnership(
996 res, state.getOwnership(operand, operand.getParentBlock()),
1006 if (state.getOwnership(res, op->
getBlock()).isUninitialized()) {
1022 struct OwnershipBasedBufferDeallocationPass
1023 :
public bufferization::impl::OwnershipBasedBufferDeallocationPassBase<
1024 OwnershipBasedBufferDeallocationPass> {
1027 void runOnOperation()
override {
1029 options.privateFuncDynamicOwnership = privateFuncDynamicOwnership;
1033 auto status = getOperation()->walk([&](func::FuncOp func) {
1034 if (func.isExternal())
1042 if (status.wasInterrupted())
1043 signalPassFailure();
1057 BufferDeallocation deallocation(op,
options, symbolTables);
1060 return deallocation.deallocate(op);
static bool isMemref(Value v)
static Value buildBoolValue(OpBuilder &builder, Location loc, bool value)
static bool hasBufferSemantics(Operation *op)
Return "true" if the given op has buffer semantics.
static bool hasNeitherAllocateNorFreeSideEffect(Operation *op)
Return "true" if the given op is guaranteed to have neither "Allocate" nor "Free" side effects.
static llvm::ManagedStatic< PassManagerOptions > options
This class represents an argument of a Block.
Location getLoc() const
Return the location for this argument.
Block represents an ordered list of Operations.
BlockArgument getArgument(unsigned i)
unsigned getNumArguments()
Region * getParent() const
Provide a 'getParent' method for ilist_node_with_parent methods.
SuccessorRange getSuccessors()
BlockArgument addArgument(Type type, Location loc)
Add one value to the argument list.
OpListType & getOperations()
bool isEntryBlock()
Return if this block is the entry block in the parent region.
Operation * getParentOp()
Returns the closest surrounding operation that contains this block.
BoolAttr getBoolAttr(bool value)
This class defines the main interface for locations in MLIR and acts as a non-nullable wrapper around...
This class provides a mutable adaptor for a range of operands.
OperandRange getAsOperandRange() const
Explicit conversion to an OperandRange.
void assign(ValueRange values)
Assign this range to the given values.
RAII guard to reset the insertion point of the builder when destroyed.
This class helps build Operations.
static OpBuilder atBlockBegin(Block *block, Listener *listener=nullptr)
Create a builder and set the insertion point to before the first operation in the block but still ins...
void setInsertionPoint(Block *block, Block::iterator insertPoint)
Set the insertion point to the specified location.
Operation * create(const OperationState &state)
Creates an operation given the fields represented as an OperationState.
Operation * insert(Operation *op)
Insert the given operation at the current insertion point and return it.
This class represents an operand of an operation.
This trait indicates that the memory effects of an operation includes the effects of operations neste...
This class provides the API for ops that are known to be terminators.
This class indicates that the regions associated with this op don't have terminators.
This class implements the operand iterators for the Operation class.
Operation is the basic unit of execution within MLIR.
DictionaryAttr getAttrDictionary()
Return all of the attributes on this operation as a DictionaryAttr.
bool hasTrait()
Returns true if the operation was registered with a particular trait, e.g.
unsigned getNumRegions()
Returns the number of regions held by this operation.
Location getLoc()
The source location the operation was defined or derived from.
static Operation * create(Location location, OperationName name, TypeRange resultTypes, ValueRange operands, NamedAttrList &&attributes, OpaqueProperties properties, BlockRange successors, unsigned numRegions)
Create a new Operation with the specific fields.
InFlightDiagnostic emitError(const Twine &message={})
Emit an error about fatal conditions with this operation, reporting up to any diagnostic handlers tha...
Block * getBlock()
Returns the operation block that contains this operation.
MutableArrayRef< Region > getRegions()
Returns the regions held by this operation.
OperationName getName()
The name of an operation is the key identifier for it.
result_type_range getResultTypes()
operand_range getOperands()
Returns an iterator on the underlying Value's.
void replaceAllUsesWith(ValuesT &&values)
Replace all uses of results of this operation with the provided 'values'.
SuccessorRange getSuccessors()
result_range getResults()
OpaqueProperties getPropertiesStorage()
Returns the properties storage.
void erase()
Remove this operation from its parent block and delete it.
unsigned getNumResults()
Return the number of results held by this operation.
static constexpr RegionBranchPoint parent()
Returns an instance of RegionBranchPoint representing the parent operation.
This class contains a list of basic blocks and a link to the parent operation it is attached to.
This class represents a collection of SymbolTables.
This class provides an abstraction over the various different ranges of value types.
This class represents an instance of an SSA value in the MLIR system, representing a computable value...
Type getType() const
Return the type of this value.
Block * getParentBlock()
Return the Block in which this Value is defined.
Location getLoc() const
Return the location of this value.
Operation * getDefiningOp() const
If this value is the result of an operation, return the operation that defines it.
static WalkResult advance()
static WalkResult interrupt()
This class collects all the state that we need to perform the buffer deallocation pass with associate...
This class is used to track the ownership of values.
bool isUnique() const
Check if this ownership value is in the 'Unique' state.
Value getIndicator() const
If this ownership value is in 'Unique' state, this function can be used to get the indicator paramete...
FailureOr< Operation * > insertDeallocOpForReturnLike(DeallocationState &state, Operation *op, ValueRange operands, SmallVectorImpl< Value > &updatedOperandOwnerships)
Insert a bufferization.dealloc operation right before op which has to be a terminator without any suc...
LogicalResult deallocateBuffersOwnershipBased(FunctionOpInterface op, DeallocationOptions options, SymbolTableCollection &symbolTables)
Run the ownership-based buffer deallocation.
Include the generated interface declarations.
bool matchPattern(Value value, const Pattern &pattern)
Entry point for matching a pattern over a Value.
InFlightDiagnostic emitError(Location loc)
Utility method to emit an error message using this location.
detail::constant_int_predicate_matcher m_One()
Matches a constant scalar / vector splat / tensor splat integer one.
Operation * clone(OpBuilder &b, Operation *op, TypeRange newResultTypes, ValueRange newOperands)
auto get(MLIRContext *context, Ts &&...params)
Helper method that injects context only if needed, this helps unify some of the attribute constructio...
This iterator enumerates elements according to their dominance relationship.
The following effect indicates that the operation allocates from some resource.
The following effect indicates that the operation frees some resource that has been allocated.
Options for BufferDeallocationOpInterface-based buffer deallocation.