32 namespace bufferization {
33 #define GEN_PASS_DEF_OWNERSHIPBASEDBUFFERDEALLOCATIONPASS
34 #include "mlir/Dialect/Bufferization/Transforms/Passes.h.inc"
46 return arith::ConstantOp::create(builder, loc, builder.
getBoolAttr(value));
54 if (isa<MemoryEffectOpInterface>(op))
55 return !hasEffect<MemoryEffects::Allocate>(op) &&
56 !hasEffect<MemoryEffects::Free>(op);
72 if (llvm::any_of(region.front().getArguments(),
isMemref))
95 size_t size()
const {
return edgeSet.size(); }
98 BackedgeSetT::const_iterator begin()
const {
return edgeSet.begin(); }
101 BackedgeSetT::const_iterator end()
const {
return edgeSet.end(); }
107 bool enter(
Block ¤t,
Block *predecessor) {
108 bool inserted = visited.insert(¤t).second;
110 edgeSet.insert(std::make_pair(predecessor, ¤t));
115 void exit(
Block ¤t) { visited.erase(¤t); }
123 if (isa<BranchOpInterface>(op)) {
125 recurse(*succ, current);
131 recurse(region.front(), current);
137 void recurse(
Block &block,
Block *predecessor) {
140 if (!enter(block, predecessor))
155 BackedgeSetT edgeSet;
167 class BufferDeallocation {
174 LogicalResult deallocate(FunctionOpInterface op);
178 template <
typename... T>
179 typename std::enable_if<
sizeof...(T) == 0, FailureOr<Operation *>>::type
212 template <
typename InterfaceT,
typename... InterfacesU>
213 FailureOr<Operation *> handleOp(
Operation *op) {
215 if (
auto concreteOp = dyn_cast<InterfaceT>(op)) {
216 FailureOr<Operation *> result = handleInterface(concreteOp);
222 return FailureOr<Operation *>(
nullptr);
223 return handleOp<InterfacesU...>(next);
227 FailureOr<Operation *> handleAllInterfaces(
Operation *op) {
228 if (
auto deallocOpInterface = dyn_cast<BufferDeallocationOpInterface>(op))
229 return deallocOpInterface.process(state,
options);
231 if (failed(verifyOperationPreconditions(op)))
234 return handleOp<MemoryEffectOpInterface, RegionBranchOpInterface,
235 CallOpInterface, BranchOpInterface,
236 RegionBranchTerminatorOpInterface>(op);
259 FailureOr<Operation *> handleInterface(BranchOpInterface op);
291 FailureOr<Operation *> handleInterface(RegionBranchOpInterface op);
320 FailureOr<Operation *> handleInterface(CallOpInterface op);
340 FailureOr<Operation *> handleInterface(MemoryEffectOpInterface op);
363 FailureOr<Operation *> handleInterface(RegionBranchTerminatorOpInterface op);
376 template <
typename OpTy>
378 return cast<OpTy>(appendOpResults(op.getOperation(), types));
384 LogicalResult deallocate(
Block *block);
397 void populateRemainingOwnerships(
Operation *op);
402 Value materializeMemrefWithGuaranteedOwnership(
OpBuilder &builder,
408 bool isFunctionWithoutDynamicOwnership(
Operation *op);
417 std::pair<Value, Value>
424 static LogicalResult verifyFunctionPreconditions(FunctionOpInterface op);
440 static LogicalResult verifyOperationPreconditions(
Operation *op);
447 static LogicalResult updateFunctionSignature(FunctionOpInterface op);
465 std::pair<Value, Value>
466 BufferDeallocation::materializeUniqueOwnership(
OpBuilder &builder,
Value memref,
471 return state.getMemrefWithUniqueOwnership(builder, memref, block);
479 if (
auto deallocOpInterface = dyn_cast<BufferDeallocationOpInterface>(owner))
480 return deallocOpInterface.materializeUniqueOwnershipForMemref(
481 state,
options, builder, memref);
484 return state.getMemrefWithUniqueOwnership(builder, memref, block);
488 BufferDeallocation::verifyFunctionPreconditions(FunctionOpInterface op) {
491 Backedges backedges(op);
492 if (backedges.size()) {
493 op->emitError(
"Only structured control-flow loops are supported.");
500 LogicalResult BufferDeallocation::verifyOperationPreconditions(
Operation *op) {
508 if (isa<DeallocOp>(op))
510 "No deallocation operations must be present when running this pass!");
520 if (!isa<MemoryEffectOpInterface>(op) &&
522 !isa<CallOpInterface>(op))
524 "ops with unknown memory side effects are not supported");
534 size_t size = regions.size();
535 if (((size == 1 && !op->
getResults().empty()) || size > 1) &&
536 !dyn_cast<RegionBranchOpInterface>(op)) {
537 return op->
emitError(
"All operations with attached regions need to "
538 "implement the RegionBranchOpInterface.");
545 return op->
emitError(
"NoTerminator trait is not supported");
550 if (!isa<BranchOpInterface, RegionBranchTerminatorOpInterface>(op) ||
551 (isa<BranchOpInterface>(op) &&
552 isa<RegionBranchTerminatorOpInterface>(op)))
555 "Terminators must implement either BranchOpInterface or "
556 "RegionBranchTerminatorOpInterface (but not both)!");
562 return op->
emitError(
"Terminators with more than one successor "
563 "are not supported!");
570 BufferDeallocation::updateFunctionSignature(FunctionOpInterface op) {
572 op.getFunctionBody().getOps<RegionBranchTerminatorOpInterface>(),
573 [](RegionBranchTerminatorOpInterface op) {
574 return op.getSuccessorOperands(RegionBranchPoint::parent()).getTypes();
576 if (!llvm::all_equal(returnOperandTypes))
577 return op->emitError(
578 "there are multiple return operations with different operand types");
580 TypeRange resultTypes = op.getResultTypes();
585 if (!returnOperandTypes.empty())
586 resultTypes = returnOperandTypes[0];
589 op->getContext(), op.getFunctionBody().front().getArgumentTypes(),
595 LogicalResult BufferDeallocation::deallocate(FunctionOpInterface op) {
597 if (failed(verifyFunctionPreconditions(op)))
603 if (failed(deallocate(block)))
607 if (result.wasInterrupted())
612 return updateFunctionSignature(op);
615 LogicalResult BufferDeallocation::deallocate(
Block *block) {
620 state.getLiveMemrefsIn(block, liveMemrefs);
621 for (
auto li : liveMemrefs) {
625 if (li.getParentRegion() == block->
getParent()) {
626 state.updateOwnership(li, state.getOwnership(li, li.getParentBlock()),
628 state.addMemrefToDeallocate(li, block);
632 if (li.getParentRegion()->isProperAncestor(block->
getParent())) {
634 state.updateOwnership(li, falseVal, block);
645 if (isa<FunctionOpInterface>(block->
getParentOp()) &&
648 state.updateOwnership(arg, newArg);
649 state.addMemrefToDeallocate(arg, block);
655 state.updateOwnership(arg, newArg);
656 state.addMemrefToDeallocate(arg, block);
661 for (
Operation &op : llvm::make_early_inc_range(*block)) {
662 FailureOr<Operation *> result = handleAllInterfaces(&op);
668 populateRemainingOwnerships(*result);
678 newTypes.append(types.begin(), types.end());
683 for (
auto [oldRegion, newRegion] :
684 llvm::zip(op->
getRegions(), newOp->getRegions()))
685 newRegion.takeBody(oldRegion);
694 FailureOr<Operation *>
695 BufferDeallocation::handleInterface(RegionBranchOpInterface op) {
713 assert(!regions.empty() &&
"Must have at least one successor region");
715 op.getEntrySuccessorOperands(regions.front()));
716 unsigned numMemrefOperands = llvm::count_if(entryOperands,
isMemref);
721 op->insertOperands(op->getNumOperands(),
724 int counter = op->getNumResults();
725 unsigned numMemrefResults = llvm::count_if(op->getResults(),
isMemref);
727 RegionBranchOpInterface newOp = appendOpResults(op, ownershipResults);
729 for (
auto result : llvm::make_filter_range(newOp->getResults(),
isMemref)) {
730 state.updateOwnership(result, newOp->getResult(counter++));
731 state.addMemrefToDeallocate(result, newOp->getBlock());
734 return newOp.getOperation();
737 Value BufferDeallocation::materializeMemrefWithGuaranteedOwnership(
740 std::pair<Value, Value> newMemrefAndOnwership =
741 materializeUniqueOwnership(builder, memref, block);
742 Value newMemref = newMemrefAndOnwership.first;
743 Value condition = newMemrefAndOnwership.second;
753 Value maybeClone = scf::IfOp::create(
754 builder, memref.
getLoc(), condition,
756 scf::YieldOp::create(builder, loc, newMemref);
759 Value clone = bufferization::CloneOp::create(
760 builder, loc, newMemref);
761 scf::YieldOp::create(builder, loc, clone);
765 state.updateOwnership(maybeClone, trueVal);
766 state.addMemrefToDeallocate(maybeClone, maybeClone.
getParentBlock());
770 FailureOr<Operation *>
771 BufferDeallocation::handleInterface(BranchOpInterface op) {
772 if (op->getNumSuccessors() > 1)
773 return op->emitError(
"BranchOpInterface operations with multiple "
774 "successors are not supported yet");
776 if (op->getNumSuccessors() != 1)
778 "only BranchOpInterface operations with exactly "
779 "one successor are supported yet");
781 if (op.getSuccessorOperands(0).getProducedOperandCount() > 0)
782 return op.emitError(
"produced operands are not supported");
786 Block *block = op->getBlock();
789 if (failed(state.getMemrefsAndConditionsToDeallocate(
790 builder, op.getLoc(), block, memrefs, conditions)))
794 op.getSuccessorOperands(0).getForwardedOperands();
795 state.getMemrefsToRetain(block, op->getSuccessor(0), forwardedOperands,
798 auto deallocOp = bufferization::DeallocOp::create(
799 builder, op.getLoc(), memrefs, conditions, toRetain);
803 state.resetOwnerships(deallocOp.getRetained(), block);
804 for (
auto [retained, ownership] :
805 llvm::zip(deallocOp.getRetained(), deallocOp.getUpdatedConditions())) {
806 state.updateOwnership(retained, ownership, block);
809 unsigned numAdditionalReturns = llvm::count_if(forwardedOperands,
isMemref);
811 auto additionalConditions =
812 deallocOp.getUpdatedConditions().take_front(numAdditionalReturns);
813 newOperands.append(additionalConditions.begin(), additionalConditions.end());
814 op.getSuccessorOperands(0).getMutableForwardedOperands().assign(newOperands);
816 return op.getOperation();
819 FailureOr<Operation *> BufferDeallocation::handleInterface(CallOpInterface op) {
826 Operation *funcOp = op.resolveCallableInTable(state.getSymbolTable());
827 bool isPrivate =
false;
828 if (
auto symbol = dyn_cast_or_null<SymbolOpInterface>(funcOp))
829 isPrivate = symbol.isPrivate() && !symbol.isDeclaration();
835 if (
options.privateFuncDynamicOwnership && isPrivate) {
836 unsigned numMemrefs = llvm::count_if(op->getResults(),
isMemref);
838 unsigned ownershipCounter = op->getNumResults();
839 op = appendOpResults(op, ownershipTypesToAppend);
841 for (
auto result : llvm::make_filter_range(op->getResults(),
isMemref)) {
842 state.updateOwnership(result, op->getResult(ownershipCounter++));
843 state.addMemrefToDeallocate(result, result.getParentBlock());
846 return op.getOperation();
853 for (
auto result : llvm::make_filter_range(op->getResults(),
isMemref)) {
854 state.updateOwnership(result, trueVal);
855 state.addMemrefToDeallocate(result, result.getParentBlock());
858 return op.getOperation();
861 FailureOr<Operation *>
862 BufferDeallocation::handleInterface(MemoryEffectOpInterface op) {
863 auto *block = op->getBlock();
866 for (
auto operand : llvm::make_filter_range(op->getOperands(),
isMemref)) {
874 if (!op->hasAttr(BufferizationDialect::kManualDeallocation))
875 return op->emitError(
876 "memory free side-effect on MemRef value not supported!");
884 Ownership ownership = state.getOwnership(operand, block);
886 Value ownershipInverted = arith::XOrIOp::create(
889 cf::AssertOp::create(builder, op.getLoc(), ownershipInverted,
890 "expected that the block does not have ownership");
895 for (
auto res : llvm::make_filter_range(op->getResults(),
isMemref)) {
897 if (allocEffect.has_value()) {
898 if (isa<SideEffects::AutomaticAllocationScopeResource>(
899 allocEffect->getResource())) {
907 state.resetOwnerships(res, block);
908 state.updateOwnership(res,
buildBoolValue(builder, op.getLoc(),
false));
912 if (op->hasAttr(BufferizationDialect::kManualDeallocation)) {
916 state.resetOwnerships(res, block);
917 state.updateOwnership(res,
buildBoolValue(builder, op.getLoc(),
false));
921 state.updateOwnership(res,
buildBoolValue(builder, op.getLoc(),
true));
922 state.addMemrefToDeallocate(res, block);
926 return op.getOperation();
929 FailureOr<Operation *>
930 BufferDeallocation::handleInterface(RegionBranchTerminatorOpInterface op) {
937 bool funcWithoutDynamicOwnership =
938 isFunctionWithoutDynamicOwnership(op->getParentOp());
939 if (funcWithoutDynamicOwnership) {
940 for (
OpOperand &val : op->getOpOperands()) {
944 val.set(materializeMemrefWithGuaranteedOwnership(builder, val.get(),
959 if (failed(result) || !*result)
963 if (!funcWithoutDynamicOwnership) {
965 newOperands.append(updatedOwnerships.begin(), updatedOwnerships.end());
966 operands.
assign(newOperands);
969 return op.getOperation();
972 bool BufferDeallocation::isFunctionWithoutDynamicOwnership(
Operation *op) {
973 auto funcOp = dyn_cast<FunctionOpInterface>(op);
974 return funcOp && (!
options.privateFuncDynamicOwnership ||
975 !funcOp.isPrivate() || funcOp.isExternal());
978 void BufferDeallocation::populateRemainingOwnerships(
Operation *op) {
982 if (!state.getOwnership(res, op->
getBlock()).isUninitialized())
992 state.updateOwnership(
993 res, state.getOwnership(operand, operand.getParentBlock()),
1003 if (state.getOwnership(res, op->
getBlock()).isUninitialized()) {
1019 struct OwnershipBasedBufferDeallocationPass
1020 :
public bufferization::impl::OwnershipBasedBufferDeallocationPassBase<
1021 OwnershipBasedBufferDeallocationPass> {
1024 void runOnOperation()
override {
1026 options.privateFuncDynamicOwnership = privateFuncDynamicOwnership;
1030 auto status = getOperation()->walk([&](func::FuncOp func) {
1031 if (func.isExternal())
1039 if (status.wasInterrupted())
1040 signalPassFailure();
1054 BufferDeallocation deallocation(op,
options, symbolTables);
1057 return deallocation.deallocate(op);
static bool isMemref(Value v)
static Value buildBoolValue(OpBuilder &builder, Location loc, bool value)
static bool hasBufferSemantics(Operation *op)
Return "true" if the given op has buffer semantics.
static bool hasNeitherAllocateNorFreeSideEffect(Operation *op)
Return "true" if the given op is guaranteed to have neither "Allocate" nor "Free" side effects.
static llvm::ManagedStatic< PassManagerOptions > options
This class represents an argument of a Block.
Location getLoc() const
Return the location for this argument.
Block represents an ordered list of Operations.
BlockArgument getArgument(unsigned i)
unsigned getNumArguments()
Region * getParent() const
Provide a 'getParent' method for ilist_node_with_parent methods.
SuccessorRange getSuccessors()
BlockArgument addArgument(Type type, Location loc)
Add one value to the argument list.
OpListType & getOperations()
bool isEntryBlock()
Return if this block is the entry block in the parent region.
Operation * getParentOp()
Returns the closest surrounding operation that contains this block.
BoolAttr getBoolAttr(bool value)
This class defines the main interface for locations in MLIR and acts as a non-nullable wrapper around...
This class provides a mutable adaptor for a range of operands.
OperandRange getAsOperandRange() const
Explicit conversion to an OperandRange.
void assign(ValueRange values)
Assign this range to the given values.
RAII guard to reset the insertion point of the builder when destroyed.
This class helps build Operations.
static OpBuilder atBlockBegin(Block *block, Listener *listener=nullptr)
Create a builder and set the insertion point to before the first operation in the block but still ins...
void setInsertionPoint(Block *block, Block::iterator insertPoint)
Set the insertion point to the specified location.
Operation * insert(Operation *op)
Insert the given operation at the current insertion point and return it.
This class represents an operand of an operation.
This trait indicates that the memory effects of an operation includes the effects of operations neste...
This class provides the API for ops that are known to be terminators.
This class indicates that the regions associated with this op don't have terminators.
This class implements the operand iterators for the Operation class.
Operation is the basic unit of execution within MLIR.
DictionaryAttr getAttrDictionary()
Return all of the attributes on this operation as a DictionaryAttr.
bool hasTrait()
Returns true if the operation was registered with a particular trait, e.g.
unsigned getNumRegions()
Returns the number of regions held by this operation.
Location getLoc()
The source location the operation was defined or derived from.
static Operation * create(Location location, OperationName name, TypeRange resultTypes, ValueRange operands, NamedAttrList &&attributes, OpaqueProperties properties, BlockRange successors, unsigned numRegions)
Create a new Operation with the specific fields.
InFlightDiagnostic emitError(const Twine &message={})
Emit an error about fatal conditions with this operation, reporting up to any diagnostic handlers tha...
Block * getBlock()
Returns the operation block that contains this operation.
MutableArrayRef< Region > getRegions()
Returns the regions held by this operation.
OperationName getName()
The name of an operation is the key identifier for it.
result_type_range getResultTypes()
operand_range getOperands()
Returns an iterator on the underlying Value's.
void replaceAllUsesWith(ValuesT &&values)
Replace all uses of results of this operation with the provided 'values'.
SuccessorRange getSuccessors()
result_range getResults()
OpaqueProperties getPropertiesStorage()
Returns the properties storage.
void erase()
Remove this operation from its parent block and delete it.
unsigned getNumResults()
Return the number of results held by this operation.
static constexpr RegionBranchPoint parent()
Returns an instance of RegionBranchPoint representing the parent operation.
This class contains a list of basic blocks and a link to the parent operation it is attached to.
This class represents a collection of SymbolTables.
This class provides an abstraction over the various different ranges of value types.
This class represents an instance of an SSA value in the MLIR system, representing a computable value...
Type getType() const
Return the type of this value.
Block * getParentBlock()
Return the Block in which this Value is defined.
Location getLoc() const
Return the location of this value.
Operation * getDefiningOp() const
If this value is the result of an operation, return the operation that defines it.
static WalkResult advance()
static WalkResult interrupt()
This class collects all the state that we need to perform the buffer deallocation pass with associate...
This class is used to track the ownership of values.
bool isUnique() const
Check if this ownership value is in the 'Unique' state.
Value getIndicator() const
If this ownership value is in 'Unique' state, this function can be used to get the indicator paramete...
FailureOr< Operation * > insertDeallocOpForReturnLike(DeallocationState &state, Operation *op, ValueRange operands, SmallVectorImpl< Value > &updatedOperandOwnerships)
Insert a bufferization.dealloc operation right before op which has to be a terminator without any suc...
LogicalResult deallocateBuffersOwnershipBased(FunctionOpInterface op, DeallocationOptions options, SymbolTableCollection &symbolTables)
Run the ownership-based buffer deallocation.
Include the generated interface declarations.
bool matchPattern(Value value, const Pattern &pattern)
Entry point for matching a pattern over a Value.
InFlightDiagnostic emitError(Location loc)
Utility method to emit an error message using this location.
detail::constant_int_predicate_matcher m_One()
Matches a constant scalar / vector splat / tensor splat integer one.
auto get(MLIRContext *context, Ts &&...params)
Helper method that injects context only if needed, this helps unify some of the attribute constructio...
This iterator enumerates elements according to their dominance relationship.
The following effect indicates that the operation allocates from some resource.
The following effect indicates that the operation frees some resource that has been allocated.
Options for BufferDeallocationOpInterface-based buffer deallocation.