MLIR  22.0.0git
OwnershipBasedBufferDeallocation.cpp
Go to the documentation of this file.
1 //===- OwnershipBasedBufferDeallocation.cpp - impl. for buffer dealloc. ---===//
2 //
3 // Part of the LLVM Project, under the Apache License v2.0 with LLVM Exceptions.
4 // See https://llvm.org/LICENSE.txt for license information.
5 // SPDX-License-Identifier: Apache-2.0 WITH LLVM-exception
6 //
7 //===----------------------------------------------------------------------===//
8 //
9 // This file implements logic for computing correct `bufferization.dealloc`
10 // positions. Furthermore, buffer deallocation also adds required new clone
11 // operations to ensure that memrefs returned by functions never alias an
12 // argument.
13 //
14 // TODO:
15 // The current implementation does not support explicit-control-flow loops and
16 // the resulting code will be invalid with respect to program semantics.
17 // However, structured control-flow loops are fully supported.
18 //
19 //===----------------------------------------------------------------------===//
20 
28 #include "mlir/IR/Iterators.h"
30 
31 namespace mlir {
32 namespace bufferization {
33 #define GEN_PASS_DEF_OWNERSHIPBASEDBUFFERDEALLOCATIONPASS
34 #include "mlir/Dialect/Bufferization/Transforms/Passes.h.inc"
35 } // namespace bufferization
36 } // namespace mlir
37 
38 using namespace mlir;
39 using namespace mlir::bufferization;
40 
41 //===----------------------------------------------------------------------===//
42 // Helpers
43 //===----------------------------------------------------------------------===//
44 
45 static Value buildBoolValue(OpBuilder &builder, Location loc, bool value) {
46  return arith::ConstantOp::create(builder, loc, builder.getBoolAttr(value));
47 }
48 
49 static bool isMemref(Value v) { return isa<BaseMemRefType>(v.getType()); }
50 
51 /// Return "true" if the given op is guaranteed to have neither "Allocate" nor
52 /// "Free" side effects.
54  if (isa<MemoryEffectOpInterface>(op))
55  return !hasEffect<MemoryEffects::Allocate>(op) &&
56  !hasEffect<MemoryEffects::Free>(op);
57  // If the op does not implement the MemoryEffectOpInterface but has has
58  // recursive memory effects, then this op in isolation (without its body) does
59  // not have any side effects. All the ops inside the regions of this op will
60  // be processed separately.
62 }
63 
64 /// Return "true" if the given op has buffer semantics. I.e., it has buffer
65 /// operands, buffer results and/or buffer region entry block arguments.
66 static bool hasBufferSemantics(Operation *op) {
67  if (llvm::any_of(op->getOperands(), isMemref) ||
68  llvm::any_of(op->getResults(), isMemref))
69  return true;
70  for (Region &region : op->getRegions())
71  if (!region.empty())
72  if (llvm::any_of(region.front().getArguments(), isMemref))
73  return true;
74  return false;
75 }
76 
77 //===----------------------------------------------------------------------===//
78 // Backedges analysis
79 //===----------------------------------------------------------------------===//
80 
81 namespace {
82 
83 /// A straight-forward program analysis which detects loop backedges induced by
84 /// explicit control flow.
85 class Backedges {
86 public:
87  using BlockSetT = SmallPtrSet<Block *, 16>;
88  using BackedgeSetT = llvm::DenseSet<std::pair<Block *, Block *>>;
89 
90 public:
91  /// Constructs a new backedges analysis using the op provided.
92  Backedges(Operation *op) { recurse(op); }
93 
94  /// Returns the number of backedges formed by explicit control flow.
95  size_t size() const { return edgeSet.size(); }
96 
97  /// Returns the start iterator to loop over all backedges.
98  BackedgeSetT::const_iterator begin() const { return edgeSet.begin(); }
99 
100  /// Returns the end iterator to loop over all backedges.
101  BackedgeSetT::const_iterator end() const { return edgeSet.end(); }
102 
103 private:
104  /// Enters the current block and inserts a backedge into the `edgeSet` if we
105  /// have already visited the current block. The inserted edge links the given
106  /// `predecessor` with the `current` block.
107  bool enter(Block &current, Block *predecessor) {
108  bool inserted = visited.insert(&current).second;
109  if (!inserted)
110  edgeSet.insert(std::make_pair(predecessor, &current));
111  return inserted;
112  }
113 
114  /// Leaves the current block.
115  void exit(Block &current) { visited.erase(&current); }
116 
117  /// Recurses into the given operation while taking all attached regions into
118  /// account.
119  void recurse(Operation *op) {
120  Block *current = op->getBlock();
121  // If the current op implements the `BranchOpInterface`, there can be
122  // cycles in the scope of all successor blocks.
123  if (isa<BranchOpInterface>(op)) {
124  for (Block *succ : current->getSuccessors())
125  recurse(*succ, current);
126  }
127  // Recurse into all distinct regions and check for explicit control-flow
128  // loops.
129  for (Region &region : op->getRegions()) {
130  if (!region.empty())
131  recurse(region.front(), current);
132  }
133  }
134 
135  /// Recurses into explicit control-flow structures that are given by
136  /// the successor relation defined on the block level.
137  void recurse(Block &block, Block *predecessor) {
138  // Try to enter the current block. If this is not possible, we are
139  // currently processing this block and can safely return here.
140  if (!enter(block, predecessor))
141  return;
142 
143  // Recurse into all operations and successor blocks.
144  for (Operation &op : block.getOperations())
145  recurse(&op);
146 
147  // Leave the current block.
148  exit(block);
149  }
150 
151  /// Stores all blocks that are currently visited and on the processing stack.
152  BlockSetT visited;
153 
154  /// Stores all backedges in the format (source, target).
155  BackedgeSetT edgeSet;
156 };
157 
158 } // namespace
159 
160 //===----------------------------------------------------------------------===//
161 // BufferDeallocation
162 //===----------------------------------------------------------------------===//
163 
164 namespace {
165 /// The buffer deallocation transformation which ensures that all allocs in the
166 /// program have a corresponding de-allocation.
167 class BufferDeallocation {
168 public:
169  BufferDeallocation(Operation *op, DeallocationOptions options,
170  SymbolTableCollection &symbolTables)
171  : state(op, symbolTables), options(options) {}
172 
173  /// Performs the actual placement/creation of all dealloc operations.
174  LogicalResult deallocate(FunctionOpInterface op);
175 
176 private:
177  /// The base case for the recursive template below.
178  template <typename... T>
179  typename std::enable_if<sizeof...(T) == 0, FailureOr<Operation *>>::type
180  handleOp(Operation *op) {
181  return op;
182  }
183 
184  /// Applies all the handlers of the interfaces in the template list
185  /// implemented by 'op'. In particular, if an operation implements more than
186  /// one of the interfaces in the template list, all the associated handlers
187  /// will be applied to the operation in the same order as the template list
188  /// specifies. If a handler reports a failure or removes the operation without
189  /// replacement (indicated by returning 'nullptr'), no further handlers are
190  /// applied and the return value is propagated to the caller of 'handleOp'.
191  ///
192  /// The interface handlers job is to update the deallocation state, most
193  /// importantly the ownership map and list of memrefs to potentially be
194  /// deallocated per block, but also to insert `bufferization.dealloc`
195  /// operations where needed. Obviously, no MemRefs that may be used at a later
196  /// point in the control-flow may be deallocated and the ownership map has to
197  /// be updated to reflect potential ownership changes caused by the dealloc
198  /// operation (e.g., if two interfaces on the same op insert a dealloc
199  /// operation each, the second one should query the ownership map and use them
200  /// as deallocation condition such that MemRefs already deallocated in the
201  /// first dealloc operation are not deallocated a second time (double-free)).
202  /// Note that currently only the interfaces on terminators may insert dealloc
203  /// operations and it is verified as a precondition that a terminator op must
204  /// implement exactly one of the interfaces handling dealloc insertion.
205  ///
206  /// The return value of the 'handleInterface' functions should be a
207  /// FailureOr<Operation *> indicating whether there was a failure or otherwise
208  /// returning the operation itself or a replacement operation.
209  ///
210  /// Note: The difference compared to `TypeSwitch` is that all
211  /// matching cases are applied instead of just the first match.
212  template <typename InterfaceT, typename... InterfacesU>
213  FailureOr<Operation *> handleOp(Operation *op) {
214  Operation *next = op;
215  if (auto concreteOp = dyn_cast<InterfaceT>(op)) {
216  FailureOr<Operation *> result = handleInterface(concreteOp);
217  if (failed(result))
218  return failure();
219  next = *result;
220  }
221  if (!next)
222  return FailureOr<Operation *>(nullptr);
223  return handleOp<InterfacesU...>(next);
224  }
225 
226  /// Apply all supported interface handlers to the given op.
227  FailureOr<Operation *> handleAllInterfaces(Operation *op) {
228  if (auto deallocOpInterface = dyn_cast<BufferDeallocationOpInterface>(op))
229  return deallocOpInterface.process(state, options);
230 
231  if (failed(verifyOperationPreconditions(op)))
232  return failure();
233 
234  return handleOp<MemoryEffectOpInterface, RegionBranchOpInterface,
235  CallOpInterface, BranchOpInterface,
236  RegionBranchTerminatorOpInterface>(op);
237  }
238 
239  /// Make sure that for each forwarded MemRef value, an ownership indicator
240  /// `i1` value is forwarded as well such that the successor block knows
241  /// whether the MemRef has to be deallocated.
242  ///
243  /// Example:
244  /// ```
245  /// ^bb1:
246  /// <more ops...>
247  /// cf.br ^bb2(<forward-to-bb2>)
248  /// ```
249  /// becomes
250  /// ```
251  /// // let (m, c) = getMemrefsAndConditionsToDeallocate(bb1)
252  /// // let r = getMemrefsToRetain(bb1, bb2, <forward-to-bb2>)
253  /// ^bb1:
254  /// <more ops...>
255  /// o = bufferization.dealloc m if c retain r
256  /// // replace ownership(r) with o element-wise
257  /// cf.br ^bb2(<forward-to-bb2>, o)
258  /// ```
259  FailureOr<Operation *> handleInterface(BranchOpInterface op);
260 
261  /// Add an ownership indicator for every forwarding MemRef operand and result.
262  /// Nested regions never take ownership of MemRefs owned by a parent region
263  /// (neither via forwarding operand nor when captured implicitly when the
264  /// region is not isolated from above). Ownerships will only be passed to peer
265  /// regions (when an operation has multiple regions, such as scf.while), or to
266  /// parent regions.
267  /// Note that the block arguments in the nested region are currently handled
268  /// centrally in the 'dealloc' function, but better interface support could
269  /// allow us to do this here for the nested region specifically to reduce the
270  /// amount of assumptions we make on the structure of ops implementing this
271  /// interface.
272  ///
273  /// Example:
274  /// ```
275  /// %ret = scf.for %i = %c0 to %c10 step %c1 iter_args(%m = %memref) {
276  /// <more ops...>
277  /// scf.yield %m : memref<2xi32>, i1
278  /// }
279  /// ```
280  /// becomes
281  /// ```
282  /// %ret:2 = scf.for %i = %c0 to %c10 step %c1
283  /// iter_args(%m = %memref, %own = %false) {
284  /// <more ops...>
285  /// // Note that the scf.yield is handled by the
286  /// // RegionBranchTerminatorOpInterface (not this handler)
287  /// // let o = getMemrefWithUniqueOwnership(%own)
288  /// scf.yield %m, o : memref<2xi32>, i1
289  /// }
290  /// ```
291  FailureOr<Operation *> handleInterface(RegionBranchOpInterface op);
292 
293  /// If the private-function-dynamic-ownership pass option is enabled and the
294  /// called function is private, additional results are added for each MemRef
295  /// result to pass the dynamic ownership indicator along. Otherwise, updates
296  /// the ownership map and list of memrefs to be deallocated according to the
297  /// function boundary ABI, i.e., assume ownership of all returned MemRefs.
298  ///
299  /// Example (assume `private-function-dynamic-ownership` is enabled):
300  /// ```
301  /// func.func @f(%arg0: memref<2xi32>) -> memref<2xi32> {...}
302  /// func.func private @g(%arg0: memref<2xi32>) -> memref<2xi32> {...}
303  ///
304  /// %ret_f = func.call @f(%memref) : (memref<2xi32>) -> memref<2xi32>
305  /// %ret_g = func.call @g(%memref) : (memref<2xi32>) -> memref<2xi32>
306  /// ```
307  /// becomes
308  /// ```
309  /// func.func @f(%arg0: memref<2xi32>) -> memref<2xi32> {...}
310  /// func.func private @g(%arg0: memref<2xi32>) -> (memref<2xi32>, i1) {...}
311  ///
312  /// %ret_f = func.call @f(%memref) : (memref<2xi32>) -> memref<2xi32>
313  /// // set ownership(%ret_f) := true
314  /// // remember to deallocate %ret_f
315  ///
316  /// %ret_g:2 = func.call @g(%memref) : (memref<2xi32>) -> (memref<2xi32>, i1)
317  /// // set ownership(%ret_g#0) := %ret_g#1
318  /// // remember to deallocate %ret_g if it comes with ownership
319  /// ```
320  FailureOr<Operation *> handleInterface(CallOpInterface op);
321 
322  /// Takes care of allocation and free side-effects. It collects allocated
323  /// MemRefs that we have to add to manually deallocate, but also removes
324  /// values again that are already deallocated before the end of the block. It
325  /// also updates the ownership map accordingly.
326  ///
327  /// Example:
328  /// ```
329  /// %alloc = memref.alloc()
330  /// %alloca = memref.alloca()
331  /// ```
332  /// becomes
333  /// ```
334  /// %alloc = memref.alloc()
335  /// %alloca = memref.alloca()
336  /// // set ownership(alloc) := true
337  /// // set ownership(alloca) := false
338  /// // remember to deallocate %alloc
339  /// ```
340  FailureOr<Operation *> handleInterface(MemoryEffectOpInterface op);
341 
342  /// Takes care that the function boundary ABI is adhered to if the parent
343  /// operation implements FunctionOpInterface, inserting a
344  /// `bufferization.clone` if necessary, and inserts the
345  /// `bufferization.dealloc` operation according to the ops operands.
346  ///
347  /// Example:
348  /// ```
349  /// ^bb1:
350  /// <more ops...>
351  /// func.return <return-vals>
352  /// ```
353  /// becomes
354  /// ```
355  /// // let (m, c) = getMemrefsAndConditionsToDeallocate(bb1)
356  /// // let r = getMemrefsToRetain(bb1, nullptr, <return-vals>)
357  /// ^bb1:
358  /// <more ops...>
359  /// o = bufferization.dealloc m if c retain r
360  /// func.return <return-vals>
361  /// (if !isFunctionWithoutDynamicOwnership: append o)
362  /// ```
363  FailureOr<Operation *> handleInterface(RegionBranchTerminatorOpInterface op);
364 
365  /// Construct a new operation which is exactly the same as the passed 'op'
366  /// except that the OpResults list is appended by new results of the passed
367  /// 'types'.
368  /// TODO: ideally, this would be implemented using an OpInterface because it
369  /// is used to append function results, loop iter_args, etc. and thus makes
370  /// some assumptions that the variadic list of those is at the end of the
371  /// OpResults range.
372  Operation *appendOpResults(Operation *op, ArrayRef<Type> types);
373 
374  /// A convenience template for the generic 'appendOpResults' function above to
375  /// avoid manual casting of the result.
376  template <typename OpTy>
377  OpTy appendOpResults(OpTy op, ArrayRef<Type> types) {
378  return cast<OpTy>(appendOpResults(op.getOperation(), types));
379  }
380 
381  /// Performs deallocation of a single basic block. This is a private function
382  /// because some internal data structures have to be set up beforehand and
383  /// this function has to be called on blocks in a region in dominance order.
384  LogicalResult deallocate(Block *block);
385 
386  /// After all relevant interfaces of an operation have been processed by the
387  /// 'handleInterface' functions, this function sets the ownership of operation
388  /// results that have not been set yet by the 'handleInterface' functions. It
389  /// generally assumes that each result can alias with every operand of the
390  /// operation, if there are MemRef typed results but no MemRef operands it
391  /// assigns 'false' as ownership. This happens, e.g., for the
392  /// memref.get_global operation. It would also be possible to query some alias
393  /// analysis to get more precise ownerships, however, the analysis would have
394  /// to be updated according to the IR modifications this pass performs (e.g.,
395  /// re-building operations to have more result values, inserting clone
396  /// operations, etc.).
397  void populateRemainingOwnerships(Operation *op);
398 
399  /// Given an SSA value of MemRef type, returns the same of a new SSA value
400  /// which has 'Unique' ownership where the ownership indicator is guaranteed
401  /// to be always 'true'.
402  Value materializeMemrefWithGuaranteedOwnership(OpBuilder &builder,
403  Value memref, Block *block);
404 
405  /// Returns whether the given operation implements FunctionOpInterface, has
406  /// private visibility, and the private-function-dynamic-ownership pass option
407  /// is enabled.
408  bool isFunctionWithoutDynamicOwnership(Operation *op);
409 
410  /// Given an SSA value of MemRef type, this function queries the
411  /// BufferDeallocationOpInterface of the defining operation of 'memref' for a
412  /// materialized ownership indicator for 'memref'. If the op does not
413  /// implement the interface or if the block for which the materialized value
414  /// is requested does not match the block in which 'memref' is defined, the
415  /// default implementation in
416  /// `DeallocationState::getMemrefWithUniqueOwnership` is queried instead.
417  std::pair<Value, Value>
418  materializeUniqueOwnership(OpBuilder &builder, Value memref, Block *block);
419 
420  /// Checks all the preconditions for operations implementing the
421  /// FunctionOpInterface that have to hold for the deallocation to be
422  /// applicable:
423  /// (1) Checks that there are not explicit control flow loops.
424  static LogicalResult verifyFunctionPreconditions(FunctionOpInterface op);
425 
426  /// Checks all the preconditions for operations inside the region of
427  /// operations implementing the FunctionOpInterface that have to hold for the
428  /// deallocation to be applicable:
429  /// (1) Checks if all operations that have at least one attached region
430  /// implement the RegionBranchOpInterface. This is not required in edge cases,
431  /// where we have a single attached region and the parent operation has no
432  /// results.
433  /// (2) Checks that no deallocations already exist. Especially deallocations
434  /// in nested regions are not properly supported yet since this requires
435  /// ownership of the memref to be transferred to the nested region, which does
436  /// not happen by default. This constrained can be lifted in the future.
437  /// (3) Checks that terminators with more than one successor except
438  /// `cf.cond_br` are not present and that either BranchOpInterface or
439  /// RegionBranchTerminatorOpInterface is implemented.
440  static LogicalResult verifyOperationPreconditions(Operation *op);
441 
442  /// When the 'private-function-dynamic-ownership' pass option is enabled,
443  /// additional `i1` return values are added for each MemRef result in the
444  /// function signature. This function takes care of updating the
445  /// `function_type` attribute of the function according to the actually
446  /// returned values from the terminators.
447  static LogicalResult updateFunctionSignature(FunctionOpInterface op);
448 
449 private:
450  /// Collects all analysis state and including liveness, caches, ownerships of
451  /// already processed values and operations, and the MemRefs that have to be
452  /// deallocated at the end of each block.
453  DeallocationState state;
454 
455  /// Collects all pass options in a single place.
457 };
458 
459 } // namespace
460 
461 //===----------------------------------------------------------------------===//
462 // BufferDeallocation Implementation
463 //===----------------------------------------------------------------------===//
464 
465 std::pair<Value, Value>
466 BufferDeallocation::materializeUniqueOwnership(OpBuilder &builder, Value memref,
467  Block *block) {
468  // The interface can only materialize ownership indicators in the same block
469  // as the defining op.
470  if (memref.getParentBlock() != block)
471  return state.getMemrefWithUniqueOwnership(builder, memref, block);
472 
473  Operation *owner = memref.getDefiningOp();
474  if (!owner)
475  owner = memref.getParentBlock()->getParentOp();
476 
477  // If the op implements the interface, query it for a materialized ownership
478  // value.
479  if (auto deallocOpInterface = dyn_cast<BufferDeallocationOpInterface>(owner))
480  return deallocOpInterface.materializeUniqueOwnershipForMemref(
481  state, options, builder, memref);
482 
483  // Otherwise use the default implementation.
484  return state.getMemrefWithUniqueOwnership(builder, memref, block);
485 }
486 
487 LogicalResult
488 BufferDeallocation::verifyFunctionPreconditions(FunctionOpInterface op) {
489  // (1) Ensure that there are supported loops only (no explicit control flow
490  // loops).
491  Backedges backedges(op);
492  if (backedges.size()) {
493  op->emitError("Only structured control-flow loops are supported.");
494  return failure();
495  }
496 
497  return success();
498 }
499 
500 LogicalResult BufferDeallocation::verifyOperationPreconditions(Operation *op) {
501  // We do not care about ops that do not operate on buffers and have no
502  // Allocate/Free side effect.
504  return success();
505 
506  // (1) The pass does not work properly when deallocations are already present.
507  // Alternatively, we could also remove all deallocations as a pre-pass.
508  if (isa<DeallocOp>(op))
509  return op->emitError(
510  "No deallocation operations must be present when running this pass!");
511 
512  // (2) Memory side effects of unregistered ops are unknown. In particular, we
513  // do not know whether an unregistered op allocates memory or not.
514  // - Ops with recursive memory effects are allowed. All nested ops in the
515  // regions of `op` will be analyzed separately.
516  // - Call ops are allowed even though they typically do not implement the
517  // MemoryEffectOpInterface. They usually do not have side effects apart
518  // from the callee, which will be analyzed separately. (This is similar to
519  // "recursive memory effects".)
520  if (!isa<MemoryEffectOpInterface>(op) &&
522  !isa<CallOpInterface>(op))
523  return op->emitError(
524  "ops with unknown memory side effects are not supported");
525 
526  // (3) Check that the control flow structures are supported.
527  auto regions = op->getRegions();
528  // Check that if the operation has at
529  // least one region it implements the RegionBranchOpInterface. If there
530  // is an operation that does not fulfill this condition, we cannot apply
531  // the deallocation steps. Furthermore, we accept cases, where we have a
532  // region that returns no results, since, in that case, the intra-region
533  // control flow does not affect the transformation.
534  size_t size = regions.size();
535  if (((size == 1 && !op->getResults().empty()) || size > 1) &&
536  !dyn_cast<RegionBranchOpInterface>(op)) {
537  return op->emitError("All operations with attached regions need to "
538  "implement the RegionBranchOpInterface.");
539  }
540 
541  // (3) Check that terminators with more than one successor except `cf.cond_br`
542  // are not present and that either BranchOpInterface or
543  // RegionBranchTerminatorOpInterface is implemented.
544  if (op->hasTrait<OpTrait::NoTerminator>())
545  return op->emitError("NoTerminator trait is not supported");
546 
547  if (op->hasTrait<OpTrait::IsTerminator>()) {
548  // Either one of those interfaces has to be implemented on terminators, but
549  // not both.
550  if (!isa<BranchOpInterface, RegionBranchTerminatorOpInterface>(op) ||
551  (isa<BranchOpInterface>(op) &&
552  isa<RegionBranchTerminatorOpInterface>(op)))
553 
554  return op->emitError(
555  "Terminators must implement either BranchOpInterface or "
556  "RegionBranchTerminatorOpInterface (but not both)!");
557 
558  // We only support terminators with 0 or 1 successors for now and
559  // special-case the conditional branch op.
560  if (op->getSuccessors().size() > 1)
561 
562  return op->emitError("Terminators with more than one successor "
563  "are not supported!");
564  }
565 
566  return success();
567 }
568 
569 LogicalResult
570 BufferDeallocation::updateFunctionSignature(FunctionOpInterface op) {
571  SmallVector<TypeRange> returnOperandTypes(llvm::map_range(
572  op.getFunctionBody().getOps<RegionBranchTerminatorOpInterface>(),
573  [](RegionBranchTerminatorOpInterface op) {
574  return op.getSuccessorOperands(RegionBranchPoint::parent()).getTypes();
575  }));
576  if (!llvm::all_equal(returnOperandTypes))
577  return op->emitError(
578  "there are multiple return operations with different operand types");
579 
580  TypeRange resultTypes = op.getResultTypes();
581  // Check if we found a return operation because that doesn't necessarily
582  // always have to be the case, e.g., consider a function with one block that
583  // has a cf.br at the end branching to itself again (i.e., an infinite loop).
584  // In that case we don't want to crash but just not update the return types.
585  if (!returnOperandTypes.empty())
586  resultTypes = returnOperandTypes[0];
587 
588  op.setFunctionTypeAttr(TypeAttr::get(FunctionType::get(
589  op->getContext(), op.getFunctionBody().front().getArgumentTypes(),
590  resultTypes)));
591 
592  return success();
593 }
594 
595 LogicalResult BufferDeallocation::deallocate(FunctionOpInterface op) {
596  // Stop and emit a proper error message if we don't support the input IR.
597  if (failed(verifyFunctionPreconditions(op)))
598  return failure();
599 
600  // Process the function block by block.
601  auto result = op->walk<WalkOrder::PostOrder, ForwardDominanceIterator<>>(
602  [&](Block *block) {
603  if (failed(deallocate(block)))
604  return WalkResult::interrupt();
605  return WalkResult::advance();
606  });
607  if (result.wasInterrupted())
608  return failure();
609 
610  // Update the function signature if the function is private, dynamic ownership
611  // is enabled, and the function has memrefs as arguments or results.
612  return updateFunctionSignature(op);
613 }
614 
615 LogicalResult BufferDeallocation::deallocate(Block *block) {
616  OpBuilder builder = OpBuilder::atBlockBegin(block);
617 
618  // Compute liveness transfers of ownership to this block.
619  SmallVector<Value> liveMemrefs;
620  state.getLiveMemrefsIn(block, liveMemrefs);
621  for (auto li : liveMemrefs) {
622  // Ownership of implicitly captured memrefs from other regions is never
623  // taken, but ownership of memrefs in the same region (but different block)
624  // is taken.
625  if (li.getParentRegion() == block->getParent()) {
626  state.updateOwnership(li, state.getOwnership(li, li.getParentBlock()),
627  block);
628  state.addMemrefToDeallocate(li, block);
629  continue;
630  }
631 
632  if (li.getParentRegion()->isProperAncestor(block->getParent())) {
633  Value falseVal = buildBoolValue(builder, li.getLoc(), false);
634  state.updateOwnership(li, falseVal, block);
635  }
636  }
637 
638  for (unsigned i = 0, e = block->getNumArguments(); i < e; ++i) {
639  BlockArgument arg = block->getArgument(i);
640  if (!isMemref(arg))
641  continue;
642 
643  // Adhere to function boundary ABI: no ownership of function argument
644  // MemRefs is taken.
645  if (isa<FunctionOpInterface>(block->getParentOp()) &&
646  block->isEntryBlock()) {
647  Value newArg = buildBoolValue(builder, arg.getLoc(), false);
648  state.updateOwnership(arg, newArg);
649  state.addMemrefToDeallocate(arg, block);
650  continue;
651  }
652 
653  // Pass MemRef ownerships along via `i1` values.
654  Value newArg = block->addArgument(builder.getI1Type(), arg.getLoc());
655  state.updateOwnership(arg, newArg);
656  state.addMemrefToDeallocate(arg, block);
657  }
658 
659  // For each operation in the block, handle the interfaces that affect aliasing
660  // and ownership of memrefs.
661  for (Operation &op : llvm::make_early_inc_range(*block)) {
662  FailureOr<Operation *> result = handleAllInterfaces(&op);
663  if (failed(result))
664  return failure();
665  if (!*result)
666  continue;
667 
668  populateRemainingOwnerships(*result);
669  }
670 
671  // TODO: if block has no terminator, handle dealloc insertion here.
672  return success();
673 }
674 
675 Operation *BufferDeallocation::appendOpResults(Operation *op,
676  ArrayRef<Type> types) {
677  SmallVector<Type> newTypes(op->getResultTypes());
678  newTypes.append(types.begin(), types.end());
679  auto *newOp = Operation::create(op->getLoc(), op->getName(), newTypes,
680  op->getOperands(), op->getAttrDictionary(),
681  op->getPropertiesStorage(),
682  op->getSuccessors(), op->getNumRegions());
683  for (auto [oldRegion, newRegion] :
684  llvm::zip(op->getRegions(), newOp->getRegions()))
685  newRegion.takeBody(oldRegion);
686 
687  OpBuilder(op).insert(newOp);
688  op->replaceAllUsesWith(newOp->getResults().take_front(op->getNumResults()));
689  op->erase();
690 
691  return newOp;
692 }
693 
694 FailureOr<Operation *>
695 BufferDeallocation::handleInterface(RegionBranchOpInterface op) {
696  OpBuilder builder = OpBuilder::atBlockBegin(op->getBlock());
697 
698  // TODO: the RegionBranchOpInterface does not provide all the necessary
699  // methods to perform this transformation without additional assumptions on
700  // the structure. In particular, that
701  // * additional values to be passed to the next region can be added to the end
702  // of the operand list, the end of the block argument list, and the end of
703  // the result value list. However, it seems to be the general guideline for
704  // operations implementing this interface to follow this structure.
705  // * and that the block arguments and result values match the forwarded
706  // operands one-to-one (i.e., that there are no other values appended to the
707  // front).
708  // These assumptions are satisfied by the `scf.if`, `scf.for`, and `scf.while`
709  // operations.
710 
712  op.getSuccessorRegions(RegionBranchPoint::parent(), regions);
713  assert(!regions.empty() && "Must have at least one successor region");
714  SmallVector<Value> entryOperands(
715  op.getEntrySuccessorOperands(regions.front()));
716  unsigned numMemrefOperands = llvm::count_if(entryOperands, isMemref);
717 
718  // No ownership is acquired for any MemRefs that are passed to the region from
719  // the outside.
720  Value falseVal = buildBoolValue(builder, op.getLoc(), false);
721  op->insertOperands(op->getNumOperands(),
722  SmallVector<Value>(numMemrefOperands, falseVal));
723 
724  int counter = op->getNumResults();
725  unsigned numMemrefResults = llvm::count_if(op->getResults(), isMemref);
726  SmallVector<Type> ownershipResults(numMemrefResults, builder.getI1Type());
727  RegionBranchOpInterface newOp = appendOpResults(op, ownershipResults);
728 
729  for (auto result : llvm::make_filter_range(newOp->getResults(), isMemref)) {
730  state.updateOwnership(result, newOp->getResult(counter++));
731  state.addMemrefToDeallocate(result, newOp->getBlock());
732  }
733 
734  return newOp.getOperation();
735 }
736 
737 Value BufferDeallocation::materializeMemrefWithGuaranteedOwnership(
738  OpBuilder &builder, Value memref, Block *block) {
739  // First, make sure we at least have 'Unique' ownership already.
740  std::pair<Value, Value> newMemrefAndOnwership =
741  materializeUniqueOwnership(builder, memref, block);
742  Value newMemref = newMemrefAndOnwership.first;
743  Value condition = newMemrefAndOnwership.second;
744 
745  // Avoid inserting additional IR if ownership is already guaranteed. In
746  // particular, this is already the case when we had 'Unknown' ownership
747  // initially and a clone was inserted to get to 'Unique' ownership.
748  if (matchPattern(condition, m_One()))
749  return newMemref;
750 
751  // Insert a runtime check and only clone if we still don't have ownership at
752  // runtime.
753  Value maybeClone = scf::IfOp::create(
754  builder, memref.getLoc(), condition,
755  [&](OpBuilder &builder, Location loc) {
756  scf::YieldOp::create(builder, loc, newMemref);
757  },
758  [&](OpBuilder &builder, Location loc) {
759  Value clone = bufferization::CloneOp::create(
760  builder, loc, newMemref);
761  scf::YieldOp::create(builder, loc, clone);
762  })
763  .getResult(0);
764  Value trueVal = buildBoolValue(builder, memref.getLoc(), true);
765  state.updateOwnership(maybeClone, trueVal);
766  state.addMemrefToDeallocate(maybeClone, maybeClone.getParentBlock());
767  return maybeClone;
768 }
769 
770 FailureOr<Operation *>
771 BufferDeallocation::handleInterface(BranchOpInterface op) {
772  if (op->getNumSuccessors() > 1)
773  return op->emitError("BranchOpInterface operations with multiple "
774  "successors are not supported yet");
775 
776  if (op->getNumSuccessors() != 1)
777  return emitError(op.getLoc(),
778  "only BranchOpInterface operations with exactly "
779  "one successor are supported yet");
780 
781  if (op.getSuccessorOperands(0).getProducedOperandCount() > 0)
782  return op.emitError("produced operands are not supported");
783 
784  // Collect the values to deallocate and retain and use them to create the
785  // dealloc operation.
786  Block *block = op->getBlock();
787  OpBuilder builder(op);
788  SmallVector<Value> memrefs, conditions, toRetain;
789  if (failed(state.getMemrefsAndConditionsToDeallocate(
790  builder, op.getLoc(), block, memrefs, conditions)))
791  return failure();
792 
793  OperandRange forwardedOperands =
794  op.getSuccessorOperands(0).getForwardedOperands();
795  state.getMemrefsToRetain(block, op->getSuccessor(0), forwardedOperands,
796  toRetain);
797 
798  auto deallocOp = bufferization::DeallocOp::create(
799  builder, op.getLoc(), memrefs, conditions, toRetain);
800 
801  // We want to replace the current ownership of the retained values with the
802  // result values of the dealloc operation as they are always unique.
803  state.resetOwnerships(deallocOp.getRetained(), block);
804  for (auto [retained, ownership] :
805  llvm::zip(deallocOp.getRetained(), deallocOp.getUpdatedConditions())) {
806  state.updateOwnership(retained, ownership, block);
807  }
808 
809  unsigned numAdditionalReturns = llvm::count_if(forwardedOperands, isMemref);
810  SmallVector<Value> newOperands(forwardedOperands);
811  auto additionalConditions =
812  deallocOp.getUpdatedConditions().take_front(numAdditionalReturns);
813  newOperands.append(additionalConditions.begin(), additionalConditions.end());
814  op.getSuccessorOperands(0).getMutableForwardedOperands().assign(newOperands);
815 
816  return op.getOperation();
817 }
818 
819 FailureOr<Operation *> BufferDeallocation::handleInterface(CallOpInterface op) {
820  OpBuilder builder(op);
821 
822  // Lookup the function operation and check if it has private visibility. If
823  // the function is referenced by SSA value instead of a Symbol, it's assumed
824  // to be public. (And we cannot easily change the type of the SSA value
825  // anyway.)
826  Operation *funcOp = op.resolveCallableInTable(state.getSymbolTable());
827  bool isPrivate = false;
828  if (auto symbol = dyn_cast_or_null<SymbolOpInterface>(funcOp))
829  isPrivate = symbol.isPrivate() && !symbol.isDeclaration();
830 
831  // If the private-function-dynamic-ownership option is enabled and we are
832  // calling a private function, we need to add an additional `i1` result for
833  // each MemRef result to dynamically pass the current ownership indicator
834  // rather than adhering to the function boundary ABI.
835  if (options.privateFuncDynamicOwnership && isPrivate) {
836  unsigned numMemrefs = llvm::count_if(op->getResults(), isMemref);
837  SmallVector<Type> ownershipTypesToAppend(numMemrefs, builder.getI1Type());
838  unsigned ownershipCounter = op->getNumResults();
839  op = appendOpResults(op, ownershipTypesToAppend);
840 
841  for (auto result : llvm::make_filter_range(op->getResults(), isMemref)) {
842  state.updateOwnership(result, op->getResult(ownershipCounter++));
843  state.addMemrefToDeallocate(result, result.getParentBlock());
844  }
845 
846  return op.getOperation();
847  }
848 
849  // According to the function boundary ABI we are guaranteed to get ownership
850  // of all MemRefs returned by the function. Thus we set ownership to constant
851  // 'true' and remember to deallocate it.
852  Value trueVal = buildBoolValue(builder, op.getLoc(), true);
853  for (auto result : llvm::make_filter_range(op->getResults(), isMemref)) {
854  state.updateOwnership(result, trueVal);
855  state.addMemrefToDeallocate(result, result.getParentBlock());
856  }
857 
858  return op.getOperation();
859 }
860 
861 FailureOr<Operation *>
862 BufferDeallocation::handleInterface(MemoryEffectOpInterface op) {
863  auto *block = op->getBlock();
864  OpBuilder builder = OpBuilder::atBlockBegin(block);
865 
866  for (auto operand : llvm::make_filter_range(op->getOperands(), isMemref)) {
867  if (op.getEffectOnValue<MemoryEffects::Free>(operand).has_value()) {
868  // The bufferization.manual_deallocation attribute can be attached to ops
869  // with an allocation and/or deallocation side effect. It indicates that
870  // the op is under a "manual deallocation" scheme. Deallocation ops are
871  // usually forbidden in the input IR (not supported by the buffer
872  // deallocation pass). However, if they are under manual deallocation,
873  // they can be safely ignored by the buffer deallocation pass.
874  if (!op->hasAttr(BufferizationDialect::kManualDeallocation))
875  return op->emitError(
876  "memory free side-effect on MemRef value not supported!");
877 
878  // Buffers that were allocated under "manual deallocation" may be
879  // manually deallocated. We insert a runtime assertion to cover certain
880  // cases of invalid IR where an automatically managed buffer allocation
881  // is manually deallocated. This is not a bulletproof check!
882  OpBuilder::InsertionGuard g(builder);
883  builder.setInsertionPoint(op);
884  Ownership ownership = state.getOwnership(operand, block);
885  if (ownership.isUnique()) {
886  Value ownershipInverted = arith::XOrIOp::create(
887  builder, op.getLoc(), ownership.getIndicator(),
888  buildBoolValue(builder, op.getLoc(), true));
889  cf::AssertOp::create(builder, op.getLoc(), ownershipInverted,
890  "expected that the block does not have ownership");
891  }
892  }
893  }
894 
895  for (auto res : llvm::make_filter_range(op->getResults(), isMemref)) {
896  auto allocEffect = op.getEffectOnValue<MemoryEffects::Allocate>(res);
897  if (allocEffect.has_value()) {
898  if (isa<SideEffects::AutomaticAllocationScopeResource>(
899  allocEffect->getResource())) {
900  // Make sure that the ownership of auto-managed allocations is set to
901  // false. This is important for operations that have at least one memref
902  // typed operand. E.g., consider an operation like `bufferization.clone`
903  // that lowers to a `memref.alloca + memref.copy` instead of a
904  // `memref.alloc`. If we wouldn't set the ownership of the result here,
905  // the default ownership population in `populateRemainingOwnerships`
906  // would assume aliasing with the MemRef operand.
907  state.resetOwnerships(res, block);
908  state.updateOwnership(res, buildBoolValue(builder, op.getLoc(), false));
909  continue;
910  }
911 
912  if (op->hasAttr(BufferizationDialect::kManualDeallocation)) {
913  // This allocation will be deallocated manually. Assign an ownership of
914  // "false", so that it will never be deallocated by the buffer
915  // deallocation pass.
916  state.resetOwnerships(res, block);
917  state.updateOwnership(res, buildBoolValue(builder, op.getLoc(), false));
918  continue;
919  }
920 
921  state.updateOwnership(res, buildBoolValue(builder, op.getLoc(), true));
922  state.addMemrefToDeallocate(res, block);
923  }
924  }
925 
926  return op.getOperation();
927 }
928 
929 FailureOr<Operation *>
930 BufferDeallocation::handleInterface(RegionBranchTerminatorOpInterface op) {
931  OpBuilder builder(op);
932 
933  // If this is a return operation of a function that is not private or the
934  // dynamic function boundary ownership is disabled, we need to return memref
935  // values for which we have guaranteed ownership to pass on to adhere to the
936  // function boundary ABI.
937  bool funcWithoutDynamicOwnership =
938  isFunctionWithoutDynamicOwnership(op->getParentOp());
939  if (funcWithoutDynamicOwnership) {
940  for (OpOperand &val : op->getOpOperands()) {
941  if (!isMemref(val.get()))
942  continue;
943 
944  val.set(materializeMemrefWithGuaranteedOwnership(builder, val.get(),
945  op->getBlock()));
946  }
947  }
948 
949  // TODO: getSuccessorRegions is not implemented by all operations we care
950  // about, but we would need to check how many successors there are and under
951  // which condition they are taken, etc.
952 
953  MutableOperandRange operands =
954  op.getMutableSuccessorOperands(RegionBranchPoint::parent());
955 
956  SmallVector<Value> updatedOwnerships;
958  state, op, operands.getAsOperandRange(), updatedOwnerships);
959  if (failed(result) || !*result)
960  return result;
961 
962  // Add an additional operand for every MemRef for the ownership indicator.
963  if (!funcWithoutDynamicOwnership) {
964  SmallVector<Value> newOperands{operands.getAsOperandRange()};
965  newOperands.append(updatedOwnerships.begin(), updatedOwnerships.end());
966  operands.assign(newOperands);
967  }
968 
969  return op.getOperation();
970 }
971 
972 bool BufferDeallocation::isFunctionWithoutDynamicOwnership(Operation *op) {
973  auto funcOp = dyn_cast<FunctionOpInterface>(op);
974  return funcOp && (!options.privateFuncDynamicOwnership ||
975  !funcOp.isPrivate() || funcOp.isExternal());
976 }
977 
978 void BufferDeallocation::populateRemainingOwnerships(Operation *op) {
979  for (auto res : op->getResults()) {
980  if (!isMemref(res))
981  continue;
982  if (!state.getOwnership(res, op->getBlock()).isUninitialized())
983  continue;
984 
985  // The op does not allocate memory, otherwise, it would have been assigned
986  // an ownership during `handleInterface`. Assume the result may alias with
987  // any memref operand and thus combine all their ownerships.
988  for (auto operand : op->getOperands()) {
989  if (!isMemref(operand))
990  continue;
991 
992  state.updateOwnership(
993  res, state.getOwnership(operand, operand.getParentBlock()),
994  op->getBlock());
995  }
996 
997  // If the ownership value is still uninitialized (e.g., because the op has
998  // no memref operands), assume that no ownership is taken. E.g., this is the
999  // case for "memref.get_global".
1000  //
1001  // Note: This can lead to memory leaks if memory side effects are not
1002  // properly specified on the op.
1003  if (state.getOwnership(res, op->getBlock()).isUninitialized()) {
1004  OpBuilder builder(op);
1005  state.updateOwnership(res, buildBoolValue(builder, op->getLoc(), false));
1006  }
1007  }
1008 }
1009 
1010 //===----------------------------------------------------------------------===//
1011 // OwnershipBasedBufferDeallocationPass
1012 //===----------------------------------------------------------------------===//
1013 
1014 namespace {
1015 
1016 /// The actual buffer deallocation pass that inserts and moves dealloc nodes
1017 /// into the right positions. Furthermore, it inserts additional clones if
1018 /// necessary. It uses the algorithm described at the top of the file.
1019 struct OwnershipBasedBufferDeallocationPass
1020  : public bufferization::impl::OwnershipBasedBufferDeallocationPassBase<
1021  OwnershipBasedBufferDeallocationPass> {
1022  using Base::Base;
1023 
1024  void runOnOperation() override {
1026  options.privateFuncDynamicOwnership = privateFuncDynamicOwnership;
1027 
1028  mlir::SymbolTableCollection symbolTables;
1029 
1030  auto status = getOperation()->walk([&](func::FuncOp func) {
1031  if (func.isExternal())
1032  return WalkResult::skip();
1033 
1034  if (failed(deallocateBuffersOwnershipBased(func, options, symbolTables)))
1035  return WalkResult::interrupt();
1036 
1037  return WalkResult::advance();
1038  });
1039  if (status.wasInterrupted())
1040  signalPassFailure();
1041  }
1042 };
1043 
1044 } // namespace
1045 
1046 //===----------------------------------------------------------------------===//
1047 // Implement bufferization API
1048 //===----------------------------------------------------------------------===//
1049 
1051  FunctionOpInterface op, DeallocationOptions options,
1052  SymbolTableCollection &symbolTables) {
1053  // Gather all required allocation nodes and prepare the deallocation phase.
1054  BufferDeallocation deallocation(op, options, symbolTables);
1055 
1056  // Place all required temporary clone and dealloc nodes.
1057  return deallocation.deallocate(op);
1058 }
static bool isMemref(Value v)
static Value buildBoolValue(OpBuilder &builder, Location loc, bool value)
static bool hasBufferSemantics(Operation *op)
Return "true" if the given op has buffer semantics.
static bool hasNeitherAllocateNorFreeSideEffect(Operation *op)
Return "true" if the given op is guaranteed to have neither "Allocate" nor "Free" side effects.
static llvm::ManagedStatic< PassManagerOptions > options
This class represents an argument of a Block.
Definition: Value.h:309
Location getLoc() const
Return the location for this argument.
Definition: Value.h:324
Block represents an ordered list of Operations.
Definition: Block.h:33
BlockArgument getArgument(unsigned i)
Definition: Block.h:129
unsigned getNumArguments()
Definition: Block.h:128
Region * getParent() const
Provide a 'getParent' method for ilist_node_with_parent methods.
Definition: Block.cpp:27
SuccessorRange getSuccessors()
Definition: Block.h:267
BlockArgument addArgument(Type type, Location loc)
Add one value to the argument list.
Definition: Block.cpp:153
OpListType & getOperations()
Definition: Block.h:137
bool isEntryBlock()
Return if this block is the entry block in the parent region.
Definition: Block.cpp:36
Operation * getParentOp()
Returns the closest surrounding operation that contains this block.
Definition: Block.cpp:31
BoolAttr getBoolAttr(bool value)
Definition: Builders.cpp:95
IntegerType getI1Type()
Definition: Builders.cpp:52
This class defines the main interface for locations in MLIR and acts as a non-nullable wrapper around...
Definition: Location.h:76
This class provides a mutable adaptor for a range of operands.
Definition: ValueRange.h:118
OperandRange getAsOperandRange() const
Explicit conversion to an OperandRange.
void assign(ValueRange values)
Assign this range to the given values.
RAII guard to reset the insertion point of the builder when destroyed.
Definition: Builders.h:346
This class helps build Operations.
Definition: Builders.h:205
static OpBuilder atBlockBegin(Block *block, Listener *listener=nullptr)
Create a builder and set the insertion point to before the first operation in the block but still ins...
Definition: Builders.h:238
void setInsertionPoint(Block *block, Block::iterator insertPoint)
Set the insertion point to the specified location.
Definition: Builders.h:396
Operation * insert(Operation *op)
Insert the given operation at the current insertion point and return it.
Definition: Builders.cpp:416
This class represents an operand of an operation.
Definition: Value.h:257
This trait indicates that the memory effects of an operation includes the effects of operations neste...
This class provides the API for ops that are known to be terminators.
Definition: OpDefinition.h:773
This class indicates that the regions associated with this op don't have terminators.
Definition: OpDefinition.h:769
This class implements the operand iterators for the Operation class.
Definition: ValueRange.h:43
Operation is the basic unit of execution within MLIR.
Definition: Operation.h:88
DictionaryAttr getAttrDictionary()
Return all of the attributes on this operation as a DictionaryAttr.
Definition: Operation.cpp:295
bool hasTrait()
Returns true if the operation was registered with a particular trait, e.g.
Definition: Operation.h:749
unsigned getNumRegions()
Returns the number of regions held by this operation.
Definition: Operation.h:674
Location getLoc()
The source location the operation was defined or derived from.
Definition: Operation.h:223
static Operation * create(Location location, OperationName name, TypeRange resultTypes, ValueRange operands, NamedAttrList &&attributes, OpaqueProperties properties, BlockRange successors, unsigned numRegions)
Create a new Operation with the specific fields.
Definition: Operation.cpp:66
InFlightDiagnostic emitError(const Twine &message={})
Emit an error about fatal conditions with this operation, reporting up to any diagnostic handlers tha...
Definition: Operation.cpp:267
Block * getBlock()
Returns the operation block that contains this operation.
Definition: Operation.h:213
MutableArrayRef< Region > getRegions()
Returns the regions held by this operation.
Definition: Operation.h:677
OperationName getName()
The name of an operation is the key identifier for it.
Definition: Operation.h:119
result_type_range getResultTypes()
Definition: Operation.h:428
operand_range getOperands()
Returns an iterator on the underlying Value's.
Definition: Operation.h:378
void replaceAllUsesWith(ValuesT &&values)
Replace all uses of results of this operation with the provided 'values'.
Definition: Operation.h:272
SuccessorRange getSuccessors()
Definition: Operation.h:703
result_range getResults()
Definition: Operation.h:415
OpaqueProperties getPropertiesStorage()
Returns the properties storage.
Definition: Operation.h:900
void erase()
Remove this operation from its parent block and delete it.
Definition: Operation.cpp:538
unsigned getNumResults()
Return the number of results held by this operation.
Definition: Operation.h:404
static constexpr RegionBranchPoint parent()
Returns an instance of RegionBranchPoint representing the parent operation.
This class contains a list of basic blocks and a link to the parent operation it is attached to.
Definition: Region.h:26
This class represents a collection of SymbolTables.
Definition: SymbolTable.h:283
This class provides an abstraction over the various different ranges of value types.
Definition: TypeRange.h:37
This class represents an instance of an SSA value in the MLIR system, representing a computable value...
Definition: Value.h:96
Type getType() const
Return the type of this value.
Definition: Value.h:105
Block * getParentBlock()
Return the Block in which this Value is defined.
Definition: Value.cpp:46
Location getLoc() const
Return the location of this value.
Definition: Value.cpp:24
Operation * getDefiningOp() const
If this value is the result of an operation, return the operation that defines it.
Definition: Value.cpp:18
static WalkResult skip()
Definition: WalkResult.h:48
static WalkResult advance()
Definition: WalkResult.h:47
static WalkResult interrupt()
Definition: WalkResult.h:46
This class collects all the state that we need to perform the buffer deallocation pass with associate...
This class is used to track the ownership of values.
bool isUnique() const
Check if this ownership value is in the 'Unique' state.
Value getIndicator() const
If this ownership value is in 'Unique' state, this function can be used to get the indicator paramete...
FailureOr< Operation * > insertDeallocOpForReturnLike(DeallocationState &state, Operation *op, ValueRange operands, SmallVectorImpl< Value > &updatedOperandOwnerships)
Insert a bufferization.dealloc operation right before op which has to be a terminator without any suc...
LogicalResult deallocateBuffersOwnershipBased(FunctionOpInterface op, DeallocationOptions options, SymbolTableCollection &symbolTables)
Run the ownership-based buffer deallocation.
Include the generated interface declarations.
bool matchPattern(Value value, const Pattern &pattern)
Entry point for matching a pattern over a Value.
Definition: Matchers.h:490
InFlightDiagnostic emitError(Location loc)
Utility method to emit an error message using this location.
detail::constant_int_predicate_matcher m_One()
Matches a constant scalar / vector splat / tensor splat integer one.
Definition: Matchers.h:478
auto get(MLIRContext *context, Ts &&...params)
Helper method that injects context only if needed, this helps unify some of the attribute constructio...
This iterator enumerates elements according to their dominance relationship.
Definition: Iterators.h:48
The following effect indicates that the operation allocates from some resource.
The following effect indicates that the operation frees some resource that has been allocated.
Options for BufferDeallocationOpInterface-based buffer deallocation.