Namespaces
namespace	deallocation_impl
namespace	detail
namespace	func_ext
namespace	impl

Classes
struct	BranchOpBufferizableOpInterfaceExternalModel
	A template that provides a default implementation of getAliasingValues for ops that implement the BranchOpInterface. More...
struct	BufferDeallocationPipelineOptions
	Options for the buffer deallocation pipeline. More...
struct	BufferizationStatistics
	Bufferization statistics for debugging. More...
class	BufferPlacementAllocs
	A simple analysis that detects allocation operations. More...
class	BufferPlacementTransformationBase
	The base class for all BufferPlacement transformations. More...
struct	BufferResultsToOutParamsOpts
struct	BufferResultsToOutParamsPassOptions
struct	DeallocationOptions
	Options for BufferDeallocationOpInterface-based buffer deallocation. More...
class	DeallocationState
	This class collects all the state that we need to perform the buffer deallocation pass with associated helper functions such that we have easy access to it in the BufferDeallocationOpInterface implementations and the BufferDeallocation pass. More...
struct	DstBufferizableOpInterfaceExternalModel
	Bufferizable ops that implement the DestinationStyleOpInterface can use this external model base class. More...
class	OneShotAnalysisState
	State for analysis-enabled bufferization. More...
struct	OneShotBufferizationOptions
	Options for analysis-enabled bufferization. More...
struct	OneShotBufferizePassOptions
struct	OpWithUnstructuredControlFlowBufferizableOpInterfaceExternalModel
	A template that provides a default implementation of getAliasingOpOperands for ops that support unstructured control flow within their regions. More...
class	Ownership
	This class is used to track the ownership of values. More...
struct	OwnershipBasedBufferDeallocationPassOptions
struct	PromoteBuffersToStackPassOptions
struct	ValueComparator
	Compare two SSA values in a deterministic manner. More...

Typedefs
using	RegisterDependenciesFn = std::function<void(ValueRange, ValueRange)>
using	DeallocHelperMap = llvm::DenseMap<Operation *, func::FuncOp>
	Maps from symbol table to its corresponding dealloc helper function.
using	ControlBuildSubsetExtractionFn
	A function type that defines a callback to control the construction of the subset extraction of the SubsetInsertionOpInterface.

Functions
void	populateDynamicDimSizes (OpBuilder &b, Location loc, Value shapedValue, SmallVector< Value > &dynamicDims)
	Populate dynamicDims with tensor::DimOp / memref::DimOp results for all dynamic dimensions of the given shaped value.
FailureOr< Value >	castOrReallocMemRefValue (OpBuilder &b, Value value, MemRefType type, const BufferizationOptions &options)
	Try to cast the given ranked MemRef-typed value to the given ranked MemRef type.
LogicalResult	foldToBufferToTensorPair (RewriterBase &rewriter, ToBufferOp toBuffer, const BufferizationOptions &options)
	Try to fold to_buffer(to_tensor(x)).
void	populateDeallocOpCanonicalizationPatterns (RewritePatternSet &patterns, MLIRContext *context)
	Add the canonicalization patterns for bufferization.dealloc to the given pattern set to make them available to other passes (such as BufferDeallocationSimplification).
void	buildBufferDeallocationPipeline (OpPassManager &pm, const BufferDeallocationPipelineOptions &options)
	Adds the buffer deallocation pipeline to the OpPassManager.
void	registerBufferizationPipelines ()
	Registers all pipelines for the bufferization dialect.
void	registerTransformDialectExtension (DialectRegistry &registry)
LogicalResult	bufferizeOp (Operation op, const BufferizationOptions &options, BufferizationState &bufferizationState, BufferizationStatistics statistics=nullptr)
	Bufferize op and its nested ops that implement BufferizableOpInterface.
LogicalResult	bufferizeBlockSignature (Block *block, RewriterBase &rewriter, const BufferizationOptions &options, BufferizationState &state)
	Bufferize the signature of block and its callers (i.e., ops that have the given block as a successor).
template<typename DominatorT>
Block *	findCommonDominator (Value value, const BufferViewFlowAnalysis::ValueSetT &values, const DominatorT &doms)
	Finds a common dominator for the given value while taking the positions of the values in the value set into account.
FailureOr< memref::GlobalOp >	getGlobalFor (arith::ConstantOp constantOp, SymbolTableCollection &symbolTables, uint64_t alignment, Attribute memorySpace={})
void	removeSymbol (Operation *op, BufferizationState &state)
void	insertSymbol (Operation *op, BufferizationState &state)
SmallVector< func::ReturnOp >	getReturnOps (func::FuncOp funcOp)
	Helper function that returns all func.return ops in the given function.
LogicalResult	analyzeOp (Operation op, OneShotAnalysisState &state, BufferizationStatistics statistics=nullptr)
	Analyze op and its nested ops.
LogicalResult	runOneShotBufferize (Operation op, const OneShotBufferizationOptions &options, BufferizationState &state, BufferizationStatistics statistics=nullptr)
	Run One-Shot Bufferize on the given op: Analysis + Bufferization.
llvm::LogicalResult	analyzeModuleOp (Operation moduleOp, OneShotAnalysisState &state, BufferizationStatistics statistics=nullptr)
	Analyze moduleOp and its nested ops.
llvm::LogicalResult	bufferizeModuleOp (Operation moduleOp, const OneShotBufferizationOptions &options, BufferizationState &state, BufferizationStatistics statistics=nullptr)
	Bufferize an ops nested ops that implement BufferizableOpInterface.
void	removeBufferizationAttributesInModule (Operation *moduleOp)
	Remove bufferization attributes on every FuncOp arguments in the SymbolTable op.
llvm::LogicalResult	runOneShotModuleBufferize (Operation moduleOp, const bufferization::OneShotBufferizationOptions &options, BufferizationState &state, BufferizationStatistics statistics=nullptr)
	Run One-Shot Module Bufferization on the given SymbolTable.
std::unique_ptr<::mlir::Pass >	createBufferDeallocationSimplificationPass ()
std::unique_ptr<::mlir::Pass >	createBufferHoistingPass ()
std::unique_ptr<::mlir::Pass >	createBufferLoopHoistingPass ()
std::unique_ptr<::mlir::Pass >	createBufferResultsToOutParamsPass ()
std::unique_ptr<::mlir::Pass >	createBufferResultsToOutParamsPass (BufferResultsToOutParamsPassOptions options)
std::unique_ptr<::mlir::Pass >	createDropEquivalentBufferResultsPass ()
std::unique_ptr<::mlir::Pass >	createEmptyTensorEliminationPass ()
std::unique_ptr<::mlir::Pass >	createEmptyTensorToAllocTensorPass ()
std::unique_ptr<::mlir::Pass >	createLowerDeallocationsPass ()
std::unique_ptr<::mlir::Pass >	createOneShotBufferizePass ()
std::unique_ptr<::mlir::Pass >	createOneShotBufferizePass (OneShotBufferizePassOptions options)
std::unique_ptr<::mlir::Pass >	createOptimizeAllocationLivenessPass ()
std::unique_ptr<::mlir::Pass >	createOwnershipBasedBufferDeallocationPass ()
std::unique_ptr<::mlir::Pass >	createOwnershipBasedBufferDeallocationPass (OwnershipBasedBufferDeallocationPassOptions options)
std::unique_ptr<::mlir::Pass >	createPromoteBuffersToStackPass ()
std::unique_ptr<::mlir::Pass >	createPromoteBuffersToStackPass (PromoteBuffersToStackPassOptions options)
void	populateBufferizationDeallocLoweringPattern (RewritePatternSet &patterns, const DeallocHelperMap &deallocHelperFuncMap)
	Adds the conversion pattern of the bufferization.dealloc operation to the given pattern set for use in other transformation passes.
func::FuncOp	buildDeallocationLibraryFunction (OpBuilder &builder, Location loc, SymbolTable &symbolTable)
	Construct the library function needed for the fully generic bufferization.dealloc lowering implemented in the LowerDeallocations pass.
LogicalResult	deallocateBuffersOwnershipBased (FunctionOpInterface op, DeallocationOptions options, SymbolTableCollection &symbolTables)
	Run the ownership-based buffer deallocation.
LogicalResult	promoteBufferResultsToOutParams (ModuleOp module, const BufferResultsToOutParamsOpts &options)
	Replace buffers that are returned from a function with an out parameter.
LogicalResult	dropEquivalentBufferResults (ModuleOp module)
	Drop all memref function results that are equivalent to a function argument.
std::unique_ptr< Pass >	createPromoteBuffersToStackPass (std::function< bool(Value)> isSmallAlloc)
	Creates a pass that promotes heap-based allocations to stack-based ones.
void	registerBufferDeallocationSimplificationPass ()
void	registerBufferDeallocationSimplificationPassPass ()
void	registerBufferHoistingPass ()
void	registerBufferHoistingPassPass ()
void	registerBufferLoopHoistingPass ()
void	registerBufferLoopHoistingPassPass ()
void	registerBufferResultsToOutParamsPass ()
void	registerBufferResultsToOutParamsPassPass ()
void	registerDropEquivalentBufferResultsPass ()
void	registerDropEquivalentBufferResultsPassPass ()
void	registerEmptyTensorEliminationPass ()
void	registerEmptyTensorEliminationPassPass ()
void	registerEmptyTensorToAllocTensorPass ()
void	registerEmptyTensorToAllocTensorPassPass ()
void	registerLowerDeallocationsPass ()
void	registerLowerDeallocationsPassPass ()
void	registerOneShotBufferizePass ()
void	registerOneShotBufferizePassPass ()
void	registerOptimizeAllocationLivenessPass ()
void	registerOptimizeAllocationLivenessPassPass ()
void	registerOwnershipBasedBufferDeallocationPass ()
void	registerOwnershipBasedBufferDeallocationPassPass ()
void	registerPromoteBuffersToStackPass ()
void	registerPromoteBuffersToStackPassPass ()
void	registerBufferizationPasses ()
LogicalResult	eliminateEmptyTensors (RewriterBase &rewriter, Operation *op)
	Try to eliminate "tensor.empty" ops inside op.
Value	buildSubsetExtraction (RewriterBase &rewriter, SubsetInsertionOpInterface op, tensor::EmptyOp emptyTensorOp, Operation *user)
	This method builds and returns a subset extraction value for the destination tensor that the given op inserts into.
LogicalResult	eliminateEmptyTensors (RewriterBase &rewriter, Operation *op, OneShotAnalysisState &state, ControlBuildSubsetExtractionFn subsetsExtractionFn=buildSubsetExtraction)
	Try to eliminate "tensor.empty" ops inside op.
void	hoistBuffersFromLoops (Operation *op)
	Within the given operation, hoist buffers from loops where possible.
LogicalResult	insertTensorCopies (Operation op, const OneShotBufferizationOptions &options, const BufferizationState &bufferizationState, BufferizationStatistics statistics=nullptr)
	Resolve RaW and other conflicts by inserting bufferization.alloc_tensor ops.
LogicalResult	insertTensorCopies (Operation *op, const AnalysisState &analysisState, const BufferizationState &bufferizationState)
	Resolve RaW and other conflicts by inserting bufferization.alloc_tensor ops.
void	populateEmptyTensorToAllocTensorPattern (RewritePatternSet &patterns)
	Populate patterns to lower tensor.empty ops to bufferization.alloc_tensor ops.

Typedef Documentation

◆ ControlBuildSubsetExtractionFn

using mlir::bufferization::ControlBuildSubsetExtractionFn

Initial value:

 
    std::function<Value(RewriterBase &, SubsetInsertionOpInterface,
                        tensor::EmptyOp emptyTensorOp, Operation *user)>

A function type that defines a callback to control the construction of the subset extraction of the SubsetInsertionOpInterface.

The subset extraction value can be used as a replacement for the emptyTensorOp value which is being consumed by user, failing of building such a value should be indicated with an empty value. This function should guarantee the legality of the replacement, i.e. the replacement should dominate the user of the emptyTensorOp being eliminated.

Definition at line 47 of file Transforms.h.

◆ DeallocHelperMap

using mlir::bufferization::DeallocHelperMap = llvm::DenseMap<Operation *, func::FuncOp>

Maps from symbol table to its corresponding dealloc helper function.

Definition at line 25 of file Passes.h.

◆ RegisterDependenciesFn

using mlir::bufferization::RegisterDependenciesFn = std::function<void(ValueRange, ValueRange)>

Definition at line 20 of file BufferViewFlowOpInterface.h.

Function Documentation

◆ analyzeModuleOp()

LogicalResult mlir::bufferization::analyzeModuleOp	(	Operation *	moduleOp,
		OneShotAnalysisState &	state,
		BufferizationStatistics *	statistics = nullptr )

Analyze moduleOp and its nested ops.

Bufferization decisions are stored in state. This operates on any SymbolTable op.

Definition at line 451 of file OneShotModuleBufferize.cpp.

References mlir::bufferization::func_ext::Analyzed, mlir::bufferization::func_ext::FuncAnalysisState::analyzedFuncOps, analyzeOp(), getFuncOpsOrderedByCalls(), mlir::bufferization::OneShotAnalysisState::getOptions(), getOrCreateFuncAnalysisState(), mlir::bufferization::func_ext::FuncAnalysisState::startFunctionAnalysis(), success(), and mlir::bufferization::func_ext::FuncAnalysisState::symbolTables.

Referenced by eliminateEmptyTensors(), and insertTensorCopies().

◆ analyzeOp()

LogicalResult mlir::bufferization::analyzeOp	(	Operation *	op,
		OneShotAnalysisState &	state,
		BufferizationStatistics *	statistics = nullptr )

Analyze op and its nested ops.

Bufferization decisions are stored in state.

Definition at line 1311 of file OneShotAnalysis.cpp.

References mlir::bufferization::OneShotAnalysisState::analyzeOp(), annotateOpsWithAliasSets(), annotateOpsWithBufferizationMarkers(), checkPreBufferizationAssumptions(), mlir::bufferization::OneShotAnalysisState::gatherUndefinedTensorUses(), mlir::bufferization::OneShotAnalysisState::getOptions(), mlir::bufferization::OneShotAnalysisState::getStatNumTensorInPlace(), mlir::bufferization::OneShotAnalysisState::getStatNumTensorOutOfPlace(), mlir::bufferization::BufferizationStatistics::numTensorInPlace, mlir::bufferization::BufferizationStatistics::numTensorOutOfPlace, options, success(), and mlir::Operation::walk().

Referenced by analyzeModuleOp(), eliminateEmptyTensors(), and insertTensorCopies().

◆ bufferizeBlockSignature()

LogicalResult mlir::bufferization::bufferizeBlockSignature	(	Block *	block,
		RewriterBase &	rewriter,
		const BufferizationOptions &	options,
		BufferizationState &	state )

Bufferize the signature of block and its callers (i.e., ops that have the given block as a successor).

All block argument types are changed to memref types. All corresponding operands of all callers are wrapped in bufferization.to_buffer ops. All uses of bufferized tensor block arguments are wrapped in bufferization.to_tensor ops.

It is expected that all callers implement the BranchOpInterface. Otherwise, this function will fail. The BranchOpInterface is used to query the range of operands that are forwarded to this block.

It is expected that the parent op of this block implements the BufferizableOpInterface. The buffer types of tensor block arguments are computed with BufferizableOpIntercace::getBufferType.

Definition at line 393 of file Bufferize.cpp.

References mlir::MutableOperandRange::assign(), mlir::Operation::emitOpError(), mlir::Block::getArguments(), mlir::SuccessorOperands::getForwardedOperands(), mlir::SuccessorOperands::getMutableForwardedOperands(), mlir::Block::getParentOp(), mlir::Operation::getSuccessors(), mlir::IRObjectWithUseList< OperandType >::getUsers(), options, mlir::OpBuilder::setInsertionPointAfterValue(), mlir::OpBuilder::setInsertionPointToStart(), and success().

Referenced by mlir::bufferization::func_ext::FuncOpInterface::bufferize().

◆ bufferizeModuleOp()

LogicalResult mlir::bufferization::bufferizeModuleOp	(	Operation *	moduleOp,
		const OneShotBufferizationOptions &	options,
		BufferizationState &	state,
		BufferizationStatistics *	statistics = nullptr )

Bufferize an ops nested ops that implement BufferizableOpInterface.

This operates on any SymbolTable op.

Note: This function does not run One-Shot Analysis. No buffer copies are inserted except two cases:

options.copyBeforeWrite is set, in which case buffers are copied before every write.
options.copyBeforeWrite is not set and options.noAnalysisFuncFilter is not empty. The FuncOps it contains were not analyzed. Buffer copies will be inserted only to these FuncOps.

Definition at line 527 of file OneShotModuleBufferize.cpp.

References bufferizeOp(), foldMemRefCasts(), mlir::Operation::getContext(), getFuncOpsOrderedByCalls(), mlir::Operation::getRegions(), mlir::Operation::hasTrait(), options, removeBufferizationAttributesInModule(), and success().

Referenced by mlir::sparse_tensor::SparsificationAndBufferizationPass::runDenseBufferization(), and runOneShotModuleBufferize().

◆ bufferizeOp()

LogicalResult mlir::bufferization::bufferizeOp	(	Operation *	op,
		const BufferizationOptions &	options,
		BufferizationState &	bufferizationState,
		BufferizationStatistics *	statistics = nullptr )

Bufferize op and its nested ops that implement BufferizableOpInterface.

Note: This function does not resolve read-after-write conflicts. Use this function only if it is guaranteed that the input IR can bufferize without additional buffer copies or set "options.copyBeforeWrite = true". The general bufferization entry point is runOneShotBufferize.

Check the result of bufferization. Return an error if an op was not bufferized, unless partial bufferization is allowed.

Definition at line 277 of file Bufferize.cpp.

References mlir::WalkResult::advance(), mlir::Operation::emitError(), mlir::Operation::emitOpError(), foldToBufferToTensorPair(), mlir::Operation::getContext(), mlir::Operation::getName(), mlir::Operation::getRegions(), mlir::Operation::getUses(), insertTensorCopies(), mlir::isMemoryEffectFree(), options, mlir::PostOrder, mlir::WalkResult::skip(), success(), and mlir::Operation::walk().

Referenced by bufferizeModuleOp(), and runOneShotBufferize().

◆ buildBufferDeallocationPipeline()

void mlir::bufferization::buildBufferDeallocationPipeline	(	OpPassManager &	pm,
		const BufferDeallocationPipelineOptions &	options )

Adds the buffer deallocation pipeline to the OpPassManager.

This is the standard pipeline for deallocating the MemRefs introduced by the One-Shot bufferization pass.

Definition at line 20 of file BufferizationPipelines.cpp.

References mlir::OpPassManager::addPass(), createBufferDeallocationSimplificationPass(), mlir::createCanonicalizerPass(), mlir::createCSEPass(), mlir::memref::createExpandReallocPass(), createLowerDeallocationsPass(), createOwnershipBasedBufferDeallocationPass(), and options.

Referenced by registerBufferizationPipelines().

◆ buildDeallocationLibraryFunction()

func::FuncOp mlir::bufferization::buildDeallocationLibraryFunction	(	OpBuilder &	builder,
		Location	loc,
		SymbolTable &	symbolTable )

Construct the library function needed for the fully generic bufferization.dealloc lowering implemented in the LowerDeallocations pass.

The function can then be called at bufferization dealloc sites to determine aliasing and ownership.

The generated function takes two memrefs of indices and three memrefs of booleans as arguments:

The first argument A should contain the result of the extract_aligned_pointer_as_index operation applied to the memrefs to be deallocated
The second argument B should contain the result of the extract_aligned_pointer_as_index operation applied to the memrefs to be retained
The third argument C should contain the conditions as passed directly to the deallocation operation.
The fourth argument D is used to pass results to the caller. Those represent the condition under which the memref at the corresponding position in A should be deallocated.
The fifth argument E is used to pass results to the caller. It provides the ownership value corresponding the the memref at the same position in B

This helper function is supposed to be called once for each bufferization.dealloc operation to determine the deallocation need and new ownership indicator for the retained values, but does not perform the deallocation itself.

Generated code:

    func.func @dealloc_helper(
        %dyn_dealloc_base_pointer_list: memref<?xindex>,
        %dyn_retain_base_pointer_list: memref<?xindex>,
        %dyn_cond_list: memref<?xi1>,
        %dyn_dealloc_cond_out: memref<?xi1>,
        %dyn_ownership_out: memref<?xi1>) {
      %c0 = arith.constant 0 : index
      %c1 = arith.constant 1 : index
      %true = arith.constant true
      %false = arith.constant false
      %num_dealloc_memrefs = memref.dim %dyn_dealloc_base_pointer_list, %c0
      %num_retain_memrefs = memref.dim %dyn_retain_base_pointer_list, %c0
      // Zero initialize result buffer.
      scf.for %i = %c0 to %num_retain_memrefs step %c1 {
        memref.store %false, %dyn_ownership_out[%i] : memref<?xi1>
      }
      scf.for %i = %c0 to %num_dealloc_memrefs step %c1 {
        %dealloc_bp = memref.load %dyn_dealloc_base_pointer_list[%i]
        %cond = memref.load %dyn_cond_list[%i]
        // Check for aliasing with retained memrefs.
        %does_not_alias_retained = scf.for %j = %c0 to %num_retain_memrefs
            step %c1 iter_args(%does_not_alias_aggregated = %true) -> (i1) {
          %retain_bp = memref.load %dyn_retain_base_pointer_list[%j]
          %does_alias = arith.cmpi eq, %retain_bp, %dealloc_bp : index
          scf.if %does_alias {
            %curr_ownership = memref.load %dyn_ownership_out[%j]
            %updated_ownership = arith.ori %curr_ownership, %cond : i1
            memref.store %updated_ownership, %dyn_ownership_out[%j]
          }
          %does_not_alias = arith.cmpi ne, %retain_bp, %dealloc_bp : index
          %updated_aggregate = arith.andi %does_not_alias_aggregated,
                                          %does_not_alias : i1
          scf.yield %updated_aggregate : i1
        }
        // Check for aliasing with dealloc memrefs in the list before the
        // current one, i.e.,
        // `fix i, forall j < i: check_aliasing(%dyn_dealloc_base_pointer[j],
///     // %dyn_dealloc_base_pointer[i])`
        %does_not_alias_any = scf.for %j = %c0 to %i step %c1
           iter_args(%does_not_alias_agg = %does_not_alias_retained) -> (i1) {
          %prev_dealloc_bp = memref.load %dyn_dealloc_base_pointer_list[%j]
          %does_not_alias = arith.cmpi ne, %prev_dealloc_bp, %dealloc_bp
          %updated_alias_agg = arith.andi %does_not_alias_agg, %does_not_alias
          scf.yield %updated_alias_agg : i1
        }
        %dealloc_cond = arith.andi %does_not_alias_any, %cond : i1
        memref.store %dealloc_cond, %dyn_dealloc_cond_out[%i] : memref<?xi1>
      }
      return
    }

Definition at line 431 of file LowerDeallocations.cpp.

References mlir::OpBuilder::clearInsertionPoint(), mlir::Builder::getBoolAttr(), mlir::Builder::getFunctionType(), mlir::Builder::getI1Type(), mlir::Builder::getIndexAttr(), mlir::Builder::getIndexType(), mlir::SymbolTable::insert(), mlir::SymbolTable::Private, mlir::OpBuilder::setInsertionPointToStart(), and ValueRange.

◆ buildSubsetExtraction()

Value mlir::bufferization::buildSubsetExtraction	(	RewriterBase &	rewriter,
		SubsetInsertionOpInterface	op,
		tensor::EmptyOp	emptyTensorOp,
		Operation *	user )

This method builds and returns a subset extraction value for the destination tensor that the given op inserts into.

It returns a value which should replace the emptyTensorOp use that is being consumed by user. If no such a value found it will return an empty Value.

Definition at line 95 of file EmptyTensorElimination.cpp.

References findValidInsertionPoint(), replacement(), and mlir::OpBuilder::setInsertionPoint().

◆ castOrReallocMemRefValue()

FailureOr< Value > mlir::bufferization::castOrReallocMemRefValue	(	OpBuilder &	b,
		Value	value,
		MemRefType	type,
		const BufferizationOptions &	options )

Try to cast the given ranked MemRef-typed value to the given ranked MemRef type.

Insert a reallocation + copy if it cannot be statically guaranteed that a direct cast would be valid.

E.g., when casting from a ranked MemRef type with dynamic layout to a ranked MemRef type with static layout, it is not statically known whether the cast will succeed or not. Such memref.cast ops may fail at runtime. This function never generates such casts and conservatively inserts a copy.

This function returns failure() in case of unsupported casts. E.g., casts with differing element types or memory spaces.

Definition at line 25 of file BufferizationOps.cpp.

References b, copy(), mlir::Value::getLoc(), mlir::Value::getType(), options, and target.

Referenced by mlir::bufferization::func_ext::CallOpInterface::bufferize(), and foldToBufferToTensorPair().

◆ createBufferDeallocationSimplificationPass()

std::unique_ptr<::mlir::Pass > mlir::bufferization::createBufferDeallocationSimplificationPass ( )

We declare an explicit private instantiation because Pass classes should only be visible by the current library.

Definition at line 92 of file BufferDeallocationSimplification.cpp.

Referenced by buildBufferDeallocationPipeline().

◆ createBufferHoistingPass()

std::unique_ptr<::mlir::Pass > mlir::bufferization::createBufferHoistingPass ( )

We declare an explicit private instantiation because Pass classes should only be visible by the current library.

Definition at line 167 of file BufferOptimizations.cpp.

◆ createBufferLoopHoistingPass()

std::unique_ptr<::mlir::Pass > mlir::bufferization::createBufferLoopHoistingPass ( )

We declare an explicit private instantiation because Pass classes should only be visible by the current library.

Definition at line 242 of file BufferOptimizations.cpp.

References mlir::Block::isEntryBlock(), and isKnownControlFlowInterface().

◆ createBufferResultsToOutParamsPass() [1/2]

std::unique_ptr<::mlir::Pass > mlir::bufferization::createBufferResultsToOutParamsPass ( )

Definition at line 339 of file BufferResultsToOutParams.cpp.

◆ createBufferResultsToOutParamsPass() [2/2]

std::unique_ptr<::mlir::Pass > mlir::bufferization::createBufferResultsToOutParamsPass ( BufferResultsToOutParamsPassOptions options )

Definition at line 343 of file BufferResultsToOutParams.cpp.

◆ createDropEquivalentBufferResultsPass()

std::unique_ptr<::mlir::Pass > mlir::bufferization::createDropEquivalentBufferResultsPass ( )

We declare an explicit private instantiation because Pass classes should only be visible by the current library.

Definition at line 418 of file DropEquivalentBufferResults.cpp.

◆ createEmptyTensorEliminationPass()

std::unique_ptr<::mlir::Pass > mlir::bufferization::createEmptyTensorEliminationPass ( )

We declare an explicit private instantiation because Pass classes should only be visible by the current library.

Definition at line 493 of file EmptyTensorElimination.cpp.

◆ createEmptyTensorToAllocTensorPass()

std::unique_ptr<::mlir::Pass > mlir::bufferization::createEmptyTensorToAllocTensorPass ( )

We declare an explicit private instantiation because Pass classes should only be visible by the current library.

Definition at line 568 of file EmptyTensorToAllocTensor.cpp.

Referenced by mlir::sparse_tensor::SparsificationAndBufferizationPass::runOnOperation().

◆ createLowerDeallocationsPass()

std::unique_ptr<::mlir::Pass > mlir::bufferization::createLowerDeallocationsPass ( )

We declare an explicit private instantiation because Pass classes should only be visible by the current library.

Definition at line 646 of file LowerDeallocations.cpp.

Referenced by buildBufferDeallocationPipeline().

◆ createOneShotBufferizePass() [1/2]

std::unique_ptr<::mlir::Pass > mlir::bufferization::createOneShotBufferizePass ( )

Definition at line 799 of file Bufferize.cpp.

◆ createOneShotBufferizePass() [2/2]

std::unique_ptr<::mlir::Pass > mlir::bufferization::createOneShotBufferizePass ( OneShotBufferizePassOptions options )

Definition at line 803 of file Bufferize.cpp.

◆ createOptimizeAllocationLivenessPass()

std::unique_ptr<::mlir::Pass > mlir::bufferization::createOptimizeAllocationLivenessPass ( )

We declare an explicit private instantiation because Pass classes should only be visible by the current library.

Definition at line 878 of file OptimizeAllocationLiveness.cpp.

◆ createOwnershipBasedBufferDeallocationPass() [1/2]

std::unique_ptr<::mlir::Pass > mlir::bufferization::createOwnershipBasedBufferDeallocationPass ( )

Definition at line 972 of file OwnershipBasedBufferDeallocation.cpp.

References mlir::Operation::getResults(), and isMemref().

Referenced by buildBufferDeallocationPipeline().

◆ createOwnershipBasedBufferDeallocationPass() [2/2]

std::unique_ptr<::mlir::Pass > mlir::bufferization::createOwnershipBasedBufferDeallocationPass ( OwnershipBasedBufferDeallocationPassOptions options )

Definition at line 976 of file OwnershipBasedBufferDeallocation.cpp.

References mlir::Operation::getBlock(), and isMemref().

◆ createPromoteBuffersToStackPass() [1/3]

std::unique_ptr<::mlir::Pass > mlir::bufferization::createPromoteBuffersToStackPass ( )

Definition at line 1070 of file BufferOptimizations.cpp.

◆ createPromoteBuffersToStackPass() [2/3]

std::unique_ptr<::mlir::Pass > mlir::bufferization::createPromoteBuffersToStackPass ( PromoteBuffersToStackPassOptions options )

Definition at line 1074 of file BufferOptimizations.cpp.

◆ createPromoteBuffersToStackPass() [3/3]

std::unique_ptr< Pass > mlir::bufferization::createPromoteBuffersToStackPass ( std::function< bool(Value)> isSmallAlloc )

Creates a pass that promotes heap-based allocations to stack-based ones.

Only buffers smaller with isSmallAlloc(alloc) == true are promoted.

Definition at line 478 of file BufferOptimizations.cpp.

◆ deallocateBuffersOwnershipBased()

LogicalResult mlir::bufferization::deallocateBuffersOwnershipBased	(	FunctionOpInterface	op,
		DeallocationOptions	options,
		SymbolTableCollection &	symbolTables )

Run the ownership-based buffer deallocation.

References options.

◆ dropEquivalentBufferResults()

LogicalResult mlir::bufferization::dropEquivalentBufferResults ( ModuleOp module )

Drop all memref function results that are equivalent to a function argument.

Definition at line 78 of file DropEquivalentBufferResults.cpp.

◆ eliminateEmptyTensors() [1/2]

LogicalResult mlir::bufferization::eliminateEmptyTensors	(	RewriterBase &	rewriter,
		Operation *	op )

Try to eliminate "tensor.empty" ops inside op.

This transformation looks for subset ops that insert a tensor that originates from a "tensor.empty" (as per the reverse use-def chain). Such "tensor.empty" ops are replaced with the destination subset.

E.g.: %0 = tensor.empty() : tensor<10xf32> %1 = linalg.fill ... outs(%0 : tensor<10xf32>) %2 = tensor.insert_slice %0 into t ...

In the above example, the subset op is "tensor.insert_slice". When tracing back the reverse use-def chain of a the source, we end up at a "tensor.empty" op.

Definition at line 202 of file EmptyTensorElimination.cpp.

References analyzeModuleOp(), analyzeOp(), eliminateEmptyTensors(), and options.

Referenced by eliminateEmptyTensors().

◆ eliminateEmptyTensors() [2/2]

LogicalResult mlir::bufferization::eliminateEmptyTensors	(	RewriterBase &	rewriter,
		Operation *	op,
		OneShotAnalysisState &	state,
		ControlBuildSubsetExtractionFn	subsetsExtractionFn = buildSubsetExtraction )

Try to eliminate "tensor.empty" ops inside op.

This function overload accepts an existing OneShotAnalysisState, which contains in-place bufferization decisions. This overload is useful if an existing analysis should be reused for empty tensor elimination.

Definition at line 117 of file EmptyTensorElimination.cpp.

References mlir::WalkResult::advance(), mlir::config, mlir::Value::getDefiningOp(), mlir::OpOperand::getOperandNumber(), mlir::detail::IROperandBase::getOwner(), mlir::Builder::getType(), mlir::bufferization::OneShotAnalysisState::isInPlace(), mlir::RewriterBase::modifyOpInPlace(), replacement(), mlir::bufferization::OneShotAnalysisState::resetCache(), mlir::OpBuilder::setInsertionPointAfterValue(), mlir::Operation::setOperand(), mlir::WalkResult::skip(), success(), and mlir::Operation::walk().

◆ findCommonDominator()

template<typename DominatorT>

Block * mlir::bufferization::findCommonDominator	(	Value	value,
		const BufferViewFlowAnalysis::ValueSetT &	values,
		const DominatorT &	doms )

Finds a common dominator for the given value while taking the positions of the values in the value set into account.

It supports dominator and post-dominator analyses via template arguments. If no common dominator can be found, this function will return "nullptr".

Definition at line 82 of file BufferUtils.h.

References mlir::Value::getParentBlock(), and mlir::Operation::getUsers().

◆ foldToBufferToTensorPair()

LogicalResult mlir::bufferization::foldToBufferToTensorPair	(	RewriterBase &	rewriter,
		ToBufferOp	toBuffer,
		const BufferizationOptions &	options )

Try to fold to_buffer(to_tensor(x)).

If x's type and the result type of the to_buffer op are different, a memref.cast is needed.

Definition at line 85 of file BufferizationOps.cpp.

References castOrReallocMemRefValue(), options, replacement(), mlir::RewriterBase::replaceOp(), mlir::RewriterBase::replaceOpWithNewOp(), and success().

Referenced by bufferizeOp().

◆ getGlobalFor()

FailureOr< memref::GlobalOp > mlir::bufferization::getGlobalFor	(	arith::ConstantOp	constantOp,
		SymbolTableCollection &	symbolTables,
		uint64_t	alignment,
		Attribute	memorySpace = {} )

Definition at line 101 of file BufferUtils.cpp.

References mlir::Builder::getI64Type(), mlir::Region::getOps(), mlir::Operation::getRegion(), mlir::Builder::getStringAttr(), mlir::SymbolTableCollection::getSymbolTable(), mlir::SymbolTable::insert(), and mlir::MemRefType::Builder::setMemorySpace().

◆ getReturnOps()

SmallVector< func::ReturnOp > mlir::bufferization::getReturnOps ( func::FuncOp funcOp )

Helper function that returns all func.return ops in the given function.

Return all func.return ops in the given function.

Definition at line 22 of file FuncBufferizableOpInterfaceImpl.cpp.

References b, and result.

Referenced by mlir::bufferization::func_ext::FuncOpInterface::bufferize(), and foldMemRefCasts().

◆ hoistBuffersFromLoops()

void mlir::bufferization::hoistBuffersFromLoops ( Operation * op )

Within the given operation, hoist buffers from loops where possible.

See "BufferLoopHoistingPass" for more information.

Definition at line 473 of file BufferOptimizations.cpp.

◆ insertSymbol()

void mlir::bufferization::insertSymbol	(	Operation *	op,
		BufferizationState &	state )

Definition at line 167 of file BufferUtils.cpp.

References mlir::Operation::getParentWithTrait(), and mlir::SymbolTable::insert().

◆ insertTensorCopies() [1/2]

LogicalResult mlir::bufferization::insertTensorCopies	(	Operation *	op,
		const AnalysisState &	analysisState,
		const BufferizationState &	bufferizationState )

Resolve RaW and other conflicts by inserting bufferization.alloc_tensor ops.

After applying this transform, the IR can be bufferized without inserting additional buffer allocations.

Definition at line 42 of file TensorCopyInsertion.cpp.

References mlir::WalkResult::advance(), mlir::Operation::getContext(), mlir::Operation::getParentWithTrait(), mlir::Operation::hasTrait(), mlir::WalkResult::interrupt(), result, mlir::OpBuilder::setInsertionPoint(), mlir::WalkResult::skip(), and mlir::Operation::walk().

◆ insertTensorCopies() [2/2]

LogicalResult mlir::bufferization::insertTensorCopies	(	Operation *	op,
		const OneShotBufferizationOptions &	options,
		const BufferizationState &	bufferizationState,
		BufferizationStatistics *	statistics = nullptr )

Resolve RaW and other conflicts by inserting bufferization.alloc_tensor ops.

After applying this transform, the IR can be bufferized without inserting additional buffer allocations.

Definition at line 20 of file TensorCopyInsertion.cpp.

References analyzeModuleOp(), analyzeOp(), insertTensorCopies(), options, and success().

Referenced by bufferizeOp(), insertTensorCopies(), runOneShotBufferize(), runOneShotModuleBufferize(), and mlir::sparse_tensor::SparsificationAndBufferizationPass::runOnOperation().

◆ populateBufferizationDeallocLoweringPattern()

void mlir::bufferization::populateBufferizationDeallocLoweringPattern	(	RewritePatternSet &	patterns,
		const DeallocHelperMap &	deallocHelperFuncMap )

Adds the conversion pattern of the bufferization.dealloc operation to the given pattern set for use in other transformation passes.

Definition at line 546 of file LowerDeallocations.cpp.

References mlir::patterns.

◆ populateDeallocOpCanonicalizationPatterns()

void mlir::bufferization::populateDeallocOpCanonicalizationPatterns	(	RewritePatternSet &	patterns,
		MLIRContext *	context )

Add the canonicalization patterns for bufferization.dealloc to the given pattern set to make them available to other passes (such as BufferDeallocationSimplification).

Definition at line 1193 of file BufferizationOps.cpp.

References mlir::patterns.

◆ populateDynamicDimSizes()

void mlir::bufferization::populateDynamicDimSizes	(	OpBuilder &	b,
		Location	loc,
		Value	shapedValue,
		SmallVector< Value > &	dynamicDims )

Populate dynamicDims with tensor::DimOp / memref::DimOp results for all dynamic dimensions of the given shaped value.

Definition at line 130 of file BufferizationOps.cpp.

References b, and mlir::Value::getType().

◆ populateEmptyTensorToAllocTensorPattern()

void mlir::bufferization::populateEmptyTensorToAllocTensorPattern ( RewritePatternSet & patterns )

Populate patterns to lower tensor.empty ops to bufferization.alloc_tensor ops.

Definition at line 51 of file EmptyTensorToAllocTensor.cpp.

References mlir::patterns.

◆ promoteBufferResultsToOutParams()

LogicalResult mlir::bufferization::promoteBufferResultsToOutParams	(	ModuleOp	module,
		const BufferResultsToOutParamsOpts &	options )

Replace buffers that are returned from a function with an out parameter.

Also update all call sites.

Definition at line 291 of file BufferResultsToOutParams.cpp.

Referenced by mlir::bufferization::impl::BufferResultsToOutParamsPassBase< DerivedT >::createBufferResultsToOutParamsPass.

◆ registerBufferDeallocationSimplificationPass()

void mlir::bufferization::registerBufferDeallocationSimplificationPass ( )

inline

Definition at line 1100 of file Passes.h.

◆ registerBufferDeallocationSimplificationPassPass()

void mlir::bufferization::registerBufferDeallocationSimplificationPassPass ( )

inline

Definition at line 1107 of file Passes.h.

◆ registerBufferHoistingPass()

void mlir::bufferization::registerBufferHoistingPass ( )

inline

Definition at line 1121 of file Passes.h.

◆ registerBufferHoistingPassPass()

void mlir::bufferization::registerBufferHoistingPassPass ( )

inline

Definition at line 1128 of file Passes.h.

◆ registerBufferizationPasses()

void mlir::bufferization::registerBufferizationPasses ( )

inline

Definition at line 1352 of file Passes.h.

Referenced by mlir::registerAllPasses().

◆ registerBufferizationPipelines()

void mlir::bufferization::registerBufferizationPipelines ( )

Registers all pipelines for the bufferization dialect.

Currently, this includes only the "buffer-deallocation-pipeline".

Definition at line 41 of file BufferizationPipelines.cpp.

References buildBufferDeallocationPipeline().

Referenced by mlir::registerAllPasses().

◆ registerBufferLoopHoistingPass()

void mlir::bufferization::registerBufferLoopHoistingPass ( )

inline

Definition at line 1142 of file Passes.h.

◆ registerBufferLoopHoistingPassPass()

void mlir::bufferization::registerBufferLoopHoistingPassPass ( )

inline

Definition at line 1149 of file Passes.h.

◆ registerBufferResultsToOutParamsPass()

void mlir::bufferization::registerBufferResultsToOutParamsPass ( )

inline

Definition at line 1163 of file Passes.h.

◆ registerBufferResultsToOutParamsPassPass()

void mlir::bufferization::registerBufferResultsToOutParamsPassPass ( )

inline

Definition at line 1170 of file Passes.h.

◆ registerDropEquivalentBufferResultsPass()

void mlir::bufferization::registerDropEquivalentBufferResultsPass ( )

inline

Definition at line 1184 of file Passes.h.

◆ registerDropEquivalentBufferResultsPassPass()

void mlir::bufferization::registerDropEquivalentBufferResultsPassPass ( )

inline

Definition at line 1191 of file Passes.h.

◆ registerEmptyTensorEliminationPass()

void mlir::bufferization::registerEmptyTensorEliminationPass ( )

inline

Definition at line 1205 of file Passes.h.

◆ registerEmptyTensorEliminationPassPass()

void mlir::bufferization::registerEmptyTensorEliminationPassPass ( )

inline

Definition at line 1212 of file Passes.h.

◆ registerEmptyTensorToAllocTensorPass()

void mlir::bufferization::registerEmptyTensorToAllocTensorPass ( )

inline

Definition at line 1226 of file Passes.h.

◆ registerEmptyTensorToAllocTensorPassPass()

void mlir::bufferization::registerEmptyTensorToAllocTensorPassPass ( )

inline

Definition at line 1233 of file Passes.h.

◆ registerLowerDeallocationsPass()

void mlir::bufferization::registerLowerDeallocationsPass ( )

inline

Definition at line 1247 of file Passes.h.

◆ registerLowerDeallocationsPassPass()

void mlir::bufferization::registerLowerDeallocationsPassPass ( )

inline

Definition at line 1254 of file Passes.h.

◆ registerOneShotBufferizePass()

void mlir::bufferization::registerOneShotBufferizePass ( )

inline

Definition at line 1268 of file Passes.h.

◆ registerOneShotBufferizePassPass()

void mlir::bufferization::registerOneShotBufferizePassPass ( )

inline

Definition at line 1275 of file Passes.h.

◆ registerOptimizeAllocationLivenessPass()

void mlir::bufferization::registerOptimizeAllocationLivenessPass ( )

inline

Definition at line 1289 of file Passes.h.

◆ registerOptimizeAllocationLivenessPassPass()

void mlir::bufferization::registerOptimizeAllocationLivenessPassPass ( )

inline

Definition at line 1296 of file Passes.h.

◆ registerOwnershipBasedBufferDeallocationPass()

void mlir::bufferization::registerOwnershipBasedBufferDeallocationPass ( )

inline

Definition at line 1310 of file Passes.h.

◆ registerOwnershipBasedBufferDeallocationPassPass()

void mlir::bufferization::registerOwnershipBasedBufferDeallocationPassPass ( )

inline

Definition at line 1317 of file Passes.h.

◆ registerPromoteBuffersToStackPass()

void mlir::bufferization::registerPromoteBuffersToStackPass ( )

inline

Definition at line 1331 of file Passes.h.

◆ registerPromoteBuffersToStackPassPass()

void mlir::bufferization::registerPromoteBuffersToStackPassPass ( )

inline

Definition at line 1338 of file Passes.h.

◆ registerTransformDialectExtension()

void mlir::bufferization::registerTransformDialectExtension ( DialectRegistry & registry )

Definition at line 180 of file BufferizationTransformOps.cpp.

References mlir::DialectRegistry::addExtensions().

Referenced by mlir::registerAllExtensions().

◆ removeBufferizationAttributesInModule()

void mlir::bufferization::removeBufferizationAttributesInModule ( Operation * moduleOp )

Remove bufferization attributes on every FuncOp arguments in the SymbolTable op.

Definition at line 515 of file OneShotModuleBufferize.cpp.

References mlir::Operation::getRegions(), and removeBufferizationAttributes().

Referenced by bufferizeModuleOp(), and mlir::sparse_tensor::SparsificationAndBufferizationPass::runDenseBufferization().

◆ removeSymbol()

void mlir::bufferization::removeSymbol	(	Operation *	op,
		BufferizationState &	state )

Definition at line 160 of file BufferUtils.cpp.

References mlir::Operation::getParentWithTrait(), and mlir::SymbolTable::remove().

◆ runOneShotBufferize()

LogicalResult mlir::bufferization::runOneShotBufferize	(	Operation *	op,
		const OneShotBufferizationOptions &	options,
		BufferizationState &	state,
		BufferizationStatistics *	statistics = nullptr )

Run One-Shot Bufferize on the given op: Analysis + Bufferization.

Definition at line 1352 of file OneShotAnalysis.cpp.

References bufferizeOp(), insertTensorCopies(), options, and success().

◆ runOneShotModuleBufferize()

LogicalResult mlir::bufferization::runOneShotModuleBufferize	(	Operation *	moduleOp,
		const bufferization::OneShotBufferizationOptions &	options,
		BufferizationState &	state,
		BufferizationStatistics *	statistics = nullptr )

Run One-Shot Module Bufferization on the given SymbolTable.

Performs a simple function call analysis to determine which function arguments are inplaceable. Then analyzes and bufferizes FuncOps one-by-one with One-Shot Bufferize.

Definition at line 598 of file OneShotModuleBufferize.cpp.

References bufferizeModuleOp(), mlir::Operation::getParentOfType(), insertTensorCopies(), options, and success().

Namespaces

Classes

Typedefs

Functions

Typedef Documentation

◆ ControlBuildSubsetExtractionFn

◆ DeallocHelperMap

◆ RegisterDependenciesFn

Function Documentation

◆ analyzeModuleOp()

◆ analyzeOp()

◆ bufferizeBlockSignature()

◆ bufferizeModuleOp()

◆ bufferizeOp()

◆ buildBufferDeallocationPipeline()

◆ buildDeallocationLibraryFunction()

◆ buildSubsetExtraction()

◆ castOrReallocMemRefValue()

◆ createBufferDeallocationSimplificationPass()

◆ createBufferHoistingPass()

◆ createBufferLoopHoistingPass()

◆ createBufferResultsToOutParamsPass() [1/2]

◆ createBufferResultsToOutParamsPass() [2/2]

◆ createDropEquivalentBufferResultsPass()

◆ createEmptyTensorEliminationPass()

◆ createEmptyTensorToAllocTensorPass()

◆ createLowerDeallocationsPass()

◆ createOneShotBufferizePass() [1/2]

◆ createOneShotBufferizePass() [2/2]

◆ createOptimizeAllocationLivenessPass()

◆ createOwnershipBasedBufferDeallocationPass() [1/2]

◆ createOwnershipBasedBufferDeallocationPass() [2/2]

◆ createPromoteBuffersToStackPass() [1/3]

◆ createPromoteBuffersToStackPass() [2/3]

◆ createPromoteBuffersToStackPass() [3/3]

◆ deallocateBuffersOwnershipBased()

◆ dropEquivalentBufferResults()

◆ eliminateEmptyTensors() [1/2]

◆ eliminateEmptyTensors() [2/2]

◆ findCommonDominator()

◆ foldToBufferToTensorPair()

◆ getGlobalFor()

◆ getReturnOps()

◆ hoistBuffersFromLoops()

◆ insertSymbol()

◆ insertTensorCopies() [1/2]

◆ insertTensorCopies() [2/2]

◆ populateBufferizationDeallocLoweringPattern()

◆ populateDeallocOpCanonicalizationPatterns()

◆ populateDynamicDimSizes()

◆ populateEmptyTensorToAllocTensorPattern()

◆ promoteBufferResultsToOutParams()

◆ registerBufferDeallocationSimplificationPass()

◆ registerBufferDeallocationSimplificationPassPass()

◆ registerBufferHoistingPass()

◆ registerBufferHoistingPassPass()

◆ registerBufferizationPasses()

◆ registerBufferizationPipelines()

◆ registerBufferLoopHoistingPass()

◆ registerBufferLoopHoistingPassPass()

◆ registerBufferResultsToOutParamsPass()

◆ registerBufferResultsToOutParamsPassPass()

◆ registerDropEquivalentBufferResultsPass()

◆ registerDropEquivalentBufferResultsPassPass()

◆ registerEmptyTensorEliminationPass()

◆ registerEmptyTensorEliminationPassPass()

◆ registerEmptyTensorToAllocTensorPass()

◆ registerEmptyTensorToAllocTensorPassPass()

◆ registerLowerDeallocationsPass()

◆ registerLowerDeallocationsPassPass()

◆ registerOneShotBufferizePass()

◆ registerOneShotBufferizePassPass()

◆ registerOptimizeAllocationLivenessPass()

◆ registerOptimizeAllocationLivenessPassPass()

◆ registerOwnershipBasedBufferDeallocationPass()

◆ registerOwnershipBasedBufferDeallocationPassPass()

◆ registerPromoteBuffersToStackPass()

◆ registerPromoteBuffersToStackPassPass()

◆ registerTransformDialectExtension()

◆ removeBufferizationAttributesInModule()