#include "mlir/Dialect/Affine/Passes.h"
#include "mlir/Dialect/Affine/Analysis/AffineStructures.h"
#include "mlir/Dialect/Affine/Analysis/LoopAnalysis.h"
#include "mlir/Dialect/Affine/Analysis/Utils.h"
#include "mlir/Dialect/Affine/LoopFusionUtils.h"
#include "mlir/Dialect/Affine/LoopUtils.h"
#include "mlir/Dialect/Affine/Utils.h"
#include "mlir/Dialect/MemRef/IR/MemRef.h"
#include "mlir/IR/AffineExpr.h"
#include "mlir/IR/AffineMap.h"
#include "mlir/IR/Builders.h"
#include "llvm/ADT/DenseMap.h"
#include "llvm/ADT/STLExtras.h"
#include "llvm/Support/CommandLine.h"
#include "llvm/Support/Debug.h"
#include "llvm/Support/DebugLog.h"
#include "llvm/Support/raw_ostream.h"
#include <iomanip>
#include <optional>
#include <sstream>
#include "mlir/Dialect/Affine/Passes.h.inc"

Classes
class	mlir::affine::impl::AffineLoopFusionBase< DerivedT >

Namespaces
namespace	mlir
	Include the generated interface declarations.
namespace	mlir::affine
namespace	mlir::affine::impl

Macros
#define	GEN_PASS_DEF_AFFINELOOPFUSION
#define	DEBUG_TYPE "affine-fusion"

Functions
static bool	canRemoveSrcNodeAfterFusion (unsigned srcId, unsigned dstId, const ComputationSliceState &fusionSlice, Operation *fusedLoopInsPoint, const DenseSet< Value > &escapingMemRefs, const MemRefDependenceGraph &mdg)
	Returns true if node 'srcId' can be removed after fusing it with node 'dstId'.
static void	getProducerCandidates (unsigned dstId, const MemRefDependenceGraph &mdg, SmallVectorImpl< unsigned > &srcIdCandidates)
	Returns in 'srcIdCandidates' the producer fusion candidates for consumer 'dstId'.
static void	gatherProducerConsumerMemrefs (unsigned srcId, unsigned dstId, const MemRefDependenceGraph &mdg, DenseSet< Value > &producerConsumerMemrefs)
	Returns in 'producerConsumerMemrefs' the memrefs involved in a producer-consumer dependence between 'srcId' and 'dstId'.
static bool	isEscapingMemref (Value memref, Block *block)
	A memref escapes in the context of the fusion pass if either:
static void	gatherEscapingMemrefs (unsigned id, const MemRefDependenceGraph &mdg, DenseSet< Value > &escapingMemRefs)
	Returns in 'escapingMemRefs' the memrefs from affine store ops in node 'id' that escape the block or are accessed in a non-affine way.
static void	sinkSequentialLoops (MemRefDependenceGraph::Node *node)
static Operation *	getDominanceFilterForPrivateMemRefRepl (Block sliceInsertionBlock, ArrayRef< Operation > producerStores)
	Get the operation that should act as a dominance filter while replacing memref uses with a private memref for which producerStores and sliceInsertionBlock are provided.
static std::optional< double >	getAdditionalComputeFraction (AffineForOp srcForOp, AffineForOp dstForOp, unsigned depth, ArrayRef< ComputationSliceState > depthSliceUnions, int64_t &sliceCost, int64_t &fusedLoopNestComputeCost)
	Returns the amount of additional (redundant) computation that will be done as a fraction of the total computation if srcForOp is fused into dstForOp at depth depth.
static Value	createPrivateMemRef (AffineForOp forOp, ArrayRef< Operation * > storeOps, unsigned dstLoopDepth, std::optional< unsigned > fastMemorySpace, Block *sliceInsertionBlock, uint64_t localBufSizeThreshold)
static bool	isFusionProfitable (AffineForOp srcForOp, ArrayRef< Operation * > producerStores, AffineForOp dstForOp, ArrayRef< ComputationSliceState > depthSliceUnions, unsigned maxLegalFusionDepth, unsigned *dstLoopDepth, double computeToleranceThreshold)

Macro Definition Documentation

◆ DEBUG_TYPE

#define DEBUG_TYPE "affine-fusion"

Definition at line 42 of file LoopFusion.cpp.

◆ GEN_PASS_DEF_AFFINELOOPFUSION

#define GEN_PASS_DEF_AFFINELOOPFUSION

Definition at line 37 of file LoopFusion.cpp.

Function Documentation

◆ canRemoveSrcNodeAfterFusion()

bool canRemoveSrcNodeAfterFusion	(	unsigned	srcId,
		unsigned	dstId,
		const ComputationSliceState &	fusionSlice,
		Operation *	fusedLoopInsPoint,
		const DenseSet< Value > &	escapingMemRefs,
		const MemRefDependenceGraph &	mdg )

static

Returns true if node 'srcId' can be removed after fusing it with node 'dstId'.

The node can be removed if any of the following conditions are met:

'srcId' has no output dependences after fusion and no escaping memrefs.
'srcId' has no output dependences after fusion, has escaping memrefs and the fusion slice is maximal.
'srcId' has output dependences after fusion, the fusion slice is maximal and the fusion insertion point dominates all the dependences.

Definition at line 76 of file LoopFusion.cpp.

References mlir::Operation::getBlock(), mlir::affine::MemRefDependenceGraph::getNode(), mlir::Operation::isBeforeInBlock(), mlir::affine::ComputationSliceState::isMaximal(), mlir::affine::MemRefDependenceGraph::Node::op, and mlir::affine::MemRefDependenceGraph::outEdges.

◆ createPrivateMemRef()

Value createPrivateMemRef	(	AffineForOp	forOp,
		ArrayRef< Operation * >	storeOps,
		unsigned	dstLoopDepth,
		std::optional< unsigned >	fastMemorySpace,
		Block *	sliceInsertionBlock,
		uint64_t	localBufSizeThreshold )

static

Definition at line 333 of file LoopFusion.cpp.

References b, mlir::affine::MemRefRegion::compute(), mlir::AffineMap::get(), mlir::getAffineDimExpr(), mlir::affine::MemRefRegion::getConstantBoundingSizeAndShape(), mlir::affine::MemRefRegion::getConstraints(), getDominanceFilterForPrivateMemRefRepl(), mlir::Operation::getLoc(), mlir::affine::getMemRefIntOrFloatEltSizeInBytes(), mlir::presburger::IntegerRelation::getNumDimAndSymbolVars(), mlir::Operation::getParentOfType(), mlir::FlatLinearValueConstraints::getValues(), and mlir::simplifyAffineExpr().

◆ gatherEscapingMemrefs()

void gatherEscapingMemrefs	(	unsigned	id,
		const MemRefDependenceGraph &	mdg,
		DenseSet< Value > &	escapingMemRefs )

static

Returns in 'escapingMemRefs' the memrefs from affine store ops in node 'id' that escape the block or are accessed in a non-affine way.

Definition at line 216 of file LoopFusion.cpp.

References mlir::affine::MemRefDependenceGraph::block, mlir::affine::MemRefDependenceGraph::getNode(), and isEscapingMemref().

◆ gatherProducerConsumerMemrefs()

void gatherProducerConsumerMemrefs	(	unsigned	srcId,
		unsigned	dstId,
		const MemRefDependenceGraph &	mdg,
		DenseSet< Value > &	producerConsumerMemrefs )

static

Returns in 'producerConsumerMemrefs' the memrefs involved in a producer-consumer dependence between 'srcId' and 'dstId'.

Definition at line 169 of file LoopFusion.cpp.

References mlir::affine::gatherProducerConsumerMemrefs(), and mlir::affine::MemRefDependenceGraph::getNode().

◆ getAdditionalComputeFraction()

std::optional< double > getAdditionalComputeFraction	(	AffineForOp	srcForOp,
		AffineForOp	dstForOp,
		unsigned	depth,
		ArrayRef< ComputationSliceState >	depthSliceUnions,
		int64_t &	sliceCost,
		int64_t &	fusedLoopNestComputeCost )

static

Returns the amount of additional (redundant) computation that will be done as a fraction of the total computation if srcForOp is fused into dstForOp at depth depth.

The method returns the compute cost of the slice and the fused nest's compute cost in the trailing output arguments.

Definition at line 279 of file LoopFusion.cpp.

Referenced by isFusionProfitable().

◆ getDominanceFilterForPrivateMemRefRepl()

Operation * getDominanceFilterForPrivateMemRefRepl	(	Block *	sliceInsertionBlock,
		ArrayRef< Operation * >	producerStores )

static

Get the operation that should act as a dominance filter while replacing memref uses with a private memref for which producerStores and sliceInsertionBlock are provided.

This effectively determines in what part of the IR we should be performing the replacement.

Definition at line 244 of file LoopFusion.cpp.

Referenced by createPrivateMemRef().

◆ getProducerCandidates()

void getProducerCandidates	(	unsigned	dstId,
		const MemRefDependenceGraph &	mdg,
		SmallVectorImpl< unsigned > &	srcIdCandidates )

static

Returns in 'srcIdCandidates' the producer fusion candidates for consumer 'dstId'.

Candidates are sorted by node id order. This order corresponds to the program order when the 'mdg' is created. However, program order is not guaranteed and must not be required by the client. Program order won't be held if the 'mdg' is reused from a previous fusion step or if the node creation order changes in the future to support more advance cases.

Definition at line 134 of file LoopFusion.cpp.

References getMemRef(), mlir::affine::MemRefDependenceGraph::getNode(), mlir::affine::MemRefDependenceGraph::inEdges, and load.

◆ isEscapingMemref()

bool isEscapingMemref	(	Value	memref,
		Block *	block )

static

A memref escapes in the context of the fusion pass if either:

it (or its alias) is a block argument, or
created by an op not known to guarantee alias freedom,
it (or its alias) are used by ops other than affine dereferencing ops (e.g., by call op, memref load/store ops, alias creating ops, unknown ops, terminator ops, etc.); such ops do not deference the memref in an affine way.

Definition at line 185 of file LoopFusion.cpp.

References mlir::hasSingleEffect(), and isEscapingMemref().

Referenced by gatherEscapingMemrefs(), and isEscapingMemref().

◆ isFusionProfitable()

bool isFusionProfitable	(	AffineForOp	srcForOp,
		ArrayRef< Operation * >	producerStores,
		AffineForOp	dstForOp,
		ArrayRef< ComputationSliceState >	depthSliceUnions,
		unsigned	maxLegalFusionDepth,
		unsigned *	dstLoopDepth,
		double	computeToleranceThreshold )

static

Definition at line 500 of file LoopFusion.cpp.

References mlir::affine::MemRefRegion::compute(), getAdditionalComputeFraction(), mlir::affine::getComputeCost(), mlir::Operation::getLoc(), mlir::affine::getLoopNestStats(), mlir::affine::getMemoryFootprintBytes(), mlir::affine::MemRefRegion::getRegionSize(), and mlir::affine::ComputationSliceState::isEmpty().

◆ sinkSequentialLoops()

void sinkSequentialLoops ( MemRefDependenceGraph::Node * node )

static

Definition at line 233 of file LoopFusion.cpp.

References mlir::affine::MemRefDependenceGraph::Node::op, and mlir::affine::sinkSequentialLoops().

Classes

Namespaces

Macros

Functions

Macro Definition Documentation

◆ DEBUG_TYPE

◆ GEN_PASS_DEF_AFFINELOOPFUSION

Function Documentation

◆ canRemoveSrcNodeAfterFusion()

◆ createPrivateMemRef()

◆ gatherEscapingMemrefs()

◆ gatherProducerConsumerMemrefs()

◆ getAdditionalComputeFraction()

◆ getDominanceFilterForPrivateMemRefRepl()

◆ getProducerCandidates()

◆ isEscapingMemref()

◆ isFusionProfitable()

◆ sinkSequentialLoops()