MLIR  21.0.0git
Namespaces | Macros | Functions
LoopFusion.cpp File Reference
#include "mlir/Dialect/Affine/Passes.h"
#include "mlir/Dialect/Affine/Analysis/AffineStructures.h"
#include "mlir/Dialect/Affine/Analysis/LoopAnalysis.h"
#include "mlir/Dialect/Affine/Analysis/Utils.h"
#include "mlir/Dialect/Affine/IR/AffineOps.h"
#include "mlir/Dialect/Affine/LoopFusionUtils.h"
#include "mlir/Dialect/Affine/LoopUtils.h"
#include "mlir/Dialect/Affine/Utils.h"
#include "mlir/Dialect/MemRef/IR/MemRef.h"
#include "mlir/IR/AffineExpr.h"
#include "mlir/IR/AffineMap.h"
#include "mlir/IR/Builders.h"
#include "mlir/Transforms/Passes.h"
#include "llvm/ADT/DenseMap.h"
#include "llvm/ADT/DenseSet.h"
#include "llvm/ADT/STLExtras.h"
#include "llvm/Support/CommandLine.h"
#include "llvm/Support/Debug.h"
#include "llvm/Support/raw_ostream.h"
#include <iomanip>
#include <optional>
#include <sstream>
#include "mlir/Dialect/Affine/Passes.h.inc"

Go to the source code of this file.

Namespaces

 mlir
 Include the generated interface declarations.
 
 mlir::affine
 

Macros

#define GEN_PASS_DEF_AFFINELOOPFUSION
 
#define DEBUG_TYPE   "affine-fusion"
 

Functions

static bool canRemoveSrcNodeAfterFusion (unsigned srcId, unsigned dstId, const ComputationSliceState &fusionSlice, Operation *fusedLoopInsPoint, const DenseSet< Value > &escapingMemRefs, const MemRefDependenceGraph &mdg)
 Returns true if node 'srcId' can be removed after fusing it with node 'dstId'. More...
 
static void getProducerCandidates (unsigned dstId, const MemRefDependenceGraph &mdg, SmallVectorImpl< unsigned > &srcIdCandidates)
 Returns in 'srcIdCandidates' the producer fusion candidates for consumer 'dstId'. More...
 
static void gatherProducerConsumerMemrefs (unsigned srcId, unsigned dstId, const MemRefDependenceGraph &mdg, DenseSet< Value > &producerConsumerMemrefs)
 Returns in 'producerConsumerMemrefs' the memrefs involved in a producer-consumer dependence between 'srcId' and 'dstId'. More...
 
static bool isEscapingMemref (Value memref, Block *block)
 A memref escapes in the context of the fusion pass if either: More...
 
static void gatherEscapingMemrefs (unsigned id, const MemRefDependenceGraph &mdg, DenseSet< Value > &escapingMemRefs)
 Returns in 'escapingMemRefs' the memrefs from affine store ops in node 'id' that escape the block or are accessed in a non-affine way. More...
 
static void sinkSequentialLoops (MemRefDependenceGraph::Node *node)
 
static OperationgetDominanceFilterForPrivateMemRefRepl (Block *sliceInsertionBlock, ArrayRef< Operation * > producerStores)
 Get the operation that should act as a dominance filter while replacing memref uses with a private memref for which producerStores and sliceInsertionBlock are provided. More...
 
static Value createPrivateMemRef (AffineForOp forOp, ArrayRef< Operation * > storeOps, unsigned dstLoopDepth, std::optional< unsigned > fastMemorySpace, Block *sliceInsertionBlock, uint64_t localBufSizeThreshold)
 
static bool isFusionProfitable (Operation *srcOpInst, Operation *srcStoreOpInst, AffineForOp dstForOp, ArrayRef< ComputationSliceState > depthSliceUnions, unsigned maxLegalFusionDepth, unsigned *dstLoopDepth, double computeToleranceThreshold)
 

Macro Definition Documentation

◆ DEBUG_TYPE

#define DEBUG_TYPE   "affine-fusion"

Definition at line 44 of file LoopFusion.cpp.

◆ GEN_PASS_DEF_AFFINELOOPFUSION

#define GEN_PASS_DEF_AFFINELOOPFUSION

Definition at line 39 of file LoopFusion.cpp.

Function Documentation

◆ canRemoveSrcNodeAfterFusion()

static bool canRemoveSrcNodeAfterFusion ( unsigned  srcId,
unsigned  dstId,
const ComputationSliceState fusionSlice,
Operation fusedLoopInsPoint,
const DenseSet< Value > &  escapingMemRefs,
const MemRefDependenceGraph mdg 
)
static

Returns true if node 'srcId' can be removed after fusing it with node 'dstId'.

The node can be removed if any of the following conditions are met:

  1. 'srcId' has no output dependences after fusion and no escaping memrefs.
  2. 'srcId' has no output dependences after fusion, has escaping memrefs and the fusion slice is maximal.
  3. 'srcId' has output dependences after fusion, the fusion slice is maximal and the fusion insertion point dominates all the dependences.

Definition at line 78 of file LoopFusion.cpp.

References mlir::Operation::getBlock(), mlir::affine::MemRefDependenceGraph::getNode(), mlir::Operation::isBeforeInBlock(), mlir::affine::ComputationSliceState::isMaximal(), mlir::affine::MemRefDependenceGraph::Node::op, and mlir::affine::MemRefDependenceGraph::outEdges.

◆ createPrivateMemRef()

static Value createPrivateMemRef ( AffineForOp  forOp,
ArrayRef< Operation * >  storeOps,
unsigned  dstLoopDepth,
std::optional< unsigned >  fastMemorySpace,
Block sliceInsertionBlock,
uint64_t  localBufSizeThreshold 
)
static

◆ gatherEscapingMemrefs()

static void gatherEscapingMemrefs ( unsigned  id,
const MemRefDependenceGraph mdg,
DenseSet< Value > &  escapingMemRefs 
)
static

Returns in 'escapingMemRefs' the memrefs from affine store ops in node 'id' that escape the block or are accessed in a non-affine way.

Definition at line 218 of file LoopFusion.cpp.

References mlir::affine::MemRefDependenceGraph::block, mlir::affine::MemRefDependenceGraph::getNode(), and isEscapingMemref().

◆ gatherProducerConsumerMemrefs()

static void gatherProducerConsumerMemrefs ( unsigned  srcId,
unsigned  dstId,
const MemRefDependenceGraph mdg,
DenseSet< Value > &  producerConsumerMemrefs 
)
static

Returns in 'producerConsumerMemrefs' the memrefs involved in a producer-consumer dependence between 'srcId' and 'dstId'.

Definition at line 172 of file LoopFusion.cpp.

References mlir::affine::gatherProducerConsumerMemrefs(), and mlir::affine::MemRefDependenceGraph::getNode().

◆ getDominanceFilterForPrivateMemRefRepl()

static Operation* getDominanceFilterForPrivateMemRefRepl ( Block sliceInsertionBlock,
ArrayRef< Operation * >  producerStores 
)
static

Get the operation that should act as a dominance filter while replacing memref uses with a private memref for which producerStores and sliceInsertionBlock are provided.

This effectively determines in what part of the IR we should be performing the replacement.

Definition at line 246 of file LoopFusion.cpp.

References mlir::Block::begin(), mlir::Block::findAncestorOpInBlock(), mlir::affine::findInnermostCommonBlockInScope(), and mlir::Operation::isBeforeInBlock().

Referenced by createPrivateMemRef().

◆ getProducerCandidates()

static void getProducerCandidates ( unsigned  dstId,
const MemRefDependenceGraph mdg,
SmallVectorImpl< unsigned > &  srcIdCandidates 
)
static

Returns in 'srcIdCandidates' the producer fusion candidates for consumer 'dstId'.

Candidates are sorted by node id order. This order corresponds to the program order when the 'mdg' is created. However, program order is not guaranteed and must not be required by the client. Program order won't be held if the 'mdg' is reused from a previous fusion step or if the node creation order changes in the future to support more advance cases.

Definition at line 137 of file LoopFusion.cpp.

References mlir::affine::MemRefDependenceGraph::getNode(), and mlir::affine::MemRefDependenceGraph::inEdges.

◆ isEscapingMemref()

static bool isEscapingMemref ( Value  memref,
Block block 
)
static

A memref escapes in the context of the fusion pass if either:

  1. it (or its alias) is a block argument, or
  2. created by an op not known to guarantee alias freedom,
  3. it (or its alias) are used by ops other than affine dereferencing ops (e.g., by call op, memref load/store ops, alias creating ops, unknown ops, terminator ops, etc.); such ops do not deference the memref in an affine way.

Definition at line 188 of file LoopFusion.cpp.

References mlir::Value::getDefiningOp(), and mlir::Value::getUsers().

Referenced by gatherEscapingMemrefs().

◆ isFusionProfitable()

static bool isFusionProfitable ( Operation srcOpInst,
Operation srcStoreOpInst,
AffineForOp  dstForOp,
ArrayRef< ComputationSliceState depthSliceUnions,
unsigned  maxLegalFusionDepth,
unsigned *  dstLoopDepth,
double  computeToleranceThreshold 
)
static

◆ sinkSequentialLoops()

static void sinkSequentialLoops ( MemRefDependenceGraph::Node node)
static