34void transform::ApplyForLoopCanonicalizationPatternsOp::populatePatterns(
39void transform::ApplySCFStructuralConversionPatternsOp::populatePatterns(
44void transform::ApplySCFStructuralConversionPatternsOp::
45 populateConversionTargetRules(
const TypeConverter &typeConverter,
51void transform::ApplySCFToControlFlowPatternsOp::populatePatterns(
65 if (!llvm::hasSingleElement(payload))
66 return emitSilenceableError() <<
"expected a single payload op";
68 auto target = dyn_cast<scf::ForallOp>(*payload.begin());
71 emitSilenceableError() <<
"expected the payload to be scf.forall";
72 diag.attachNote((*payload.begin())->getLoc()) <<
"payload op";
76 if (!
target.getOutputs().empty()) {
77 return emitSilenceableError()
78 <<
"unsupported shared outputs (didn't bufferize?)";
83 if (getNumResults() != lbs.size()) {
85 emitSilenceableError()
86 <<
"op expects as many results (" << getNumResults()
87 <<
") as payload has induction variables (" << lbs.size() <<
")";
88 diag.attachNote(
target.getLoc()) <<
"payload op";
95 <<
"failed to convert forall into for";
99 for (
auto &&[i, res] : llvm::enumerate(opResults)) {
100 results.
set(cast<OpResult>(getTransformed()[i]), {res});
114 if (!llvm::hasSingleElement(payload))
115 return emitSilenceableError() <<
"expected a single payload op";
117 auto target = dyn_cast<scf::ForallOp>(*payload.begin());
120 emitSilenceableError() <<
"expected the payload to be scf.forall";
121 diag.attachNote((*payload.begin())->getLoc()) <<
"payload op";
125 if (!
target.getOutputs().empty()) {
126 return emitSilenceableError()
127 <<
"unsupported shared outputs (didn't bufferize?)";
130 if (getNumResults() != 1) {
132 <<
"op expects one result, given "
134 diag.attachNote(
target.getLoc()) <<
"payload op";
138 scf::ParallelOp opResult;
141 emitSilenceableError() <<
"failed to convert forall into parallel";
145 results.
set(cast<OpResult>(getTransformed()[0]), {opResult});
157 if (!llvm::hasSingleElement(payload))
158 return emitSilenceableError() <<
"expected a single payload op";
160 auto target = dyn_cast<scf::ParallelOp>(*payload.begin());
163 emitSilenceableError() <<
"expected the payload to be scf.parallel";
164 diag.attachNote((*payload.begin())->getLoc()) <<
"payload op";
168 if (getNumResults() != 1) {
170 <<
"op expects one result, given "
172 diag.attachNote(
target.getLoc()) <<
"payload op";
176 FailureOr<scf::LoopNest> loopNest =
180 emitSilenceableError() <<
"failed to convert parallel into nested fors";
184 results.
set(cast<OpResult>(getTransformed()[0]), {loopNest->loops.front()});
200 b.setInsertionPoint(op);
201 scf::ExecuteRegionOp executeRegionOp =
205 b.setInsertionPointToStart(&executeRegionOp.getRegion().emplaceBlock());
206 Operation *clonedOp =
b.cloneWithoutRegions(*op);
208 assert(clonedRegion.
empty() &&
"expected empty region");
209 b.inlineRegionBefore(op->
getRegions().front(), clonedRegion,
214 return executeRegionOp;
230 <<
"failed to outline";
231 diag.attachNote(
target->getLoc()) <<
"target op";
236 rewriter, location, exec.getRegion(), getFuncName(), &call);
239 return emitDefaultDefiniteFailure(
target);
243 symbolTables.try_emplace(symbolTableOp, symbolTableOp)
245 symbolTable.
insert(*outlined);
248 functions.push_back(*outlined);
249 calls.push_back(call);
251 results.
set(cast<OpResult>(getFunction()), functions);
252 results.
set(cast<OpResult>(getCall()), calls);
266 if (getPeelFront()) {
267 LogicalResult status =
271 emitSilenceableError() <<
"failed to peel the first iteration";
275 LogicalResult status =
279 <<
"failed to peel the last iteration";
285 if (getPartialAsIf())
302 std::vector<std::pair<Operation *, unsigned>> &schedule,
303 unsigned iterationInterval,
unsigned readLatency) {
304 auto getLatency = [&](
Operation *op) ->
unsigned {
305 if (isa<vector::TransferReadOp>(op))
310 std::optional<int64_t> ubConstant =
312 std::optional<int64_t> lbConstant =
315 std::map<unsigned, std::vector<Operation *>> wrappedSchedule;
316 for (
Operation &op : forOp.getBody()->getOperations()) {
317 if (isa<scf::YieldOp>(op))
319 unsigned earlyCycle = 0;
320 for (
Value operand : op.getOperands()) {
321 Operation *def = operand.getDefiningOp();
324 if (ubConstant && lbConstant) {
325 unsigned ubInt = ubConstant.value();
326 unsigned lbInt = lbConstant.value();
327 auto minLatency = std::min(ubInt - lbInt - 1, getLatency(def));
328 earlyCycle = std::max(earlyCycle, opCycles[def] + minLatency);
330 earlyCycle = std::max(earlyCycle, opCycles[def] + getLatency(def));
333 opCycles[&op] = earlyCycle;
334 wrappedSchedule[earlyCycle % iterationInterval].push_back(&op);
336 for (
const auto &it : wrappedSchedule) {
338 unsigned cycle = opCycles[op];
339 schedule.emplace_back(op, cycle / iterationInterval);
351 [
this](scf::ForOp forOp,
352 std::vector<std::pair<Operation *, unsigned>> &schedule)
mutable {
358 FailureOr<scf::ForOp> patternResult =
360 if (succeeded(patternResult)) {
364 return emitDefaultSilenceableFailure(
target);
375 (
void)
target.promoteIfSingleIteration(rewriter);
379void transform::LoopPromoteIfOneIterationOp::getEffects(
394 LogicalResult
result(failure());
395 if (scf::ForOp scfFor = dyn_cast<scf::ForOp>(op))
397 else if (AffineForOp affineFor = dyn_cast<AffineForOp>(op))
400 return emitSilenceableError()
401 <<
"failed to unroll, incorrect type of payload";
404 return emitSilenceableError() <<
"failed to unroll";
417 LogicalResult
result(failure());
418 if (scf::ForOp scfFor = dyn_cast<scf::ForOp>(op))
420 else if (AffineForOp affineFor = dyn_cast<AffineForOp>(op))
423 return emitSilenceableError()
424 <<
"failed to fully unroll, incorrect type of payload";
427 return emitSilenceableError() <<
"failed to fully unroll";
440 LogicalResult
result(failure());
441 if (scf::ForOp scfFor = dyn_cast<scf::ForOp>(op))
443 else if (AffineForOp affineFor = dyn_cast<AffineForOp>(op))
446 return emitSilenceableError()
447 <<
"failed to unroll and jam, incorrect type of payload";
450 return emitSilenceableError() <<
"failed to unroll and jam";
464 LogicalResult
result(failure());
465 if (scf::ForOp scfForOp = dyn_cast<scf::ForOp>(op))
467 else if (AffineForOp affineForOp = dyn_cast<AffineForOp>(op))
468 result = coalescePerfectlyNestedAffineLoops(affineForOp);
473 <<
"failed to coalesce";
486 auto forOp = dyn_cast<scf::ForOp>(op);
488 return emitSilenceableError() <<
"expected scf.for operation";
493 scf::ForOp currentLoop = forOp;
495 while (currentLoop) {
496 nestedLoops.push_back(currentLoop);
497 Block &body = currentLoop.getRegion().
front();
500 scf::ForOp nextLoop =
nullptr;
502 if (
auto innerFor = dyn_cast<scf::ForOp>(&bodyOp)) {
508 currentLoop = nextLoop;
512 if (nestedLoops.size() < 2) {
513 return emitSilenceableError() <<
"need at least 2 nested loops to coalesce";
518 return emitSilenceableError() <<
"failed to coalesce nested loops";
532 assert(region.
hasOneBlock() &&
"expected single-block region");
547 getTakeElseBranch() ? ifOp.getElseRegion() : ifOp.getThenRegion();
550 <<
"requires an scf.if op with a single-block "
551 << ((getTakeElseBranch()) ?
"`else`" :
"`then`") <<
" region";
557void transform::TakeAssumedBranchOp::getEffects(
578 <<
"target and source need to be different loops";
583 <<
"target and source are not in the same block";
587 if (
target->isBeforeInBlock(source)) {
593 <<
"user of results of target should be properly dominated by "
603 Operation *operandOp = operand.getDefiningOp();
613 <<
"operands of target should be properly dominated by source";
620 Operation *operandOp = operand->get().getDefiningOp();
621 if (operandOp && !domInfo.properlyDominates(operandOp, source,
626 failedValue = operand;
632 <<
"values used inside regions of target should be properly "
633 "dominated by source";
646 auto targetOp = dyn_cast<scf::ForallOp>(
target);
647 auto sourceOp = dyn_cast<scf::ForallOp>(source);
648 if (!targetOp || !sourceOp)
651 return targetOp.getMixedLowerBound() == sourceOp.getMixedLowerBound() &&
652 targetOp.getMixedUpperBound() == sourceOp.getMixedUpperBound() &&
653 targetOp.getMixedStep() == sourceOp.getMixedStep() &&
654 targetOp.getMapping() == sourceOp.getMapping();
664 auto targetOp = dyn_cast<scf::ForOp>(
target);
665 auto sourceOp = dyn_cast<scf::ForOp>(source);
666 if (!targetOp || !sourceOp)
669 return targetOp.getLowerBound() == sourceOp.getLowerBound() &&
670 targetOp.getUpperBound() == sourceOp.getUpperBound() &&
671 targetOp.getStep() == sourceOp.getStep();
681 if (!llvm::hasSingleElement(targetOps) ||
682 !llvm::hasSingleElement(sourceOps)) {
684 <<
"requires exactly one target handle (got "
685 << llvm::range_size(targetOps) <<
") and exactly one "
686 <<
"source handle (got " << llvm::range_size(sourceOps) <<
")";
694 if (!
diag.succeeded())
701 cast<scf::ForOp>(
target), cast<scf::ForOp>(source), rewriter);
704 cast<scf::ForallOp>(
target), cast<scf::ForallOp>(source), rewriter);
707 <<
"operations cannot be fused";
710 assert(fusedLoop &&
"failed to fuse operations");
712 results.
set(cast<OpResult>(getFusedLoop()), {fusedLoop});
721class SCFTransformDialectExtension
723 SCFTransformDialectExtension> {
730 declareGeneratedDialect<affine::AffineDialect>();
731 declareGeneratedDialect<func::FuncDialect>();
733 registerTransformOps<
735#include "mlir/Dialect/SCF/TransformOps/SCFTransformOps.cpp.inc"
741#define GET_OP_CLASSES
742#include "mlir/Dialect/SCF/TransformOps/SCFTransformOps.cpp.inc"
static llvm::ManagedStatic< PassManagerOptions > options
Block represents an ordered list of Operations.
Operation * getTerminator()
Get the terminator operation of this block.
The result of a transform IR operation application.
static DiagnosedSilenceableFailure success()
Constructs a DiagnosedSilenceableFailure in the success state.
The DialectRegistry maps a dialect namespace to a constructor for the matching dialect.
void addExtensions()
Add the given extensions to the registry.
A class for computing basic dominance information.
bool properlyDominates(Operation *a, Operation *b, bool enclosingOpOk=true) const
Return true if operation A properly dominates operation B, i.e.
static FlatSymbolRefAttr get(StringAttr value)
Construct a symbol reference for the given value name.
This class defines the main interface for locations in MLIR and acts as a non-nullable wrapper around...
RAII guard to reset the insertion point of the builder when destroyed.
void setInsertionPoint(Block *block, Block::iterator insertPoint)
Set the insertion point to the specified location.
This class represents an operand of an operation.
Operation is the basic unit of execution within MLIR.
Block * getBlock()
Returns the operation block that contains this operation.
unsigned getNumRegions()
Returns the number of regions held by this operation.
Location getLoc()
The source location the operation was defined or derived from.
MutableArrayRef< Region > getRegions()
Returns the regions held by this operation.
result_type_range getResultTypes()
operand_range getOperands()
Returns an iterator on the underlying Value's.
result_range getResults()
This class contains a list of basic blocks and a link to the parent operation it is attached to.
bool hasOneBlock()
Return true if this region has exactly one block.
This class coordinates the application of a rewrite on a set of IR, providing a way for clients to tr...
virtual void replaceOp(Operation *op, ValueRange newValues)
Replace the results of the given (original) operation with the specified list of values (replacements...
virtual void eraseOp(Operation *op)
This method erases an operation that is known to have no uses.
virtual void inlineBlockBefore(Block *source, Block *dest, Block::iterator before, ValueRange argValues={})
Inline the operations of block 'source' into block 'dest' before the given position.
This class allows for representing and managing the symbol table used by operations with the 'SymbolT...
StringAttr insert(Operation *symbol, Block::iterator insertPt={})
Insert a new symbol into the table, and rename it as necessary to avoid collisions.
static Operation * getNearestSymbolTable(Operation *from)
Returns the nearest symbol table from a given operation from.
This class provides an abstraction over the different types of ranges over Values.
This class represents an instance of an SSA value in the MLIR system, representing a computable value...
Base class for extensions of the Transform dialect that supports injecting operations into the Transf...
LogicalResult loopUnrollFull(AffineForOp forOp)
Unrolls this for operation completely if the trip count is known to be constant.
LogicalResult loopUnrollByFactor(AffineForOp forOp, uint64_t unrollFactor, function_ref< void(unsigned, Operation *, OpBuilder)> annotateFn=nullptr, bool cleanUpUnroll=false)
Unrolls this for operation by the specified unroll factor.
LogicalResult loopUnrollJamByFactor(AffineForOp forOp, uint64_t unrollJamFactor)
Unrolls and jams this loop by the specified factor.
void registerTransformDialectExtension(DialectRegistry ®istry)
void populateSCFStructuralTypeConversions(const TypeConverter &typeConverter, RewritePatternSet &patterns, PatternBenefit benefit=1)
Similar to populateSCFStructuralTypeConversionsAndLegality but does not populate the conversion targe...
IfOp convertPartialIterationToIf(RewriterBase &rewriter, ForOp partialIteration)
Rewrite partialIteration, the partial iteration created by peelForLoopAndSimplifyBounds or peelForLoo...
LogicalResult peelForLoopAndSimplifyBounds(RewriterBase &rewriter, ForOp forOp, scf::ForOp &partialIteration)
Rewrite a for loop with bounds/step that potentially do not divide evenly into a for loop where the s...
LogicalResult forallToForLoop(RewriterBase &rewriter, ForallOp forallOp, SmallVectorImpl< Operation * > *results=nullptr)
Try converting scf.forall into a set of nested scf.for loops.
LogicalResult peelForLoopFirstIteration(RewriterBase &rewriter, ForOp forOp, scf::ForOp &partialIteration)
Peel the first iteration out of the scf.for loop.
LogicalResult forallToParallelLoop(RewriterBase &rewriter, ForallOp forallOp, ParallelOp *result=nullptr)
Try converting scf.forall into an scf.parallel loop.
FailureOr< scf::LoopNest > parallelForToNestedFors(RewriterBase &rewriter, ParallelOp parallelOp)
Try converting scf.forall into an scf.parallel loop.
void populateSCFForLoopCanonicalizationPatterns(RewritePatternSet &patterns)
Populate patterns for canonicalizing operations inside SCF loop bodies.
FailureOr< ForOp > pipelineForLoop(RewriterBase &rewriter, ForOp forOp, const PipeliningOption &options, bool *modifiedIR=nullptr)
Generate a pipelined version of the scf.for loop based on the schedule given as option.
void populateSCFStructuralTypeConversionTarget(const TypeConverter &typeConverter, ConversionTarget &target)
Updates the ConversionTarget with dynamic legality of SCF operations based on the provided type conve...
Include the generated interface declarations.
std::optional< int64_t > getConstantIntValue(OpFoldResult ofr)
If ofr is a constant integer or an IntegerAttr, return the integer.
LogicalResult coalescePerfectlyNestedSCFForLoops(scf::ForOp op)
Walk an affine.for to find a band to coalesce.
DiagnosedSilenceableFailure emitSilenceableFailure(Location loc, const Twine &message={})
Emits a silenceable failure with the given message.
DiagnosedDefiniteFailure emitDefiniteFailure(Location loc, const Twine &message={})
Emits a definite failure with the given message.
void populateSCFToControlFlowConversionPatterns(RewritePatternSet &patterns)
Collect a set of patterns to convert SCF operations to CFG branch-based operations within the Control...
llvm::DenseMap< KeyT, ValueT, KeyInfoT, BucketT > DenseMap
FailureOr< func::FuncOp > outlineSingleBlockRegion(RewriterBase &rewriter, Location loc, Region ®ion, StringRef funcName, func::CallOp *callOp=nullptr)
Outline a region with a single block into a new FuncOp.
scf::ForallOp fuseIndependentSiblingForallLoops(scf::ForallOp target, scf::ForallOp source, RewriterBase &rewriter)
Given two scf.forall loops, target and source, fuses target into source.
LogicalResult coalesceLoops(MutableArrayRef< scf::ForOp > loops)
Replace a perfect nest of "for" loops with a single linearized loop.
scf::ForOp fuseIndependentSiblingForLoops(scf::ForOp target, scf::ForOp source, RewriterBase &rewriter)
Given two scf.for loops, target and source, fuses target into source.
void visitUsedValuesDefinedAbove(Region ®ion, Region &limit, function_ref< void(OpOperand *)> callback)
Calls callback for each use of a value within region or its descendants that was defined at the ances...
Options to dictate how loops should be pipelined.