MLIR 24.0.0git
SCFTransformOps.cpp
Go to the documentation of this file.
1//===- SCFTransformOps.cpp - Implementation of SCF transformation ops -----===//
2//
3// Part of the LLVM Project, under the Apache License v2.0 with LLVM Exceptions.
4// See https://llvm.org/LICENSE.txt for license information.
5// SPDX-License-Identifier: Apache-2.0 WITH LLVM-exception
6//
7//===----------------------------------------------------------------------===//
8
10
24#include "mlir/IR/Dominance.h"
26
27using namespace mlir;
28using namespace mlir::affine;
29
30//===----------------------------------------------------------------------===//
31// Apply...PatternsOp
32//===----------------------------------------------------------------------===//
33
34void transform::ApplyForLoopCanonicalizationPatternsOp::populatePatterns(
35 RewritePatternSet &patterns) {
37}
38
39void transform::ApplySCFStructuralConversionPatternsOp::populatePatterns(
40 TypeConverter &typeConverter, RewritePatternSet &patterns) {
41 scf::populateSCFStructuralTypeConversions(typeConverter, patterns);
42}
43
44void transform::ApplySCFStructuralConversionPatternsOp::
45 populateConversionTargetRules(const TypeConverter &typeConverter,
46 ConversionTarget &conversionTarget) {
48 conversionTarget);
49}
50
51void transform::ApplySCFToControlFlowPatternsOp::populatePatterns(
52 TypeConverter &typeConverter, RewritePatternSet &patterns) {
54}
55
56//===----------------------------------------------------------------------===//
57// ForallToForOp
58//===----------------------------------------------------------------------===//
59
61transform::ForallToForOp::apply(transform::TransformRewriter &rewriter,
64 auto payload = state.getPayloadOps(getTarget());
65 if (!llvm::hasSingleElement(payload))
66 return emitSilenceableError() << "expected a single payload op";
67
68 auto target = dyn_cast<scf::ForallOp>(*payload.begin());
69 if (!target) {
71 emitSilenceableError() << "expected the payload to be scf.forall";
72 diag.attachNote((*payload.begin())->getLoc()) << "payload op";
73 return diag;
74 }
75
76 if (!target.getOutputs().empty()) {
77 return emitSilenceableError()
78 << "unsupported shared outputs (didn't bufferize?)";
79 }
80
81 SmallVector<OpFoldResult> lbs = target.getMixedLowerBound();
82
83 if (getNumResults() != lbs.size()) {
85 emitSilenceableError()
86 << "op expects as many results (" << getNumResults()
87 << ") as payload has induction variables (" << lbs.size() << ")";
88 diag.attachNote(target.getLoc()) << "payload op";
89 return diag;
90 }
91
93 if (failed(scf::forallToForLoop(rewriter, target, &opResults))) {
94 DiagnosedSilenceableFailure diag = emitSilenceableError()
95 << "failed to convert forall into for";
96 return diag;
97 }
98
99 for (auto &&[i, res] : llvm::enumerate(opResults)) {
100 results.set(cast<OpResult>(getTransformed()[i]), {res});
101 }
103}
104
105//===----------------------------------------------------------------------===//
106// ForallToForOp
107//===----------------------------------------------------------------------===//
108
110transform::ForallToParallelOp::apply(transform::TransformRewriter &rewriter,
113 auto payload = state.getPayloadOps(getTarget());
114 if (!llvm::hasSingleElement(payload))
115 return emitSilenceableError() << "expected a single payload op";
116
117 auto target = dyn_cast<scf::ForallOp>(*payload.begin());
118 if (!target) {
120 emitSilenceableError() << "expected the payload to be scf.forall";
121 diag.attachNote((*payload.begin())->getLoc()) << "payload op";
122 return diag;
123 }
124
125 if (!target.getOutputs().empty()) {
126 return emitSilenceableError()
127 << "unsupported shared outputs (didn't bufferize?)";
128 }
129
130 if (getNumResults() != 1) {
131 DiagnosedSilenceableFailure diag = emitSilenceableError()
132 << "op expects one result, given "
133 << getNumResults();
134 diag.attachNote(target.getLoc()) << "payload op";
135 return diag;
136 }
137
138 scf::ParallelOp opResult;
139 if (failed(scf::forallToParallelLoop(rewriter, target, &opResult))) {
141 emitSilenceableError() << "failed to convert forall into parallel";
142 return diag;
143 }
144
145 results.set(cast<OpResult>(getTransformed()[0]), {opResult});
147}
148
149//===----------------------------------------------------------------------===//
150// ParallelForToNestedForOps
151//===----------------------------------------------------------------------===//
152
153DiagnosedSilenceableFailure transform::ParallelForToNestedForOps::apply(
156 auto payload = state.getPayloadOps(getTarget());
157 if (!llvm::hasSingleElement(payload))
158 return emitSilenceableError() << "expected a single payload op";
159
160 auto target = dyn_cast<scf::ParallelOp>(*payload.begin());
161 if (!target) {
163 emitSilenceableError() << "expected the payload to be scf.parallel";
164 diag.attachNote((*payload.begin())->getLoc()) << "payload op";
165 return diag;
166 }
167
168 if (getNumResults() != 1) {
169 DiagnosedSilenceableFailure diag = emitSilenceableError()
170 << "op expects one result, given "
171 << getNumResults();
172 diag.attachNote(target.getLoc()) << "payload op";
173 return diag;
174 }
175
176 FailureOr<scf::LoopNest> loopNest =
178 if (failed(loopNest)) {
180 emitSilenceableError() << "failed to convert parallel into nested fors";
181 return diag;
182 }
183
184 results.set(cast<OpResult>(getTransformed()[0]), {loopNest->loops.front()});
186}
187
188//===----------------------------------------------------------------------===//
189// LoopOutlineOp
190//===----------------------------------------------------------------------===//
191
192/// Wraps the given operation `op` into an `scf.execute_region` operation. Uses
193/// the provided rewriter for all operations to remain compatible with the
194/// rewriting infra, as opposed to just splicing the op in place.
195static scf::ExecuteRegionOp wrapInExecuteRegion(RewriterBase &b,
196 Operation *op) {
197 if (op->getNumRegions() != 1)
198 return nullptr;
200 b.setInsertionPoint(op);
201 scf::ExecuteRegionOp executeRegionOp =
202 scf::ExecuteRegionOp::create(b, op->getLoc(), op->getResultTypes());
203 {
205 b.setInsertionPointToStart(&executeRegionOp.getRegion().emplaceBlock());
206 Operation *clonedOp = b.cloneWithoutRegions(*op);
207 Region &clonedRegion = clonedOp->getRegions().front();
208 assert(clonedRegion.empty() && "expected empty region");
209 b.inlineRegionBefore(op->getRegions().front(), clonedRegion,
210 clonedRegion.end());
211 scf::YieldOp::create(b, op->getLoc(), clonedOp->getResults());
212 }
213 b.replaceOp(op, executeRegionOp.getResults());
214 return executeRegionOp;
215}
216
218transform::LoopOutlineOp::apply(transform::TransformRewriter &rewriter,
221 SmallVector<Operation *> functions;
224 for (Operation *target : state.getPayloadOps(getTarget())) {
225 Location location = target->getLoc();
227 scf::ExecuteRegionOp exec = wrapInExecuteRegion(rewriter, target);
228 if (!exec) {
229 DiagnosedSilenceableFailure diag = emitSilenceableError()
230 << "failed to outline";
231 diag.attachNote(target->getLoc()) << "target op";
232 return diag;
233 }
234 func::CallOp call;
235 FailureOr<func::FuncOp> outlined = outlineSingleBlockRegion(
236 rewriter, location, exec.getRegion(), getFuncName(), &call);
237
238 if (failed(outlined))
239 return emitDefaultDefiniteFailure(target);
240
241 if (symbolTableOp) {
242 SymbolTable &symbolTable =
243 symbolTables.try_emplace(symbolTableOp, symbolTableOp)
244 .first->getSecond();
245 symbolTable.insert(*outlined);
246 call.setCalleeAttr(FlatSymbolRefAttr::get(*outlined));
247 }
248 functions.push_back(*outlined);
249 calls.push_back(call);
250 }
251 results.set(cast<OpResult>(getFunction()), functions);
252 results.set(cast<OpResult>(getCall()), calls);
254}
255
256//===----------------------------------------------------------------------===//
257// LoopPeelOp
258//===----------------------------------------------------------------------===//
259
261transform::LoopPeelOp::applyToOne(transform::TransformRewriter &rewriter,
262 scf::ForOp target,
265 scf::ForOp result;
266 if (getPeelFront()) {
267 LogicalResult status =
269 if (failed(status)) {
271 emitSilenceableError() << "failed to peel the first iteration";
272 return diag;
273 }
274 } else {
275 LogicalResult status =
277 if (failed(status)) {
278 DiagnosedSilenceableFailure diag = emitSilenceableError()
279 << "failed to peel the last iteration";
280 return diag;
281 }
282 }
283
284 results.push_back(target);
285 if (getPartialAsIf())
287 else
288 results.push_back(result);
289
291}
292
293//===----------------------------------------------------------------------===//
294// LoopPipelineOp
295//===----------------------------------------------------------------------===//
296
297/// Callback for PipeliningOption. Populates `schedule` with the mapping from an
298/// operation to its logical time position given the iteration interval and the
299/// read latency. The latter is only relevant for vector transfers.
300static void
301loopScheduling(scf::ForOp forOp,
302 std::vector<std::pair<Operation *, unsigned>> &schedule,
303 unsigned iterationInterval, unsigned readLatency) {
304 auto getLatency = [&](Operation *op) -> unsigned {
305 if (isa<vector::TransferReadOp>(op))
306 return readLatency;
307 return 1;
308 };
309
310 std::optional<int64_t> ubConstant =
311 getConstantIntValue(forOp.getUpperBound());
312 std::optional<int64_t> lbConstant =
313 getConstantIntValue(forOp.getLowerBound());
315 std::map<unsigned, std::vector<Operation *>> wrappedSchedule;
316 for (Operation &op : forOp.getBody()->getOperations()) {
317 if (isa<scf::YieldOp>(op))
318 continue;
319 unsigned earlyCycle = 0;
320 for (Value operand : op.getOperands()) {
321 Operation *def = operand.getDefiningOp();
322 if (!def)
323 continue;
324 if (ubConstant && lbConstant) {
325 unsigned ubInt = ubConstant.value();
326 unsigned lbInt = lbConstant.value();
327 auto minLatency = std::min(ubInt - lbInt - 1, getLatency(def));
328 earlyCycle = std::max(earlyCycle, opCycles[def] + minLatency);
329 } else {
330 earlyCycle = std::max(earlyCycle, opCycles[def] + getLatency(def));
331 }
332 }
333 opCycles[&op] = earlyCycle;
334 wrappedSchedule[earlyCycle % iterationInterval].push_back(&op);
335 }
336 for (const auto &it : wrappedSchedule) {
337 for (Operation *op : it.second) {
338 unsigned cycle = opCycles[op];
339 schedule.emplace_back(op, cycle / iterationInterval);
340 }
341 }
342}
343
345transform::LoopPipelineOp::applyToOne(transform::TransformRewriter &rewriter,
346 scf::ForOp target,
350 options.getScheduleFn =
351 [this](scf::ForOp forOp,
352 std::vector<std::pair<Operation *, unsigned>> &schedule) mutable {
353 loopScheduling(forOp, schedule, getIterationInterval(),
354 getReadLatency());
355 };
356 scf::ForLoopPipeliningPattern pattern(options, target->getContext());
357 rewriter.setInsertionPoint(target);
358 FailureOr<scf::ForOp> patternResult =
360 if (succeeded(patternResult)) {
361 results.push_back(*patternResult);
363 }
364 return emitDefaultSilenceableFailure(target);
365}
366
367//===----------------------------------------------------------------------===//
368// LoopPromoteIfOneIterationOp
369//===----------------------------------------------------------------------===//
370
371DiagnosedSilenceableFailure transform::LoopPromoteIfOneIterationOp::applyToOne(
372 transform::TransformRewriter &rewriter, LoopLikeOpInterface target,
375 (void)target.promoteIfSingleIteration(rewriter);
377}
378
379void transform::LoopPromoteIfOneIterationOp::getEffects(
381 consumesHandle(getTargetMutable(), effects);
382 modifiesPayload(effects);
383}
384
385//===----------------------------------------------------------------------===//
386// LoopUnrollOp
387//===----------------------------------------------------------------------===//
388
390transform::LoopUnrollOp::applyToOne(transform::TransformRewriter &rewriter,
391 Operation *op,
394 LogicalResult result(failure());
395 if (scf::ForOp scfFor = dyn_cast<scf::ForOp>(op))
396 result = loopUnrollByFactor(scfFor, getFactor());
397 else if (AffineForOp affineFor = dyn_cast<AffineForOp>(op))
398 result = loopUnrollByFactor(affineFor, getFactor());
399 else
400 return emitSilenceableError()
401 << "failed to unroll, incorrect type of payload";
402
403 if (failed(result))
404 return emitSilenceableError() << "failed to unroll";
405
407}
408
409//===----------------------------------------------------------------------===//
410// LoopUnrollFullOp
411//===----------------------------------------------------------------------===//
412
413DiagnosedSilenceableFailure transform::LoopUnrollFullOp::applyToOne(
417 LogicalResult result(failure());
418 if (scf::ForOp scfFor = dyn_cast<scf::ForOp>(op))
419 result = loopUnrollFull(scfFor);
420 else if (AffineForOp affineFor = dyn_cast<AffineForOp>(op))
421 result = loopUnrollFull(affineFor);
422 else
423 return emitSilenceableError()
424 << "failed to fully unroll, incorrect type of payload";
425
426 if (failed(result))
427 return emitSilenceableError() << "failed to fully unroll";
428
430}
431
432//===----------------------------------------------------------------------===//
433// LoopUnrollAndJamOp
434//===----------------------------------------------------------------------===//
435
436DiagnosedSilenceableFailure transform::LoopUnrollAndJamOp::applyToOne(
440 LogicalResult result(failure());
441 if (scf::ForOp scfFor = dyn_cast<scf::ForOp>(op))
442 result = loopUnrollJamByFactor(scfFor, getFactor());
443 else if (AffineForOp affineFor = dyn_cast<AffineForOp>(op))
444 result = loopUnrollJamByFactor(affineFor, getFactor());
445 else
446 return emitSilenceableError()
447 << "failed to unroll and jam, incorrect type of payload";
448
449 if (failed(result))
450 return emitSilenceableError() << "failed to unroll and jam";
451
453}
454
455//===----------------------------------------------------------------------===//
456// LoopCoalesceOp
457//===----------------------------------------------------------------------===//
458
460transform::LoopCoalesceOp::applyToOne(transform::TransformRewriter &rewriter,
461 Operation *op,
464 LogicalResult result(failure());
465 if (scf::ForOp scfForOp = dyn_cast<scf::ForOp>(op))
467 else if (AffineForOp affineForOp = dyn_cast<AffineForOp>(op))
468 result = coalescePerfectlyNestedAffineLoops(affineForOp);
469
470 results.push_back(op);
471 if (failed(result)) {
472 DiagnosedSilenceableFailure diag = emitSilenceableError()
473 << "failed to coalesce";
474 return diag;
475 }
477}
478
479//===----------------------------------------------------------------------===//
480// LoopCoalesceNestedOp
481//===----------------------------------------------------------------------===//
482DiagnosedSilenceableFailure transform::LoopCoalesceNestedOp::applyToOne(
486 auto forOp = dyn_cast<scf::ForOp>(op);
487 if (!forOp) {
488 return emitSilenceableError() << "expected scf.for operation";
489 }
490
491 // Collect nested loops (including imperfectly nested ones)
492 SmallVector<scf::ForOp> nestedLoops;
493 scf::ForOp currentLoop = forOp;
494
495 while (currentLoop) {
496 nestedLoops.push_back(currentLoop);
497 Block &body = currentLoop.getRegion().front();
498
499 // Look for the next nested loop
500 scf::ForOp nextLoop = nullptr;
501 for (Operation &bodyOp : body) {
502 if (auto innerFor = dyn_cast<scf::ForOp>(&bodyOp)) {
503 nextLoop = innerFor;
504 break;
505 }
506 }
507
508 currentLoop = nextLoop;
509 }
510
511 // Need at least 2 loops to coalesce
512 if (nestedLoops.size() < 2) {
513 return emitSilenceableError() << "need at least 2 nested loops to coalesce";
514 }
515
516 // Call coalesceLoops directly
517 if (failed(coalesceLoops(rewriter, nestedLoops))) {
518 return emitSilenceableError() << "failed to coalesce nested loops";
519 }
520
521 results.push_back(nestedLoops.front());
523}
524
525//===----------------------------------------------------------------------===//
526// TakeAssumedBranchOp
527//===----------------------------------------------------------------------===//
528/// Replaces the given op with the contents of the given single-block region,
529/// using the operands of the block terminator to replace operation results.
530static void replaceOpWithRegion(RewriterBase &rewriter, Operation *op,
531 Region &region) {
532 assert(region.hasOneBlock() && "expected single-block region");
533 Block *block = &region.front();
534 Operation *terminator = block->getTerminator();
535 ValueRange results = terminator->getOperands();
536 rewriter.inlineBlockBefore(block, op, /*blockArgs=*/{});
537 rewriter.replaceOp(op, results);
538 rewriter.eraseOp(terminator);
539}
540
541DiagnosedSilenceableFailure transform::TakeAssumedBranchOp::applyToOne(
542 transform::TransformRewriter &rewriter, scf::IfOp ifOp,
545 rewriter.setInsertionPoint(ifOp);
546 Region &region =
547 getTakeElseBranch() ? ifOp.getElseRegion() : ifOp.getThenRegion();
548 if (!region.hasOneBlock()) {
549 return emitDefiniteFailure()
550 << "requires an scf.if op with a single-block "
551 << ((getTakeElseBranch()) ? "`else`" : "`then`") << " region";
552 }
553 replaceOpWithRegion(rewriter, ifOp, region);
555}
556
557void transform::TakeAssumedBranchOp::getEffects(
559 onlyReadsHandle(getTargetMutable(), effects);
560 modifiesPayload(effects);
561}
562
563//===----------------------------------------------------------------------===//
564// LoopFuseSiblingOp
565//===----------------------------------------------------------------------===//
566
567/// Check if `target` and `source` are siblings, in the context that `target`
568/// is being fused into `source`.
569///
570/// This is a simple check that just checks if both operations are in the same
571/// block and some checks to ensure that the fused IR does not violate
572/// dominance.
574 Operation *source) {
575 // Check if both operations are same.
576 if (target == source)
577 return emitSilenceableFailure(source)
578 << "target and source need to be different loops";
579
580 // Check if both operations are in the same block.
581 if (target->getBlock() != source->getBlock())
582 return emitSilenceableFailure(source)
583 << "target and source are not in the same block";
584
585 // Check if fusion will violate dominance.
586 DominanceInfo domInfo(source);
587 if (target->isBeforeInBlock(source)) {
588 // Since `target` is before `source`, all users of results of `target`
589 // need to be dominated by `source`.
590 for (Operation *user : target->getUsers()) {
591 if (!domInfo.properlyDominates(source, user, /*enclosingOpOk=*/false)) {
593 << "user of results of target should be properly dominated by "
594 "source";
595 }
596 }
597 } else {
598 // Since `target` is after `source`, all values used by `target` need
599 // to dominate `source`.
600
601 // Check if operands of `target` are dominated by `source`.
602 for (Value operand : target->getOperands()) {
603 Operation *operandOp = operand.getDefiningOp();
604 // Operands without defining operations are block arguments. When `target`
605 // and `source` occur in the same block, these operands dominate `source`.
606 if (!operandOp)
607 continue;
608
609 // Operand's defining operation should properly dominate `source`.
610 if (!domInfo.properlyDominates(operandOp, source,
611 /*enclosingOpOk=*/false))
613 << "operands of target should be properly dominated by source";
614 }
615
616 // Check if values used by `target` are dominated by `source`.
617 bool failed = false;
618 OpOperand *failedValue = nullptr;
619 visitUsedValuesDefinedAbove(target->getRegions(), [&](OpOperand *operand) {
620 Operation *operandOp = operand->get().getDefiningOp();
621 if (operandOp && !domInfo.properlyDominates(operandOp, source,
622 /*enclosingOpOk=*/false)) {
623 // `operand` is not an argument of an enclosing block and the defining
624 // op of `operand` is outside `target` but does not dominate `source`.
625 failed = true;
626 failedValue = operand;
627 }
628 });
629
630 if (failed)
631 return emitSilenceableFailure(failedValue->getOwner())
632 << "values used inside regions of target should be properly "
633 "dominated by source";
634 }
635
637}
638
639/// Check if `target` scf.forall can be fused into `source` scf.forall.
640///
641/// This simply checks if both loops have the same bounds, steps and mapping.
642/// No attempt is made at checking that the side effects of `target` and
643/// `source` are independent of each other.
645 Operation *source) {
646 auto targetOp = dyn_cast<scf::ForallOp>(target);
647 auto sourceOp = dyn_cast<scf::ForallOp>(source);
648 if (!targetOp || !sourceOp)
649 return false;
650
651 return targetOp.getMixedLowerBound() == sourceOp.getMixedLowerBound() &&
652 targetOp.getMixedUpperBound() == sourceOp.getMixedUpperBound() &&
653 targetOp.getMixedStep() == sourceOp.getMixedStep() &&
654 targetOp.getMapping() == sourceOp.getMapping();
655}
656
657/// Check if `target` scf.for can be fused into `source` scf.for.
658///
659/// This simply checks if both loops have the same bounds and steps. No attempt
660/// is made at checking that the side effects of `target` and `source` are
661/// independent of each other.
663 Operation *source) {
664 auto targetOp = dyn_cast<scf::ForOp>(target);
665 auto sourceOp = dyn_cast<scf::ForOp>(source);
666 if (!targetOp || !sourceOp)
667 return false;
668
669 return targetOp.getLowerBound() == sourceOp.getLowerBound() &&
670 targetOp.getUpperBound() == sourceOp.getUpperBound() &&
671 targetOp.getStep() == sourceOp.getStep();
672}
673
675transform::LoopFuseSiblingOp::apply(transform::TransformRewriter &rewriter,
678 auto targetOps = state.getPayloadOps(getTarget());
679 auto sourceOps = state.getPayloadOps(getSource());
680
681 if (!llvm::hasSingleElement(targetOps) ||
682 !llvm::hasSingleElement(sourceOps)) {
683 return emitDefiniteFailure()
684 << "requires exactly one target handle (got "
685 << llvm::range_size(targetOps) << ") and exactly one "
686 << "source handle (got " << llvm::range_size(sourceOps) << ")";
687 }
688
689 Operation *target = *targetOps.begin();
690 Operation *source = *sourceOps.begin();
691
692 // Check if the target and source are siblings.
694 if (!diag.succeeded())
695 return diag;
696
697 Operation *fusedLoop;
698 /// TODO: Support fusion for loop-like ops besides scf.for and scf.forall.
701 cast<scf::ForOp>(target), cast<scf::ForOp>(source), rewriter);
702 } else if (isForallWithIdenticalConfiguration(target, source)) {
704 cast<scf::ForallOp>(target), cast<scf::ForallOp>(source), rewriter);
705 } else {
706 return emitSilenceableFailure(target->getLoc())
707 << "operations cannot be fused";
708 }
709
710 assert(fusedLoop && "failed to fuse operations");
711
712 results.set(cast<OpResult>(getFusedLoop()), {fusedLoop});
714}
715
716//===----------------------------------------------------------------------===//
717// Transform op registration
718//===----------------------------------------------------------------------===//
719
720namespace {
721class SCFTransformDialectExtension
723 SCFTransformDialectExtension> {
724public:
725 MLIR_DEFINE_EXPLICIT_INTERNAL_INLINE_TYPE_ID(SCFTransformDialectExtension)
726
727 using Base::Base;
728
729 void init() {
730 declareGeneratedDialect<affine::AffineDialect>();
731 declareGeneratedDialect<func::FuncDialect>();
732
733 registerTransformOps<
734#define GET_OP_LIST
735#include "mlir/Dialect/SCF/TransformOps/SCFTransformOps.cpp.inc"
736 >();
737 }
738};
739} // namespace
740
741#define GET_OP_CLASSES
742#include "mlir/Dialect/SCF/TransformOps/SCFTransformOps.cpp.inc"
743
745 registry.addExtensions<SCFTransformDialectExtension>();
746}
b
Return true if permutation is a valid permutation of the outer_dims_perm (case OuterOrInnerPerm::Oute...
static llvm::ManagedStatic< PassManagerOptions > options
static bool isForWithIdenticalConfiguration(Operation *target, Operation *source)
Check if target scf.for can be fused into source scf.for.
#define MLIR_DEFINE_EXPLICIT_INTERNAL_INLINE_TYPE_ID(CLASS_NAME)
static DiagnosedSilenceableFailure isOpSibling(Operation *target, Operation *source)
Check if target and source are siblings, in the context that target is being fused into source.
static void loopScheduling(scf::ForOp forOp, std::vector< std::pair< Operation *, unsigned > > &schedule, unsigned iterationInterval, unsigned readLatency)
Callback for PipeliningOption.
static bool isForallWithIdenticalConfiguration(Operation *target, Operation *source)
Check if target scf.forall can be fused into source scf.forall.
static void replaceOpWithRegion(RewriterBase &rewriter, Operation *op, Region &region)
Replaces the given op with the contents of the given single-block region, using the operands of the b...
static scf::ExecuteRegionOp wrapInExecuteRegion(RewriterBase &b, Operation *op)
Wraps the given operation op into an scf.execute_region operation.
Block represents an ordered list of Operations.
Definition Block.h:34
Operation & front()
Definition Block.h:178
Operation * getTerminator()
Get the terminator operation of this block.
Definition Block.cpp:249
The result of a transform IR operation application.
static DiagnosedSilenceableFailure success()
Constructs a DiagnosedSilenceableFailure in the success state.
The DialectRegistry maps a dialect namespace to a constructor for the matching dialect.
void addExtensions()
Add the given extensions to the registry.
A class for computing basic dominance information.
Definition Dominance.h:143
bool properlyDominates(Operation *a, Operation *b, bool enclosingOpOk=true) const
Return true if operation A properly dominates operation B, i.e.
static FlatSymbolRefAttr get(StringAttr value)
Construct a symbol reference for the given value name.
This class defines the main interface for locations in MLIR and acts as a non-nullable wrapper around...
Definition Location.h:76
RAII guard to reset the insertion point of the builder when destroyed.
Definition Builders.h:351
void setInsertionPoint(Block *block, Block::iterator insertPoint)
Set the insertion point to the specified location.
Definition Builders.h:401
This class represents an operand of an operation.
Definition Value.h:256
Operation is the basic unit of execution within MLIR.
Definition Operation.h:87
Block * getBlock()
Returns the operation block that contains this operation.
Definition Operation.h:230
unsigned getNumRegions()
Returns the number of regions held by this operation.
Definition Operation.h:728
Location getLoc()
The source location the operation was defined or derived from.
Definition Operation.h:240
MutableArrayRef< Region > getRegions()
Returns the regions held by this operation.
Definition Operation.h:731
result_type_range getResultTypes()
Definition Operation.h:455
operand_range getOperands()
Returns an iterator on the underlying Value's.
Definition Operation.h:405
result_range getResults()
Definition Operation.h:442
This class contains a list of basic blocks and a link to the parent operation it is attached to.
Definition Region.h:26
Block & front()
Definition Region.h:65
bool empty()
Definition Region.h:60
iterator end()
Definition Region.h:56
bool hasOneBlock()
Return true if this region has exactly one block.
Definition Region.h:68
This class coordinates the application of a rewrite on a set of IR, providing a way for clients to tr...
virtual void replaceOp(Operation *op, ValueRange newValues)
Replace the results of the given (original) operation with the specified list of values (replacements...
virtual void eraseOp(Operation *op)
This method erases an operation that is known to have no uses.
virtual void inlineBlockBefore(Block *source, Block *dest, Block::iterator before, ValueRange argValues={})
Inline the operations of block 'source' into block 'dest' before the given position.
This class allows for representing and managing the symbol table used by operations with the 'SymbolT...
Definition SymbolTable.h:25
StringAttr insert(Operation *symbol, Block::iterator insertPt={})
Insert a new symbol into the table, and rename it as necessary to avoid collisions.
static Operation * getNearestSymbolTable(Operation *from)
Returns the nearest symbol table from a given operation from.
This class provides an abstraction over the different types of ranges over Values.
Definition ValueRange.h:392
This class represents an instance of an SSA value in the MLIR system, representing a computable value...
Definition Value.h:96
A list of results of applying a transform op with ApplyEachOpTrait to a single payload operation,...
void push_back(Operation *op)
Appends an element to the list.
Base class for extensions of the Transform dialect that supports injecting operations into the Transf...
Local mapping between values defined by a specific op implementing the TransformOpInterface and the p...
void set(OpResult value, Range &&ops)
Indicates that the result of the transform IR op at the given position corresponds to the given list ...
This is a special rewriter to be used in transform op implementations, providing additional helper fu...
The state maintained across applications of various ops implementing the TransformOpInterface.
auto getPayloadOps(Value value) const
Returns an iterator that enumerates all ops that the given transform IR value corresponds to.
LogicalResult loopUnrollFull(AffineForOp forOp)
Unrolls this for operation completely if the trip count is known to be constant.
LogicalResult loopUnrollByFactor(AffineForOp forOp, uint64_t unrollFactor, function_ref< void(unsigned, Operation *, OpBuilder)> annotateFn=nullptr, bool cleanUpUnroll=false)
Unrolls this for operation by the specified unroll factor.
LogicalResult loopUnrollJamByFactor(AffineForOp forOp, uint64_t unrollJamFactor)
Unrolls and jams this loop by the specified factor.
detail::InFlightRemark failed(Location loc, RemarkOpts opts)
Report an optimization remark that failed.
Definition Remarks.h:853
void registerTransformDialectExtension(DialectRegistry &registry)
void populateSCFStructuralTypeConversions(const TypeConverter &typeConverter, RewritePatternSet &patterns, PatternBenefit benefit=1)
Similar to populateSCFStructuralTypeConversionsAndLegality but does not populate the conversion targe...
IfOp convertPartialIterationToIf(RewriterBase &rewriter, ForOp partialIteration)
Rewrite partialIteration, the partial iteration created by peelForLoopAndSimplifyBounds or peelForLoo...
LogicalResult peelForLoopAndSimplifyBounds(RewriterBase &rewriter, ForOp forOp, scf::ForOp &partialIteration)
Rewrite a for loop with bounds/step that potentially do not divide evenly into a for loop where the s...
LogicalResult forallToForLoop(RewriterBase &rewriter, ForallOp forallOp, SmallVectorImpl< Operation * > *results=nullptr)
Try converting scf.forall into a set of nested scf.for loops.
LogicalResult peelForLoopFirstIteration(RewriterBase &rewriter, ForOp forOp, scf::ForOp &partialIteration)
Peel the first iteration out of the scf.for loop.
LogicalResult forallToParallelLoop(RewriterBase &rewriter, ForallOp forallOp, ParallelOp *result=nullptr)
Try converting scf.forall into an scf.parallel loop.
FailureOr< scf::LoopNest > parallelForToNestedFors(RewriterBase &rewriter, ParallelOp parallelOp)
Try converting scf.forall into an scf.parallel loop.
void populateSCFForLoopCanonicalizationPatterns(RewritePatternSet &patterns)
Populate patterns for canonicalizing operations inside SCF loop bodies.
FailureOr< ForOp > pipelineForLoop(RewriterBase &rewriter, ForOp forOp, const PipeliningOption &options, bool *modifiedIR=nullptr)
Generate a pipelined version of the scf.for loop based on the schedule given as option.
void populateSCFStructuralTypeConversionTarget(const TypeConverter &typeConverter, ConversionTarget &target)
Updates the ConversionTarget with dynamic legality of SCF operations based on the provided type conve...
void consumesHandle(MutableArrayRef< OpOperand > handles, SmallVectorImpl< MemoryEffects::EffectInstance > &effects)
Populates effects with the memory effects indicating the operation on the given handle value:
void onlyReadsHandle(MutableArrayRef< OpOperand > handles, SmallVectorImpl< MemoryEffects::EffectInstance > &effects)
void modifiesPayload(SmallVectorImpl< MemoryEffects::EffectInstance > &effects)
Populates effects with the memory effects indicating the access to payload IR resource.
Include the generated interface declarations.
std::optional< int64_t > getConstantIntValue(OpFoldResult ofr)
If ofr is a constant integer or an IntegerAttr, return the integer.
LogicalResult coalescePerfectlyNestedSCFForLoops(scf::ForOp op)
Walk an affine.for to find a band to coalesce.
Definition Utils.cpp:1104
DiagnosedSilenceableFailure emitSilenceableFailure(Location loc, const Twine &message={})
Emits a silenceable failure with the given message.
DiagnosedDefiniteFailure emitDefiniteFailure(Location loc, const Twine &message={})
Emits a definite failure with the given message.
void populateSCFToControlFlowConversionPatterns(RewritePatternSet &patterns)
Collect a set of patterns to convert SCF operations to CFG branch-based operations within the Control...
llvm::DenseMap< KeyT, ValueT, KeyInfoT, BucketT > DenseMap
Definition LLVM.h:123
FailureOr< func::FuncOp > outlineSingleBlockRegion(RewriterBase &rewriter, Location loc, Region &region, StringRef funcName, func::CallOp *callOp=nullptr)
Outline a region with a single block into a new FuncOp.
Definition Utils.cpp:115
scf::ForallOp fuseIndependentSiblingForallLoops(scf::ForallOp target, scf::ForallOp source, RewriterBase &rewriter)
Given two scf.forall loops, target and source, fuses target into source.
Definition Utils.cpp:1478
LogicalResult coalesceLoops(MutableArrayRef< scf::ForOp > loops)
Replace a perfect nest of "for" loops with a single linearized loop.
Definition Utils.cpp:1096
scf::ForOp fuseIndependentSiblingForLoops(scf::ForOp target, scf::ForOp source, RewriterBase &rewriter)
Given two scf.for loops, target and source, fuses target into source.
Definition Utils.cpp:1531
void visitUsedValuesDefinedAbove(Region &region, Region &limit, function_ref< void(OpOperand *)> callback)
Calls callback for each use of a value within region or its descendants that was defined at the ances...
Options to dictate how loops should be pipelined.
Definition Transforms.h:140