MLIR 24.0.0git
LoopSpecialization.cpp
Go to the documentation of this file.
1//===- LoopSpecialization.cpp - scf.parallel/SCR.for specialization -------===//
2//
3// Part of the LLVM Project, under the Apache License v2.0 with LLVM Exceptions.
4// See https://llvm.org/LICENSE.txt for license information.
5// SPDX-License-Identifier: Apache-2.0 WITH LLVM-exception
6//
7//===----------------------------------------------------------------------===//
8//
9// Specializes parallel loops and for loops for easier unrolling and
10// vectorization.
11//
12//===----------------------------------------------------------------------===//
13
15
23#include "mlir/IR/AffineExpr.h"
24#include "mlir/IR/IRMapping.h"
27
28namespace mlir {
29#define GEN_PASS_DEF_SCFFORLOOPPEELING
30#define GEN_PASS_DEF_SCFFORLOOPSPECIALIZATION
31#define GEN_PASS_DEF_SCFPARALLELLOOPSPECIALIZATION
32#include "mlir/Dialect/SCF/Transforms/Passes.h.inc"
33} // namespace mlir
34
35using namespace mlir;
36using namespace mlir::affine;
37using scf::ForOp;
38using scf::ParallelOp;
39
40/// Rewrite a parallel loop with bounds defined by an affine.min with a constant
41/// into 2 loops after checking if the bounds are equal to that constant. This
42/// is beneficial if the loop will almost always have the constant bound and
43/// that version can be fully unrolled and vectorized.
44static void specializeParallelLoopForUnrolling(ParallelOp op) {
45 SmallVector<int64_t, 2> constantIndices;
46 constantIndices.reserve(op.getUpperBound().size());
47 for (auto bound : op.getUpperBound()) {
48 auto minOp = bound.getDefiningOp<AffineMinOp>();
49 if (!minOp)
50 return;
51 int64_t minConstant = std::numeric_limits<int64_t>::max();
52 for (AffineExpr expr : minOp.getMap().getResults()) {
53 if (auto constantIndex = dyn_cast<AffineConstantExpr>(expr))
54 minConstant = std::min(minConstant, constantIndex.getValue());
55 }
56 if (minConstant == std::numeric_limits<int64_t>::max())
57 return;
58 constantIndices.push_back(minConstant);
59 }
60
61 OpBuilder b(op);
62 IRMapping map;
63 Value cond;
64 for (auto bound : llvm::zip(op.getUpperBound(), constantIndices)) {
65 Value constant =
66 arith::ConstantIndexOp::create(b, op.getLoc(), std::get<1>(bound));
67 Value cmp = arith::CmpIOp::create(b, op.getLoc(), arith::CmpIPredicate::eq,
68 std::get<0>(bound), constant);
69 cond = cond ? arith::AndIOp::create(b, op.getLoc(), cond, cmp) : cmp;
70 map.map(std::get<0>(bound), constant);
71 }
72 auto ifOp = scf::IfOp::create(b, op.getLoc(), op.getResultTypes(), cond,
73 /*addThenBlock=*/true, /*addElseBlock=*/true);
74
75 OpBuilder thenBuilder = OpBuilder::atBlockEnd(&ifOp.getThenRegion().front());
76 Operation *thenOp = thenBuilder.clone(*op.getOperation(), map);
77 scf::YieldOp::create(thenBuilder, op.getLoc(), thenOp->getResults());
78
79 OpBuilder elseBuilder = OpBuilder::atBlockEnd(&ifOp.getElseRegion().front());
80 Operation *elseOp = elseBuilder.clone(*op.getOperation());
81 scf::YieldOp::create(elseBuilder, op.getLoc(), elseOp->getResults());
82
83 op.replaceAllUsesWith(ifOp.getResults());
84 op.erase();
85}
86
87/// Rewrite a for loop with bounds defined by an affine.min with a constant into
88/// 2 loops after checking if the bounds are equal to that constant. This is
89/// beneficial if the loop will almost always have the constant bound and that
90/// version can be fully unrolled and vectorized.
91static void specializeForLoopForUnrolling(ForOp op) {
92 auto bound = op.getUpperBound();
93 auto minOp = bound.getDefiningOp<AffineMinOp>();
94 if (!minOp)
95 return;
96 AffineMap canonicalizedMap = minOp.getMap();
97 SmallVector<Value> operands(minOp.getMapOperands());
98 affine::canonicalizeMapAndOperands(&canonicalizedMap, &operands);
99 int64_t minConstant = std::numeric_limits<int64_t>::max();
100 for (AffineExpr expr : canonicalizedMap.getResults()) {
101 if (auto constantIndex = dyn_cast<AffineConstantExpr>(expr))
102 minConstant = std::min(minConstant, constantIndex.getValue());
103 }
104 if (minConstant == std::numeric_limits<int64_t>::max())
105 return;
106
107 OpBuilder b(op);
108 IRMapping map;
109 Value constant = arith::ConstantOp::create(
110 b, op.getLoc(),
111 IntegerAttr::get(op.getUpperBound().getType(), minConstant));
112 Value cond = arith::CmpIOp::create(b, op.getLoc(), arith::CmpIPredicate::eq,
113 bound, constant);
114 map.map(bound, constant);
115 auto ifOp = scf::IfOp::create(b, op.getLoc(), op->getResultTypes(), cond,
116 /*withElseRegion=*/true);
117
118 OpBuilder thenB = ifOp.getThenBodyBuilder();
119 Operation *thenLoop = thenB.clone(*op.getOperation(), map);
120 OpBuilder elseB = ifOp.getElseBodyBuilder();
121 Operation *elseLoop = elseB.clone(*op.getOperation());
122
123 if (op->getNumResults() > 0) {
124 scf::YieldOp::create(thenB, op.getLoc(), thenLoop->getResults());
125 scf::YieldOp::create(elseB, op.getLoc(), elseLoop->getResults());
126 op->replaceAllUsesWith(ifOp.getResults());
127 }
128 op.erase();
129}
130
131/// Rewrite a for loop with bounds/step that potentially do not divide evenly
132/// into a for loop where the step divides the iteration space evenly, followed
133/// by another scf.for for the last (partial) iteration (if any).
134///
135/// This function rewrites the given scf.for loop in-place and creates a new
136/// scf.for operation for the last iteration. It replaces all uses of the
137/// unpeeled loop with the results of the newly generated scf.for.
138///
139/// The newly generated scf.for operation is returned via `partialIteration`.
140/// The boundary at which the loop is split (new upper bound) is returned via
141/// `splitBound`. The return value indicates whether the loop was rewritten or
142/// not.
143///
144/// Note: Loops with a step size of 0 cannot be peeled. Applying this function
145/// to such a loop may result in IR with undefined behavior.
146static LogicalResult peelForLoop(RewriterBase &b, ForOp forOp,
147 ForOp &partialIteration, Value &splitBound) {
149 Value lbVal = forOp.getLowerBound();
150 Value ubVal = forOp.getUpperBound();
151 Value stepVal = forOp.getStep();
152 b.setInsertionPoint(forOp);
153 auto loc = forOp.getLoc();
154
155 bool isUnsigned = forOp.getUnsignedCmp();
156 IntegerAttr lbAttr, ubAttr, stepAttr;
157 if (matchPattern(stepVal, m_Constant(&stepAttr))) {
158 APInt step = stepAttr.getValue();
159 if (isUnsigned ? step.ule(1) : step.sle(1))
160 return failure();
161 }
162 if (matchPattern(lbVal, m_Constant(&lbAttr)) &&
163 matchPattern(ubVal, m_Constant(&ubAttr))) {
164
165 APInt lb = lbAttr.getValue();
166 APInt ub = ubAttr.getValue();
167
168 if (isUnsigned ? ub.ule(lb) : ub.sle(lb))
169 return failure();
170
171 // lb < ub here, so ub - lb as an unsigned value is the exact distance, even
172 // for signed loops whose range exceeds the signed max. hence urem for both.
173 if (stepAttr && (ub - lb).urem(stepAttr.getValue()).isZero())
174 return failure();
175 }
176
177 // affine reasoning is signed and index-only,
178 // so use it only for signed index loops.
179 if (!isUnsigned && forOp.getInductionVar().getType().isIndex()) {
180 AffineExpr sym0, sym1, sym2;
181 bindSymbols(b.getContext(), sym0, sym1, sym2);
182 SmallVector<Value> operands{forOp.getLowerBound(), forOp.getUpperBound(),
183 forOp.getStep()};
184 AffineMap map = AffineMap::get(0, 3, {(sym1 - sym0) % sym2});
186 if (auto constExpr = dyn_cast<AffineConstantExpr>(map.getResult(0)))
187 if (constExpr.getValue() == 0)
188 return failure();
189 }
190
191 // split = ub - ((ub - min(lb, ub)) remui step), with a signed or unsigned min
192 // to match the loop. For ub <= lb the min gives a zero remainder, so
193 // split = ub and both loops stay empty. For ub > lb the wrapping subtraction
194 // is the exact distance when read as unsigned, even beyond the signed range,
195 // which is why remui is used for signed loops as well.
196 Value lbEff = isUnsigned ? b.createOrFold<arith::MinUIOp>(loc, lbVal, ubVal)
197 : b.createOrFold<arith::MinSIOp>(loc, lbVal, ubVal);
198 Value diff = b.createOrFold<arith::SubIOp>(loc, ubVal, lbEff);
199 Value rem = b.createOrFold<arith::RemUIOp>(loc, diff, stepVal);
200 splitBound = b.createOrFold<arith::SubIOp>(loc, ubVal, rem);
201
202 // Create ForOp for partial iteration.
203 b.setInsertionPointAfter(forOp);
204 partialIteration = cast<ForOp>(b.clone(*forOp.getOperation()));
205 partialIteration.getLowerBoundMutable().assign(splitBound);
206 b.replaceAllUsesWith(forOp.getResults(), partialIteration->getResults());
207 partialIteration.getInitArgsMutable().assign(forOp->getResults());
208
209 // Set new upper loop bound.
210 b.modifyOpInPlace(forOp,
211 [&]() { forOp.getUpperBoundMutable().assign(splitBound); });
212
213 return success();
214}
215
216static void rewriteAffineOpAfterPeeling(RewriterBase &rewriter, ForOp forOp,
217 ForOp partialIteration,
218 Value previousUb) {
219 Value mainIv = forOp.getInductionVar();
220 Value partialIv = partialIteration.getInductionVar();
221 assert(forOp.getStep() == partialIteration.getStep() &&
222 "expected same step in main and partial loop");
223 Value step = forOp.getStep();
224
225 forOp.walk([&](Operation *affineOp) {
226 if (!isa<AffineMinOp, AffineMaxOp>(affineOp))
227 return WalkResult::advance();
228 (void)scf::rewritePeeledMinMaxOp(rewriter, affineOp, mainIv, previousUb,
229 step,
230 /*insideLoop=*/true);
231 return WalkResult::advance();
232 });
233 partialIteration.walk([&](Operation *affineOp) {
234 if (!isa<AffineMinOp, AffineMaxOp>(affineOp))
235 return WalkResult::advance();
236 (void)scf::rewritePeeledMinMaxOp(rewriter, affineOp, partialIv, previousUb,
237 step, /*insideLoop=*/false);
238 return WalkResult::advance();
239 });
240}
241
243 ForOp forOp,
244 ForOp &partialIteration) {
245 Value previousUb = forOp.getUpperBound();
246 Value splitBound;
247 if (failed(peelForLoop(rewriter, forOp, partialIteration, splitBound)))
248 return failure();
249
250 // The affine.min/max simplification relies on signed reasoning, which does
251 // not hold for unsigned loops, so leave those ops untouched.
252 if (forOp.getUnsignedCmp())
253 return success();
254
255 // Rewrite affine.min and affine.max ops.
256 rewriteAffineOpAfterPeeling(rewriter, forOp, partialIteration, previousUb);
257
258 return success();
259}
260
261/// Rewrites the original scf::ForOp as two scf::ForOp Ops, the first
262/// scf::ForOp corresponds to the first iteration of the loop which can be
263/// canonicalized away in the following optimizations. The second loop Op
264/// contains the remaining iterations, with a lower bound updated as the
265/// original lower bound plus the step (i.e. skips the first iteration).
266LogicalResult mlir::scf::peelForLoopFirstIteration(RewriterBase &b, ForOp forOp,
267 ForOp &firstIteration) {
269 Value lbVal = forOp.getLowerBound();
270 Value ubVal = forOp.getUpperBound();
271 Value stepVal = forOp.getStep();
272 auto loc = forOp.getLoc();
273
274 Value splitBound;
275 b.setInsertionPoint(forOp);
276
277 // Check whether we have a unsigned loop
278 if (forOp.getUnsignedCmp()) {
279 IntegerAttr lbAttr, ubAttr, stepAttr;
280 if (matchPattern(lbVal, m_Constant(&lbAttr)) &&
281 matchPattern(ubVal, m_Constant(&ubAttr)) &&
282 matchPattern(stepVal, m_Constant(&stepAttr))) {
283
284 // Get the constants as APInt at the loop’s own bit width
285 APInt lb = lbAttr.getValue();
286 APInt ub = ubAttr.getValue();
287 APInt step = stepAttr.getValue();
288
289 // bail out if ub <= lb or ub - lb <= step (unsigned constants)
290 if (ub.ule(lb) || (ub - lb).ule(step))
291 return failure();
292 }
293
294 // split = max(lb, min(lb + step, ub)): the first loop runs one iteration,
295 // or none if the loop is empty. If lb + step wraps, the loop has at most
296 // one iteration and split falls back to lb, so the main loop does the work.
297 // use unsigned ops (MinUIOp, MaxUIOp)
298 Value sum = b.createOrFold<arith::AddIOp>(loc, lbVal, stepVal);
299 Value capped = b.createOrFold<arith::MinUIOp>(loc, sum, ubVal);
300 splitBound = b.createOrFold<arith::MaxUIOp>(loc, lbVal, capped);
301 } else {
302 auto lbInt = getConstantIntValue(lbVal);
303 auto ubInt = getConstantIntValue(ubVal);
304 auto stepInt = getConstantIntValue(stepVal);
305
306 // Peeling is not needed if there is at most one iteration. scf.for requires
307 // a positive step, so the trip count is ceil((UB - LB) / Step) when UB > LB
308 // and 0 otherwise. With a positive step, "UB - LB <= Step" therefore covers
309 // both the empty loop and the single-iteration loop. Bail out if the
310 // subtraction overflows, since we can't reason about the trip count then.
311 if (lbInt && ubInt && stepInt) {
312 int64_t diff;
313 if (llvm::SubOverflow(*ubInt, *lbInt, diff) || diff <= *stepInt)
314 return failure();
315 }
316
317 // split = max(lb, min(lb + step, ub)): the first loop runs one iteration,
318 // or none if the loop is empty. If lb + step wraps, the loop has at most
319 // one iteration and split falls back to lb, so the main loop does the work.
320 Value sum = b.createOrFold<arith::AddIOp>(loc, lbVal, stepVal);
321 Value capped = b.createOrFold<arith::MinSIOp>(loc, sum, ubVal);
322 splitBound = b.createOrFold<arith::MaxSIOp>(loc, lbVal, capped);
323 }
324 // Peel the first iteration.
325 firstIteration = cast<ForOp>(b.clone(*forOp.getOperation()));
326 b.modifyOpInPlace(firstIteration, [&]() {
327 firstIteration.getUpperBoundMutable().assign(splitBound);
328 });
329 // Update main loop with new lower bound.
330 b.modifyOpInPlace(forOp, [&]() {
331 forOp.getInitArgsMutable().assign(firstIteration->getResults());
332 forOp.getLowerBoundMutable().assign(splitBound);
333 });
334
335 return success();
336}
337
339 ForOp partialIteration) {
340 RewriterBase::InsertionGuard guard(rewriter);
341 rewriter.setInsertionPoint(partialIteration);
342 Location loc = partialIteration.getLoc();
343
344 // The partial iteration runs at most once: once if lb < ub, else not at all.
345 auto pred = partialIteration.getUnsignedCmp() ? arith::CmpIPredicate::ult
346 : arith::CmpIPredicate::slt;
347 Value cond = arith::CmpIOp::create(rewriter, loc, pred,
348 partialIteration.getLowerBound(),
349 partialIteration.getUpperBound());
350 bool hasResults = partialIteration.getNumResults() > 0;
351 auto ifOp = IfOp::create(rewriter, loc, partialIteration.getResultTypes(),
352 cond, /*addThenBlock=*/true,
353 /*addElseBlock=*/hasResults);
354
355 // Move the body into the "then" block. Its scf.yield becomes the terminator.
356 SmallVector<Value> replacements = {partialIteration.getLowerBound()};
357 llvm::append_range(replacements, partialIteration.getInitArgs());
358 rewriter.inlineBlockBefore(partialIteration.getBody(), ifOp.thenBlock(),
359 ifOp.thenBlock()->end(), replacements);
360
361 // If the loop does not run, it returns its init args.
362 if (hasResults) {
363 rewriter.setInsertionPointToStart(ifOp.elseBlock());
364 YieldOp::create(rewriter, loc, partialIteration.getInitArgs());
365 }
366
367 rewriter.replaceOp(partialIteration, ifOp.getResults());
368 return ifOp;
369}
370
371static constexpr char kPeeledLoopLabel[] = "__peeled_loop__";
372static constexpr char kPartialIterationLabel[] = "__partial_iteration__";
373
374namespace {
375struct ForLoopPeelingPattern : public OpRewritePattern<ForOp> {
376 ForLoopPeelingPattern(MLIRContext *ctx, bool peelFront, bool skipPartial,
377 bool partialAsIf)
378 : OpRewritePattern<ForOp>(ctx), peelFront(peelFront),
379 skipPartial(skipPartial), partialAsIf(partialAsIf) {}
380
381 LogicalResult matchAndRewrite(ForOp forOp,
382 PatternRewriter &rewriter) const override {
383
384 // Do not peel already peeled loops.
385 if (forOp->hasDiscardableAttr(kPeeledLoopLabel))
386 return failure();
387
388 scf::ForOp partialIteration;
389 // The case for peeling the first iteration of the loop.
390 if (peelFront) {
391 if (failed(
392 peelForLoopFirstIteration(rewriter, forOp, partialIteration))) {
393 return failure();
394 }
395 } else {
396 if (skipPartial) {
397 // No peeling of loops inside the partial iteration of another peeled
398 // loop. The partial iteration is an scf.for, or an scf.if if
399 // `partialAsIf` is set.
400 Operation *op = forOp.getOperation();
401 while ((op = op->getParentOp())) {
403 return failure();
404 }
405 }
406 // Apply loop peeling.
407 if (failed(
408 peelForLoopAndSimplifyBounds(rewriter, forOp, partialIteration)))
409 return failure();
410 }
411
412 Operation *partialOp = partialIteration;
413 if (partialAsIf)
414 partialOp = convertPartialIterationToIf(rewriter, partialIteration);
415
416 // Apply label, so that the same loop is not rewritten a second time.
417 rewriter.modifyOpInPlace(partialOp, [&]() {
418 partialOp->setDiscardableAttr(kPeeledLoopLabel, rewriter.getUnitAttr());
420 rewriter.getUnitAttr());
421 });
422 rewriter.modifyOpInPlace(forOp, [&]() {
423 forOp->setDiscardableAttr(kPeeledLoopLabel, rewriter.getUnitAttr());
424 });
425 return success();
426 }
427
428 // If set to true, the first iteration of the loop will be peeled. Otherwise,
429 // the unevenly divisible loop will be peeled at the end.
430 bool peelFront;
431
432 /// If set to true, loops inside partial iterations of another peeled loop
433 /// are not peeled. This reduces the size of the generated code. Partial
434 /// iterations are not usually performance critical.
435 /// Note: Takes into account the entire chain of parent operations, not just
436 /// the direct parent.
437 bool skipPartial;
438
439 /// If set to true, the partial iteration is emitted as an scf.if instead of
440 /// an scf.for.
441 bool partialAsIf;
442};
443} // namespace
444
445namespace {
446struct ParallelLoopSpecialization
447 : public impl::SCFParallelLoopSpecializationBase<
448 ParallelLoopSpecialization> {
449 void runOnOperation() override {
450 getOperation()->walk(
451 [](ParallelOp op) { specializeParallelLoopForUnrolling(op); });
452 }
453};
454
455struct ForLoopSpecialization
456 : public impl::SCFForLoopSpecializationBase<ForLoopSpecialization> {
457 void runOnOperation() override {
458 getOperation()->walk([](ForOp op) { specializeForLoopForUnrolling(op); });
459 }
460};
461
462struct ForLoopPeeling : public impl::SCFForLoopPeelingBase<ForLoopPeeling> {
463 using impl::SCFForLoopPeelingBase<ForLoopPeeling>::SCFForLoopPeelingBase;
464
465 void runOnOperation() override {
466 auto *parentOp = getOperation();
467 MLIRContext *ctx = parentOp->getContext();
468 RewritePatternSet patterns(ctx);
469 patterns.add<ForLoopPeelingPattern>(ctx, peelFront, skipPartial,
470 partialAsIf);
471 (void)applyPatternsGreedily(parentOp, std::move(patterns));
472
473 // Drop the markers.
474 parentOp->walk([](Operation *op) {
477 });
478 }
479};
480} // namespace
481
483 return std::make_unique<ParallelLoopSpecialization>();
484}
485
487 return std::make_unique<ForLoopSpecialization>();
488}
return success()
b
Return true if permutation is a valid permutation of the outer_dims_perm (case OuterOrInnerPerm::Oute...
static void specializeForLoopForUnrolling(ForOp op)
Rewrite a for loop with bounds defined by an affine.min with a constant into 2 loops after checking i...
static void specializeParallelLoopForUnrolling(ParallelOp op)
Rewrite a parallel loop with bounds defined by an affine.min with a constant into 2 loops after check...
static constexpr char kPeeledLoopLabel[]
static void rewriteAffineOpAfterPeeling(RewriterBase &rewriter, ForOp forOp, ForOp partialIteration, Value previousUb)
static LogicalResult peelForLoop(RewriterBase &b, ForOp forOp, ForOp &partialIteration, Value &splitBound)
Rewrite a for loop with bounds/step that potentially do not divide evenly into a for loop where the s...
static constexpr char kPartialIterationLabel[]
#define rem(a, b)
Base type for affine expression.
Definition AffineExpr.h:68
A multi-dimensional affine map Affine map's are immutable like Type's, and they are uniqued.
Definition AffineMap.h:46
static AffineMap get(MLIRContext *context)
Returns a zero result affine map with no dimensions or symbols: () -> ().
ArrayRef< AffineExpr > getResults() const
AffineExpr getResult(unsigned idx) const
UnitAttr getUnitAttr()
Definition Builders.cpp:106
This is a utility class for mapping one set of IR entities to another.
Definition IRMapping.h:26
void map(Value from, Value to)
Inserts a new mapping for 'from' to 'to'.
Definition IRMapping.h:30
This class defines the main interface for locations in MLIR and acts as a non-nullable wrapper around...
Definition Location.h:76
MLIRContext is the top-level object for a collection of MLIR operations.
Definition MLIRContext.h:71
RAII guard to reset the insertion point of the builder when destroyed.
Definition Builders.h:351
This class helps build Operations.
Definition Builders.h:210
static OpBuilder atBlockEnd(Block *block, Listener *listener=nullptr)
Create a builder and set the insertion point to after the last operation in the block but still insid...
Definition Builders.h:249
Operation * clone(Operation &op, IRMapping &mapper)
Creates a deep copy of the specified operation, remapping any operands that use values outside of the...
Definition Builders.cpp:581
void setInsertionPointToStart(Block *block)
Sets the insertion point to the start of the specified block.
Definition Builders.h:434
void setInsertionPoint(Block *block, Block::iterator insertPoint)
Set the insertion point to the specified location.
Definition Builders.h:401
Operation is the basic unit of execution within MLIR.
Definition Operation.h:87
void setDiscardableAttr(StringAttr name, Attribute value)
Set a discardable attribute by name.
Definition Operation.h:514
Operation * getParentOp()
Returns the closest surrounding operation that contains this operation or nullptr if this is a top-le...
Definition Operation.h:251
bool hasDiscardableAttr(StringRef name)
Return true if this operation has a discardable attribute with the provided name.
Definition Operation.h:505
Attribute removeDiscardableAttr(StringAttr name)
Remove the discardable attribute with the specified name if it exists.
Definition Operation.h:526
result_range getResults()
Definition Operation.h:442
void erase()
Remove this operation from its parent block and delete it.
A special type of RewriterBase that coordinates the application of a rewrite pattern on the current I...
This class coordinates the application of a rewrite on a set of IR, providing a way for clients to tr...
virtual void replaceOp(Operation *op, ValueRange newValues)
Replace the results of the given (original) operation with the specified list of values (replacements...
virtual void inlineBlockBefore(Block *source, Block *dest, Block::iterator before, ValueRange argValues={})
Inline the operations of block 'source' into block 'dest' before the given position.
void modifyOpInPlace(Operation *root, CallableT &&callable)
This method is a utility wrapper around an in-place modification of an operation.
This class represents an instance of an SSA value in the MLIR system, representing a computable value...
Definition Value.h:96
Location getLoc() const
Return the location of this value.
Definition Value.cpp:24
static WalkResult advance()
Definition WalkResult.h:47
static ConstantIndexOp create(OpBuilder &builder, Location location, int64_t value)
Definition ArithOps.cpp:434
void canonicalizeMapAndOperands(AffineMap *map, SmallVectorImpl< Value > *operands)
Modifies both map and operands in-place so as to:
void fullyComposeAffineMapAndOperands(AffineMap *map, SmallVectorImpl< Value > *operands, bool composeAffineMin=false)
Given an affine map map and its input operands, this method composes into map, maps of AffineApplyOps...
detail::InFlightRemark failed(Location loc, RemarkOpts opts)
Report an optimization remark that failed.
Definition Remarks.h:744
IfOp convertPartialIterationToIf(RewriterBase &rewriter, ForOp partialIteration)
Rewrite partialIteration, the partial iteration created by peelForLoopAndSimplifyBounds or peelForLoo...
LogicalResult peelForLoopAndSimplifyBounds(RewriterBase &rewriter, ForOp forOp, scf::ForOp &partialIteration)
Rewrite a for loop with bounds/step that potentially do not divide evenly into a for loop where the s...
LogicalResult peelForLoopFirstIteration(RewriterBase &rewriter, ForOp forOp, scf::ForOp &partialIteration)
Peel the first iteration out of the scf.for loop.
LogicalResult rewritePeeledMinMaxOp(RewriterBase &rewriter, Operation *op, Value iv, Value ub, Value step, bool insideLoop)
Try to simplify the given affine.min/max operation op after loop peeling.
Include the generated interface declarations.
bool matchPattern(Value value, const Pattern &pattern)
Entry point for matching a pattern over a Value.
Definition Matchers.h:490
std::unique_ptr< Pass > createParallelLoopSpecializationPass()
Creates a pass that specializes parallel loop for unrolling and vectorization.
std::unique_ptr< Pass > createForLoopSpecializationPass()
Creates a pass that specializes for loop for unrolling and vectorization.
std::optional< int64_t > getConstantIntValue(OpFoldResult ofr)
If ofr is a constant integer or an IntegerAttr, return the integer.
LogicalResult applyPatternsGreedily(Region &region, const FrozenRewritePatternSet &patterns, GreedyRewriteConfig config=GreedyRewriteConfig(), bool *changed=nullptr)
Rewrite ops in the given region, which must be isolated from above, by repeatedly applying the highes...
void bindSymbols(MLIRContext *ctx, AffineExprTy &...exprs)
Bind a list of AffineExpr references to SymbolExpr at positions: [0 .
Definition AffineExpr.h:327
detail::constant_op_matcher m_Constant()
Matches a constant foldable operation.
Definition Matchers.h:369
OpRewritePattern is a wrapper around RewritePattern that allows for matching and rewriting against an...