MLIR  22.0.0git
SCFTransformOps.cpp
Go to the documentation of this file.
1 //===- SCFTransformOps.cpp - Implementation of SCF transformation ops -----===//
2 //
3 // Part of the LLVM Project, under the Apache License v2.0 with LLVM Exceptions.
4 // See https://llvm.org/LICENSE.txt for license information.
5 // SPDX-License-Identifier: Apache-2.0 WITH LLVM-exception
6 //
7 //===----------------------------------------------------------------------===//
8 
10 
24 #include "mlir/IR/Dominance.h"
25 #include "mlir/IR/OpDefinition.h"
26 
27 using namespace mlir;
28 using namespace mlir::affine;
29 
30 //===----------------------------------------------------------------------===//
31 // Apply...PatternsOp
32 //===----------------------------------------------------------------------===//
33 
34 void transform::ApplyForLoopCanonicalizationPatternsOp::populatePatterns(
37 }
38 
39 void transform::ApplySCFStructuralConversionPatternsOp::populatePatterns(
40  TypeConverter &typeConverter, RewritePatternSet &patterns) {
42 }
43 
44 void transform::ApplySCFStructuralConversionPatternsOp::
45  populateConversionTargetRules(const TypeConverter &typeConverter,
46  ConversionTarget &conversionTarget) {
48  conversionTarget);
49 }
50 
51 void transform::ApplySCFToControlFlowPatternsOp::populatePatterns(
52  TypeConverter &typeConverter, RewritePatternSet &patterns) {
54 }
55 
56 //===----------------------------------------------------------------------===//
57 // ForallToForOp
58 //===----------------------------------------------------------------------===//
59 
61 transform::ForallToForOp::apply(transform::TransformRewriter &rewriter,
64  auto payload = state.getPayloadOps(getTarget());
65  if (!llvm::hasSingleElement(payload))
66  return emitSilenceableError() << "expected a single payload op";
67 
68  auto target = dyn_cast<scf::ForallOp>(*payload.begin());
69  if (!target) {
71  emitSilenceableError() << "expected the payload to be scf.forall";
72  diag.attachNote((*payload.begin())->getLoc()) << "payload op";
73  return diag;
74  }
75 
76  if (!target.getOutputs().empty()) {
77  return emitSilenceableError()
78  << "unsupported shared outputs (didn't bufferize?)";
79  }
80 
81  SmallVector<OpFoldResult> lbs = target.getMixedLowerBound();
82 
83  if (getNumResults() != lbs.size()) {
85  emitSilenceableError()
86  << "op expects as many results (" << getNumResults()
87  << ") as payload has induction variables (" << lbs.size() << ")";
88  diag.attachNote(target.getLoc()) << "payload op";
89  return diag;
90  }
91 
92  SmallVector<Operation *> opResults;
93  if (failed(scf::forallToForLoop(rewriter, target, &opResults))) {
94  DiagnosedSilenceableFailure diag = emitSilenceableError()
95  << "failed to convert forall into for";
96  return diag;
97  }
98 
99  for (auto &&[i, res] : llvm::enumerate(opResults)) {
100  results.set(cast<OpResult>(getTransformed()[i]), {res});
101  }
103 }
104 
105 //===----------------------------------------------------------------------===//
106 // ForallToForOp
107 //===----------------------------------------------------------------------===//
108 
110 transform::ForallToParallelOp::apply(transform::TransformRewriter &rewriter,
112  transform::TransformState &state) {
113  auto payload = state.getPayloadOps(getTarget());
114  if (!llvm::hasSingleElement(payload))
115  return emitSilenceableError() << "expected a single payload op";
116 
117  auto target = dyn_cast<scf::ForallOp>(*payload.begin());
118  if (!target) {
120  emitSilenceableError() << "expected the payload to be scf.forall";
121  diag.attachNote((*payload.begin())->getLoc()) << "payload op";
122  return diag;
123  }
124 
125  if (!target.getOutputs().empty()) {
126  return emitSilenceableError()
127  << "unsupported shared outputs (didn't bufferize?)";
128  }
129 
130  if (getNumResults() != 1) {
131  DiagnosedSilenceableFailure diag = emitSilenceableError()
132  << "op expects one result, given "
133  << getNumResults();
134  diag.attachNote(target.getLoc()) << "payload op";
135  return diag;
136  }
137 
138  scf::ParallelOp opResult;
139  if (failed(scf::forallToParallelLoop(rewriter, target, &opResult))) {
141  emitSilenceableError() << "failed to convert forall into parallel";
142  return diag;
143  }
144 
145  results.set(cast<OpResult>(getTransformed()[0]), {opResult});
147 }
148 
149 //===----------------------------------------------------------------------===//
150 // LoopOutlineOp
151 //===----------------------------------------------------------------------===//
152 
153 /// Wraps the given operation `op` into an `scf.execute_region` operation. Uses
154 /// the provided rewriter for all operations to remain compatible with the
155 /// rewriting infra, as opposed to just splicing the op in place.
156 static scf::ExecuteRegionOp wrapInExecuteRegion(RewriterBase &b,
157  Operation *op) {
158  if (op->getNumRegions() != 1)
159  return nullptr;
161  b.setInsertionPoint(op);
162  scf::ExecuteRegionOp executeRegionOp =
163  scf::ExecuteRegionOp::create(b, op->getLoc(), op->getResultTypes());
164  {
166  b.setInsertionPointToStart(&executeRegionOp.getRegion().emplaceBlock());
167  Operation *clonedOp = b.cloneWithoutRegions(*op);
168  Region &clonedRegion = clonedOp->getRegions().front();
169  assert(clonedRegion.empty() && "expected empty region");
170  b.inlineRegionBefore(op->getRegions().front(), clonedRegion,
171  clonedRegion.end());
172  scf::YieldOp::create(b, op->getLoc(), clonedOp->getResults());
173  }
174  b.replaceOp(op, executeRegionOp.getResults());
175  return executeRegionOp;
176 }
177 
179 transform::LoopOutlineOp::apply(transform::TransformRewriter &rewriter,
181  transform::TransformState &state) {
182  SmallVector<Operation *> functions;
185  for (Operation *target : state.getPayloadOps(getTarget())) {
186  Location location = target->getLoc();
187  Operation *symbolTableOp = SymbolTable::getNearestSymbolTable(target);
188  scf::ExecuteRegionOp exec = wrapInExecuteRegion(rewriter, target);
189  if (!exec) {
190  DiagnosedSilenceableFailure diag = emitSilenceableError()
191  << "failed to outline";
192  diag.attachNote(target->getLoc()) << "target op";
193  return diag;
194  }
195  func::CallOp call;
196  FailureOr<func::FuncOp> outlined = outlineSingleBlockRegion(
197  rewriter, location, exec.getRegion(), getFuncName(), &call);
198 
199  if (failed(outlined))
200  return emitDefaultDefiniteFailure(target);
201 
202  if (symbolTableOp) {
203  SymbolTable &symbolTable =
204  symbolTables.try_emplace(symbolTableOp, symbolTableOp)
205  .first->getSecond();
206  symbolTable.insert(*outlined);
207  call.setCalleeAttr(FlatSymbolRefAttr::get(*outlined));
208  }
209  functions.push_back(*outlined);
210  calls.push_back(call);
211  }
212  results.set(cast<OpResult>(getFunction()), functions);
213  results.set(cast<OpResult>(getCall()), calls);
215 }
216 
217 //===----------------------------------------------------------------------===//
218 // LoopPeelOp
219 //===----------------------------------------------------------------------===//
220 
222 transform::LoopPeelOp::applyToOne(transform::TransformRewriter &rewriter,
223  scf::ForOp target,
225  transform::TransformState &state) {
226  scf::ForOp result;
227  if (getPeelFront()) {
228  LogicalResult status =
229  scf::peelForLoopFirstIteration(rewriter, target, result);
230  if (failed(status)) {
232  emitSilenceableError() << "failed to peel the first iteration";
233  return diag;
234  }
235  } else {
236  LogicalResult status =
237  scf::peelForLoopAndSimplifyBounds(rewriter, target, result);
238  if (failed(status)) {
239  DiagnosedSilenceableFailure diag = emitSilenceableError()
240  << "failed to peel the last iteration";
241  return diag;
242  }
243  }
244 
245  results.push_back(target);
246  results.push_back(result);
247 
249 }
250 
251 //===----------------------------------------------------------------------===//
252 // LoopPipelineOp
253 //===----------------------------------------------------------------------===//
254 
255 /// Callback for PipeliningOption. Populates `schedule` with the mapping from an
256 /// operation to its logical time position given the iteration interval and the
257 /// read latency. The latter is only relevant for vector transfers.
258 static void
259 loopScheduling(scf::ForOp forOp,
260  std::vector<std::pair<Operation *, unsigned>> &schedule,
261  unsigned iterationInterval, unsigned readLatency) {
262  auto getLatency = [&](Operation *op) -> unsigned {
263  if (isa<vector::TransferReadOp>(op))
264  return readLatency;
265  return 1;
266  };
267 
268  std::optional<int64_t> ubConstant =
269  getConstantIntValue(forOp.getUpperBound());
270  std::optional<int64_t> lbConstant =
271  getConstantIntValue(forOp.getLowerBound());
273  std::map<unsigned, std::vector<Operation *>> wrappedSchedule;
274  for (Operation &op : forOp.getBody()->getOperations()) {
275  if (isa<scf::YieldOp>(op))
276  continue;
277  unsigned earlyCycle = 0;
278  for (Value operand : op.getOperands()) {
279  Operation *def = operand.getDefiningOp();
280  if (!def)
281  continue;
282  if (ubConstant && lbConstant) {
283  unsigned ubInt = ubConstant.value();
284  unsigned lbInt = lbConstant.value();
285  auto minLatency = std::min(ubInt - lbInt - 1, getLatency(def));
286  earlyCycle = std::max(earlyCycle, opCycles[def] + minLatency);
287  } else {
288  earlyCycle = std::max(earlyCycle, opCycles[def] + getLatency(def));
289  }
290  }
291  opCycles[&op] = earlyCycle;
292  wrappedSchedule[earlyCycle % iterationInterval].push_back(&op);
293  }
294  for (const auto &it : wrappedSchedule) {
295  for (Operation *op : it.second) {
296  unsigned cycle = opCycles[op];
297  schedule.emplace_back(op, cycle / iterationInterval);
298  }
299  }
300 }
301 
303 transform::LoopPipelineOp::applyToOne(transform::TransformRewriter &rewriter,
304  scf::ForOp target,
306  transform::TransformState &state) {
308  options.getScheduleFn =
309  [this](scf::ForOp forOp,
310  std::vector<std::pair<Operation *, unsigned>> &schedule) mutable {
311  loopScheduling(forOp, schedule, getIterationInterval(),
312  getReadLatency());
313  };
314  scf::ForLoopPipeliningPattern pattern(options, target->getContext());
315  rewriter.setInsertionPoint(target);
316  FailureOr<scf::ForOp> patternResult =
317  scf::pipelineForLoop(rewriter, target, options);
318  if (succeeded(patternResult)) {
319  results.push_back(*patternResult);
321  }
322  return emitDefaultSilenceableFailure(target);
323 }
324 
325 //===----------------------------------------------------------------------===//
326 // LoopPromoteIfOneIterationOp
327 //===----------------------------------------------------------------------===//
328 
329 DiagnosedSilenceableFailure transform::LoopPromoteIfOneIterationOp::applyToOne(
330  transform::TransformRewriter &rewriter, LoopLikeOpInterface target,
332  transform::TransformState &state) {
333  (void)target.promoteIfSingleIteration(rewriter);
335 }
336 
337 void transform::LoopPromoteIfOneIterationOp::getEffects(
339  consumesHandle(getTargetMutable(), effects);
340  modifiesPayload(effects);
341 }
342 
343 //===----------------------------------------------------------------------===//
344 // LoopUnrollOp
345 //===----------------------------------------------------------------------===//
346 
348 transform::LoopUnrollOp::applyToOne(transform::TransformRewriter &rewriter,
349  Operation *op,
351  transform::TransformState &state) {
352  LogicalResult result(failure());
353  if (scf::ForOp scfFor = dyn_cast<scf::ForOp>(op))
354  result = loopUnrollByFactor(scfFor, getFactor());
355  else if (AffineForOp affineFor = dyn_cast<AffineForOp>(op))
356  result = loopUnrollByFactor(affineFor, getFactor());
357  else
358  return emitSilenceableError()
359  << "failed to unroll, incorrect type of payload";
360 
361  if (failed(result))
362  return emitSilenceableError() << "failed to unroll";
363 
365 }
366 
367 //===----------------------------------------------------------------------===//
368 // LoopUnrollAndJamOp
369 //===----------------------------------------------------------------------===//
370 
371 DiagnosedSilenceableFailure transform::LoopUnrollAndJamOp::applyToOne(
374  transform::TransformState &state) {
375  LogicalResult result(failure());
376  if (scf::ForOp scfFor = dyn_cast<scf::ForOp>(op))
377  result = loopUnrollJamByFactor(scfFor, getFactor());
378  else if (AffineForOp affineFor = dyn_cast<AffineForOp>(op))
379  result = loopUnrollJamByFactor(affineFor, getFactor());
380  else
381  return emitSilenceableError()
382  << "failed to unroll and jam, incorrect type of payload";
383 
384  if (failed(result))
385  return emitSilenceableError() << "failed to unroll and jam";
386 
388 }
389 
390 //===----------------------------------------------------------------------===//
391 // LoopCoalesceOp
392 //===----------------------------------------------------------------------===//
393 
395 transform::LoopCoalesceOp::applyToOne(transform::TransformRewriter &rewriter,
396  Operation *op,
398  transform::TransformState &state) {
399  LogicalResult result(failure());
400  if (scf::ForOp scfForOp = dyn_cast<scf::ForOp>(op))
401  result = coalescePerfectlyNestedSCFForLoops(scfForOp);
402  else if (AffineForOp affineForOp = dyn_cast<AffineForOp>(op))
403  result = coalescePerfectlyNestedAffineLoops(affineForOp);
404 
405  results.push_back(op);
406  if (failed(result)) {
407  DiagnosedSilenceableFailure diag = emitSilenceableError()
408  << "failed to coalesce";
409  return diag;
410  }
412 }
413 
414 //===----------------------------------------------------------------------===//
415 // TakeAssumedBranchOp
416 //===----------------------------------------------------------------------===//
417 /// Replaces the given op with the contents of the given single-block region,
418 /// using the operands of the block terminator to replace operation results.
419 static void replaceOpWithRegion(RewriterBase &rewriter, Operation *op,
420  Region &region) {
421  assert(region.hasOneBlock() && "expected single-block region");
422  Block *block = &region.front();
423  Operation *terminator = block->getTerminator();
424  ValueRange results = terminator->getOperands();
425  rewriter.inlineBlockBefore(block, op, /*blockArgs=*/{});
426  rewriter.replaceOp(op, results);
427  rewriter.eraseOp(terminator);
428 }
429 
430 DiagnosedSilenceableFailure transform::TakeAssumedBranchOp::applyToOne(
431  transform::TransformRewriter &rewriter, scf::IfOp ifOp,
433  transform::TransformState &state) {
434  rewriter.setInsertionPoint(ifOp);
435  Region &region =
436  getTakeElseBranch() ? ifOp.getElseRegion() : ifOp.getThenRegion();
437  if (!region.hasOneBlock()) {
438  return emitDefiniteFailure()
439  << "requires an scf.if op with a single-block "
440  << ((getTakeElseBranch()) ? "`else`" : "`then`") << " region";
441  }
442  replaceOpWithRegion(rewriter, ifOp, region);
444 }
445 
446 void transform::TakeAssumedBranchOp::getEffects(
448  onlyReadsHandle(getTargetMutable(), effects);
449  modifiesPayload(effects);
450 }
451 
452 //===----------------------------------------------------------------------===//
453 // LoopFuseSiblingOp
454 //===----------------------------------------------------------------------===//
455 
456 /// Check if `target` and `source` are siblings, in the context that `target`
457 /// is being fused into `source`.
458 ///
459 /// This is a simple check that just checks if both operations are in the same
460 /// block and some checks to ensure that the fused IR does not violate
461 /// dominance.
463  Operation *source) {
464  // Check if both operations are same.
465  if (target == source)
466  return emitSilenceableFailure(source)
467  << "target and source need to be different loops";
468 
469  // Check if both operations are in the same block.
470  if (target->getBlock() != source->getBlock())
471  return emitSilenceableFailure(source)
472  << "target and source are not in the same block";
473 
474  // Check if fusion will violate dominance.
475  DominanceInfo domInfo(source);
476  if (target->isBeforeInBlock(source)) {
477  // Since `target` is before `source`, all users of results of `target`
478  // need to be dominated by `source`.
479  for (Operation *user : target->getUsers()) {
480  if (!domInfo.properlyDominates(source, user, /*enclosingOpOk=*/false)) {
481  return emitSilenceableFailure(target)
482  << "user of results of target should be properly dominated by "
483  "source";
484  }
485  }
486  } else {
487  // Since `target` is after `source`, all values used by `target` need
488  // to dominate `source`.
489 
490  // Check if operands of `target` are dominated by `source`.
491  for (Value operand : target->getOperands()) {
492  Operation *operandOp = operand.getDefiningOp();
493  // Operands without defining operations are block arguments. When `target`
494  // and `source` occur in the same block, these operands dominate `source`.
495  if (!operandOp)
496  continue;
497 
498  // Operand's defining operation should properly dominate `source`.
499  if (!domInfo.properlyDominates(operandOp, source,
500  /*enclosingOpOk=*/false))
501  return emitSilenceableFailure(target)
502  << "operands of target should be properly dominated by source";
503  }
504 
505  // Check if values used by `target` are dominated by `source`.
506  bool failed = false;
507  OpOperand *failedValue = nullptr;
508  visitUsedValuesDefinedAbove(target->getRegions(), [&](OpOperand *operand) {
509  Operation *operandOp = operand->get().getDefiningOp();
510  if (operandOp && !domInfo.properlyDominates(operandOp, source,
511  /*enclosingOpOk=*/false)) {
512  // `operand` is not an argument of an enclosing block and the defining
513  // op of `operand` is outside `target` but does not dominate `source`.
514  failed = true;
515  failedValue = operand;
516  }
517  });
518 
519  if (failed)
520  return emitSilenceableFailure(failedValue->getOwner())
521  << "values used inside regions of target should be properly "
522  "dominated by source";
523  }
524 
526 }
527 
528 /// Check if `target` scf.forall can be fused into `source` scf.forall.
529 ///
530 /// This simply checks if both loops have the same bounds, steps and mapping.
531 /// No attempt is made at checking that the side effects of `target` and
532 /// `source` are independent of each other.
534  Operation *source) {
535  auto targetOp = dyn_cast<scf::ForallOp>(target);
536  auto sourceOp = dyn_cast<scf::ForallOp>(source);
537  if (!targetOp || !sourceOp)
538  return false;
539 
540  return targetOp.getMixedLowerBound() == sourceOp.getMixedLowerBound() &&
541  targetOp.getMixedUpperBound() == sourceOp.getMixedUpperBound() &&
542  targetOp.getMixedStep() == sourceOp.getMixedStep() &&
543  targetOp.getMapping() == sourceOp.getMapping();
544 }
545 
546 /// Check if `target` scf.for can be fused into `source` scf.for.
547 ///
548 /// This simply checks if both loops have the same bounds and steps. No attempt
549 /// is made at checking that the side effects of `target` and `source` are
550 /// independent of each other.
552  Operation *source) {
553  auto targetOp = dyn_cast<scf::ForOp>(target);
554  auto sourceOp = dyn_cast<scf::ForOp>(source);
555  if (!targetOp || !sourceOp)
556  return false;
557 
558  return targetOp.getLowerBound() == sourceOp.getLowerBound() &&
559  targetOp.getUpperBound() == sourceOp.getUpperBound() &&
560  targetOp.getStep() == sourceOp.getStep();
561 }
562 
564 transform::LoopFuseSiblingOp::apply(transform::TransformRewriter &rewriter,
566  transform::TransformState &state) {
567  auto targetOps = state.getPayloadOps(getTarget());
568  auto sourceOps = state.getPayloadOps(getSource());
569 
570  if (!llvm::hasSingleElement(targetOps) ||
571  !llvm::hasSingleElement(sourceOps)) {
572  return emitDefiniteFailure()
573  << "requires exactly one target handle (got "
574  << llvm::range_size(targetOps) << ") and exactly one "
575  << "source handle (got " << llvm::range_size(sourceOps) << ")";
576  }
577 
578  Operation *target = *targetOps.begin();
579  Operation *source = *sourceOps.begin();
580 
581  // Check if the target and source are siblings.
582  DiagnosedSilenceableFailure diag = isOpSibling(target, source);
583  if (!diag.succeeded())
584  return diag;
585 
586  Operation *fusedLoop;
587  /// TODO: Support fusion for loop-like ops besides scf.for and scf.forall.
588  if (isForWithIdenticalConfiguration(target, source)) {
589  fusedLoop = fuseIndependentSiblingForLoops(
590  cast<scf::ForOp>(target), cast<scf::ForOp>(source), rewriter);
591  } else if (isForallWithIdenticalConfiguration(target, source)) {
593  cast<scf::ForallOp>(target), cast<scf::ForallOp>(source), rewriter);
594  } else {
595  return emitSilenceableFailure(target->getLoc())
596  << "operations cannot be fused";
597  }
598 
599  assert(fusedLoop && "failed to fuse operations");
600 
601  results.set(cast<OpResult>(getFusedLoop()), {fusedLoop});
603 }
604 
605 //===----------------------------------------------------------------------===//
606 // Transform op registration
607 //===----------------------------------------------------------------------===//
608 
609 namespace {
610 class SCFTransformDialectExtension
612  SCFTransformDialectExtension> {
613 public:
614  MLIR_DEFINE_EXPLICIT_INTERNAL_INLINE_TYPE_ID(SCFTransformDialectExtension)
615 
616  using Base::Base;
617 
618  void init() {
619  declareGeneratedDialect<affine::AffineDialect>();
620  declareGeneratedDialect<func::FuncDialect>();
621 
622  registerTransformOps<
623 #define GET_OP_LIST
624 #include "mlir/Dialect/SCF/TransformOps/SCFTransformOps.cpp.inc"
625  >();
626  }
627 };
628 } // namespace
629 
630 #define GET_OP_CLASSES
631 #include "mlir/Dialect/SCF/TransformOps/SCFTransformOps.cpp.inc"
632 
634  registry.addExtensions<SCFTransformDialectExtension>();
635 }
static std::string diag(const llvm::Value &value)
static llvm::ManagedStatic< PassManagerOptions > options
static Value max(ImplicitLocOpBuilder &builder, Value value, Value bound)
static Value min(ImplicitLocOpBuilder &builder, Value value, Value bound)
static bool isForWithIdenticalConfiguration(Operation *target, Operation *source)
Check if target scf.for can be fused into source scf.for.
static DiagnosedSilenceableFailure isOpSibling(Operation *target, Operation *source)
Check if target and source are siblings, in the context that target is being fused into source.
static void loopScheduling(scf::ForOp forOp, std::vector< std::pair< Operation *, unsigned >> &schedule, unsigned iterationInterval, unsigned readLatency)
Callback for PipeliningOption.
static bool isForallWithIdenticalConfiguration(Operation *target, Operation *source)
Check if target scf.forall can be fused into source scf.forall.
static void replaceOpWithRegion(RewriterBase &rewriter, Operation *op, Region &region)
Replaces the given op with the contents of the given single-block region, using the operands of the b...
static scf::ExecuteRegionOp wrapInExecuteRegion(RewriterBase &b, Operation *op)
Wraps the given operation op into an scf.execute_region operation.
#define MLIR_DEFINE_EXPLICIT_INTERNAL_INLINE_TYPE_ID(CLASS_NAME)
Definition: TypeID.h:331
Block represents an ordered list of Operations.
Definition: Block.h:33
Operation * getTerminator()
Get the terminator operation of this block.
Definition: Block.cpp:244
This class describes a specific conversion target.
The result of a transform IR operation application.
static DiagnosedSilenceableFailure success()
Constructs a DiagnosedSilenceableFailure in the success state.
The DialectRegistry maps a dialect namespace to a constructor for the matching dialect.
void addExtensions()
Add the given extensions to the registry.
A class for computing basic dominance information.
Definition: Dominance.h:140
bool properlyDominates(Operation *a, Operation *b, bool enclosingOpOk=true) const
Return true if operation A properly dominates operation B, i.e.
Definition: Dominance.cpp:323
static FlatSymbolRefAttr get(StringAttr value)
Construct a symbol reference for the given value name.
This class defines the main interface for locations in MLIR and acts as a non-nullable wrapper around...
Definition: Location.h:76
RAII guard to reset the insertion point of the builder when destroyed.
Definition: Builders.h:346
void setInsertionPointToStart(Block *block)
Sets the insertion point to the start of the specified block.
Definition: Builders.h:429
void setInsertionPoint(Block *block, Block::iterator insertPoint)
Set the insertion point to the specified location.
Definition: Builders.h:396
Operation * cloneWithoutRegions(Operation &op, IRMapping &mapper)
Creates a deep copy of this operation but keep the operation regions empty.
Definition: Builders.h:585
This class represents an operand of an operation.
Definition: Value.h:257
Operation is the basic unit of execution within MLIR.
Definition: Operation.h:88
bool isBeforeInBlock(Operation *other)
Given an operation 'other' that is within the same parent block, return whether the current operation...
Definition: Operation.cpp:385
unsigned getNumRegions()
Returns the number of regions held by this operation.
Definition: Operation.h:674
Location getLoc()
The source location the operation was defined or derived from.
Definition: Operation.h:223
Block * getBlock()
Returns the operation block that contains this operation.
Definition: Operation.h:213
MutableArrayRef< Region > getRegions()
Returns the regions held by this operation.
Definition: Operation.h:677
result_type_range getResultTypes()
Definition: Operation.h:428
operand_range getOperands()
Returns an iterator on the underlying Value's.
Definition: Operation.h:378
user_range getUsers()
Returns a range of all users.
Definition: Operation.h:873
result_range getResults()
Definition: Operation.h:415
This class contains a list of basic blocks and a link to the parent operation it is attached to.
Definition: Region.h:26
bool empty()
Definition: Region.h:60
iterator end()
Definition: Region.h:56
Block & front()
Definition: Region.h:65
bool hasOneBlock()
Return true if this region has exactly one block.
Definition: Region.h:68
This class coordinates the application of a rewrite on a set of IR, providing a way for clients to tr...
Definition: PatternMatch.h:358
virtual void replaceOp(Operation *op, ValueRange newValues)
Replace the results of the given (original) operation with the specified list of values (replacements...
virtual void eraseOp(Operation *op)
This method erases an operation that is known to have no uses.
virtual void inlineBlockBefore(Block *source, Block *dest, Block::iterator before, ValueRange argValues={})
Inline the operations of block 'source' into block 'dest' before the given position.
void inlineRegionBefore(Region &region, Region &parent, Region::iterator before)
Move the blocks that belong to "region" before the given position in another region "parent".
This class allows for representing and managing the symbol table used by operations with the 'SymbolT...
Definition: SymbolTable.h:24
StringAttr insert(Operation *symbol, Block::iterator insertPt={})
Insert a new symbol into the table, and rename it as necessary to avoid collisions.
static Operation * getNearestSymbolTable(Operation *from)
Returns the nearest symbol table from a given operation from.
Type conversion class.
This class provides an abstraction over the different types of ranges over Values.
Definition: ValueRange.h:387
This class represents an instance of an SSA value in the MLIR system, representing a computable value...
Definition: Value.h:96
A list of results of applying a transform op with ApplyEachOpTrait to a single payload operation,...
void push_back(Operation *op)
Appends an element to the list.
Base class for extensions of the Transform dialect that supports injecting operations into the Transf...
Local mapping between values defined by a specific op implementing the TransformOpInterface and the p...
void set(OpResult value, Range &&ops)
Indicates that the result of the transform IR op at the given position corresponds to the given list ...
This is a special rewriter to be used in transform op implementations, providing additional helper fu...
The state maintained across applications of various ops implementing the TransformOpInterface.
LogicalResult loopUnrollByFactor(AffineForOp forOp, uint64_t unrollFactor, function_ref< void(unsigned, Operation *, OpBuilder)> annotateFn=nullptr, bool cleanUpUnroll=false)
Unrolls this for operation by the specified unroll factor.
Definition: LoopUtils.cpp:995
LogicalResult loopUnrollJamByFactor(AffineForOp forOp, uint64_t unrollJamFactor)
Unrolls and jams this loop by the specified factor.
Definition: LoopUtils.cpp:1084
LogicalResult coalescePerfectlyNestedAffineLoops(AffineForOp op)
Walk an affine.for to find a band to coalesce.
Definition: LoopUtils.cpp:2770
constexpr void enumerate(std::tuple< Tys... > &tuple, CallbackT &&callback)
Definition: Matchers.h:344
void registerTransformDialectExtension(DialectRegistry &registry)
LogicalResult peelForLoopAndSimplifyBounds(RewriterBase &rewriter, ForOp forOp, scf::ForOp &partialIteration)
Rewrite a for loop with bounds/step that potentially do not divide evenly into a for loop where the s...
LogicalResult forallToForLoop(RewriterBase &rewriter, ForallOp forallOp, SmallVectorImpl< Operation * > *results=nullptr)
Try converting scf.forall into a set of nested scf.for loops.
LogicalResult peelForLoopFirstIteration(RewriterBase &rewriter, ForOp forOp, scf::ForOp &partialIteration)
Peel the first iteration out of the scf.for loop.
LogicalResult forallToParallelLoop(RewriterBase &rewriter, ForallOp forallOp, ParallelOp *result=nullptr)
Try converting scf.forall into an scf.parallel loop.
void populateSCFStructuralTypeConversions(const TypeConverter &typeConverter, RewritePatternSet &patterns)
Similar to populateSCFStructuralTypeConversionsAndLegality but does not populate the conversion targe...
void populateSCFForLoopCanonicalizationPatterns(RewritePatternSet &patterns)
Populate patterns for canonicalizing operations inside SCF loop bodies.
FailureOr< ForOp > pipelineForLoop(RewriterBase &rewriter, ForOp forOp, const PipeliningOption &options, bool *modifiedIR=nullptr)
Generate a pipelined version of the scf.for loop based on the schedule given as option.
void populateSCFStructuralTypeConversionTarget(const TypeConverter &typeConverter, ConversionTarget &target)
Updates the ConversionTarget with dynamic legality of SCF operations based on the provided type conve...
void consumesHandle(MutableArrayRef< OpOperand > handles, SmallVectorImpl< MemoryEffects::EffectInstance > &effects)
Populates effects with the memory effects indicating the operation on the given handle value:
void onlyReadsHandle(MutableArrayRef< OpOperand > handles, SmallVectorImpl< MemoryEffects::EffectInstance > &effects)
void modifiesPayload(SmallVectorImpl< MemoryEffects::EffectInstance > &effects)
Populates effects with the memory effects indicating the access to payload IR resource.
Include the generated interface declarations.
std::optional< int64_t > getConstantIntValue(OpFoldResult ofr)
If ofr is a constant integer or an IntegerAttr, return the integer.
LogicalResult coalescePerfectlyNestedSCFForLoops(scf::ForOp op)
Walk an affine.for to find a band to coalesce.
Definition: Utils.cpp:988
DiagnosedSilenceableFailure emitSilenceableFailure(Location loc, const Twine &message={})
Emits a silenceable failure with the given message.
DiagnosedDefiniteFailure emitDefiniteFailure(Location loc, const Twine &message={})
Emits a definite failure with the given message.
const FrozenRewritePatternSet & patterns
void populateSCFToControlFlowConversionPatterns(RewritePatternSet &patterns)
Collect a set of patterns to convert SCF operations to CFG branch-based operations within the Control...
FailureOr< func::FuncOp > outlineSingleBlockRegion(RewriterBase &rewriter, Location loc, Region &region, StringRef funcName, func::CallOp *callOp=nullptr)
Outline a region with a single block into a new FuncOp.
Definition: Utils.cpp:115
scf::ForallOp fuseIndependentSiblingForallLoops(scf::ForallOp target, scf::ForallOp source, RewriterBase &rewriter)
Given two scf.forall loops, target and source, fuses target into source.
Definition: Utils.cpp:1364
scf::ForOp fuseIndependentSiblingForLoops(scf::ForOp target, scf::ForOp source, RewriterBase &rewriter)
Given two scf.for loops, target and source, fuses target into source.
Definition: Utils.cpp:1417
void visitUsedValuesDefinedAbove(Region &region, Region &limit, function_ref< void(OpOperand *)> callback)
Calls callback for each use of a value within region or its descendants that was defined at the ances...
Definition: RegionUtils.cpp:40
Options to dictate how loops should be pipelined.
Definition: Transforms.h:123