MLIR  21.0.0git
SCFTransformOps.cpp
Go to the documentation of this file.
1 //===- SCFTransformOps.cpp - Implementation of SCF transformation ops -----===//
2 //
3 // Part of the LLVM Project, under the Apache License v2.0 with LLVM Exceptions.
4 // See https://llvm.org/LICENSE.txt for license information.
5 // SPDX-License-Identifier: Apache-2.0 WITH LLVM-exception
6 //
7 //===----------------------------------------------------------------------===//
8 
10 
27 #include "mlir/IR/Dominance.h"
28 #include "mlir/IR/OpDefinition.h"
29 
30 using namespace mlir;
31 using namespace mlir::affine;
32 
33 //===----------------------------------------------------------------------===//
34 // Apply...PatternsOp
35 //===----------------------------------------------------------------------===//
36 
37 void transform::ApplyForLoopCanonicalizationPatternsOp::populatePatterns(
40 }
41 
42 void transform::ApplySCFStructuralConversionPatternsOp::populatePatterns(
43  TypeConverter &typeConverter, RewritePatternSet &patterns) {
45 }
46 
47 void transform::ApplySCFStructuralConversionPatternsOp::
48  populateConversionTargetRules(const TypeConverter &typeConverter,
49  ConversionTarget &conversionTarget) {
51  conversionTarget);
52 }
53 
54 void transform::ApplySCFToControlFlowPatternsOp::populatePatterns(
55  TypeConverter &typeConverter, RewritePatternSet &patterns) {
57 }
58 
59 //===----------------------------------------------------------------------===//
60 // ForallToForOp
61 //===----------------------------------------------------------------------===//
62 
64 transform::ForallToForOp::apply(transform::TransformRewriter &rewriter,
67  auto payload = state.getPayloadOps(getTarget());
68  if (!llvm::hasSingleElement(payload))
69  return emitSilenceableError() << "expected a single payload op";
70 
71  auto target = dyn_cast<scf::ForallOp>(*payload.begin());
72  if (!target) {
74  emitSilenceableError() << "expected the payload to be scf.forall";
75  diag.attachNote((*payload.begin())->getLoc()) << "payload op";
76  return diag;
77  }
78 
79  if (!target.getOutputs().empty()) {
80  return emitSilenceableError()
81  << "unsupported shared outputs (didn't bufferize?)";
82  }
83 
84  SmallVector<OpFoldResult> lbs = target.getMixedLowerBound();
85 
86  if (getNumResults() != lbs.size()) {
88  emitSilenceableError()
89  << "op expects as many results (" << getNumResults()
90  << ") as payload has induction variables (" << lbs.size() << ")";
91  diag.attachNote(target.getLoc()) << "payload op";
92  return diag;
93  }
94 
95  SmallVector<Operation *> opResults;
96  if (failed(scf::forallToForLoop(rewriter, target, &opResults))) {
97  DiagnosedSilenceableFailure diag = emitSilenceableError()
98  << "failed to convert forall into for";
99  return diag;
100  }
101 
102  for (auto &&[i, res] : llvm::enumerate(opResults)) {
103  results.set(cast<OpResult>(getTransformed()[i]), {res});
104  }
106 }
107 
108 //===----------------------------------------------------------------------===//
109 // ForallToForOp
110 //===----------------------------------------------------------------------===//
111 
113 transform::ForallToParallelOp::apply(transform::TransformRewriter &rewriter,
115  transform::TransformState &state) {
116  auto payload = state.getPayloadOps(getTarget());
117  if (!llvm::hasSingleElement(payload))
118  return emitSilenceableError() << "expected a single payload op";
119 
120  auto target = dyn_cast<scf::ForallOp>(*payload.begin());
121  if (!target) {
123  emitSilenceableError() << "expected the payload to be scf.forall";
124  diag.attachNote((*payload.begin())->getLoc()) << "payload op";
125  return diag;
126  }
127 
128  if (!target.getOutputs().empty()) {
129  return emitSilenceableError()
130  << "unsupported shared outputs (didn't bufferize?)";
131  }
132 
133  if (getNumResults() != 1) {
134  DiagnosedSilenceableFailure diag = emitSilenceableError()
135  << "op expects one result, given "
136  << getNumResults();
137  diag.attachNote(target.getLoc()) << "payload op";
138  return diag;
139  }
140 
141  scf::ParallelOp opResult;
142  if (failed(scf::forallToParallelLoop(rewriter, target, &opResult))) {
144  emitSilenceableError() << "failed to convert forall into parallel";
145  return diag;
146  }
147 
148  results.set(cast<OpResult>(getTransformed()[0]), {opResult});
150 }
151 
152 //===----------------------------------------------------------------------===//
153 // LoopOutlineOp
154 //===----------------------------------------------------------------------===//
155 
156 /// Wraps the given operation `op` into an `scf.execute_region` operation. Uses
157 /// the provided rewriter for all operations to remain compatible with the
158 /// rewriting infra, as opposed to just splicing the op in place.
159 static scf::ExecuteRegionOp wrapInExecuteRegion(RewriterBase &b,
160  Operation *op) {
161  if (op->getNumRegions() != 1)
162  return nullptr;
164  b.setInsertionPoint(op);
165  scf::ExecuteRegionOp executeRegionOp =
166  b.create<scf::ExecuteRegionOp>(op->getLoc(), op->getResultTypes());
167  {
169  b.setInsertionPointToStart(&executeRegionOp.getRegion().emplaceBlock());
170  Operation *clonedOp = b.cloneWithoutRegions(*op);
171  Region &clonedRegion = clonedOp->getRegions().front();
172  assert(clonedRegion.empty() && "expected empty region");
173  b.inlineRegionBefore(op->getRegions().front(), clonedRegion,
174  clonedRegion.end());
175  b.create<scf::YieldOp>(op->getLoc(), clonedOp->getResults());
176  }
177  b.replaceOp(op, executeRegionOp.getResults());
178  return executeRegionOp;
179 }
180 
182 transform::LoopOutlineOp::apply(transform::TransformRewriter &rewriter,
184  transform::TransformState &state) {
185  SmallVector<Operation *> functions;
188  for (Operation *target : state.getPayloadOps(getTarget())) {
189  Location location = target->getLoc();
190  Operation *symbolTableOp = SymbolTable::getNearestSymbolTable(target);
191  scf::ExecuteRegionOp exec = wrapInExecuteRegion(rewriter, target);
192  if (!exec) {
193  DiagnosedSilenceableFailure diag = emitSilenceableError()
194  << "failed to outline";
195  diag.attachNote(target->getLoc()) << "target op";
196  return diag;
197  }
198  func::CallOp call;
199  FailureOr<func::FuncOp> outlined = outlineSingleBlockRegion(
200  rewriter, location, exec.getRegion(), getFuncName(), &call);
201 
202  if (failed(outlined))
203  return emitDefaultDefiniteFailure(target);
204 
205  if (symbolTableOp) {
206  SymbolTable &symbolTable =
207  symbolTables.try_emplace(symbolTableOp, symbolTableOp)
208  .first->getSecond();
209  symbolTable.insert(*outlined);
210  call.setCalleeAttr(FlatSymbolRefAttr::get(*outlined));
211  }
212  functions.push_back(*outlined);
213  calls.push_back(call);
214  }
215  results.set(cast<OpResult>(getFunction()), functions);
216  results.set(cast<OpResult>(getCall()), calls);
218 }
219 
220 //===----------------------------------------------------------------------===//
221 // LoopPeelOp
222 //===----------------------------------------------------------------------===//
223 
225 transform::LoopPeelOp::applyToOne(transform::TransformRewriter &rewriter,
226  scf::ForOp target,
228  transform::TransformState &state) {
229  scf::ForOp result;
230  if (getPeelFront()) {
231  LogicalResult status =
232  scf::peelForLoopFirstIteration(rewriter, target, result);
233  if (failed(status)) {
235  emitSilenceableError() << "failed to peel the first iteration";
236  return diag;
237  }
238  } else {
239  LogicalResult status =
240  scf::peelForLoopAndSimplifyBounds(rewriter, target, result);
241  if (failed(status)) {
242  DiagnosedSilenceableFailure diag = emitSilenceableError()
243  << "failed to peel the last iteration";
244  return diag;
245  }
246  }
247 
248  results.push_back(target);
249  results.push_back(result);
250 
252 }
253 
254 //===----------------------------------------------------------------------===//
255 // LoopPipelineOp
256 //===----------------------------------------------------------------------===//
257 
258 /// Callback for PipeliningOption. Populates `schedule` with the mapping from an
259 /// operation to its logical time position given the iteration interval and the
260 /// read latency. The latter is only relevant for vector transfers.
261 static void
262 loopScheduling(scf::ForOp forOp,
263  std::vector<std::pair<Operation *, unsigned>> &schedule,
264  unsigned iterationInterval, unsigned readLatency) {
265  auto getLatency = [&](Operation *op) -> unsigned {
266  if (isa<vector::TransferReadOp>(op))
267  return readLatency;
268  return 1;
269  };
270 
271  std::optional<int64_t> ubConstant =
272  getConstantIntValue(forOp.getUpperBound());
273  std::optional<int64_t> lbConstant =
274  getConstantIntValue(forOp.getLowerBound());
276  std::map<unsigned, std::vector<Operation *>> wrappedSchedule;
277  for (Operation &op : forOp.getBody()->getOperations()) {
278  if (isa<scf::YieldOp>(op))
279  continue;
280  unsigned earlyCycle = 0;
281  for (Value operand : op.getOperands()) {
282  Operation *def = operand.getDefiningOp();
283  if (!def)
284  continue;
285  if (ubConstant && lbConstant) {
286  unsigned ubInt = ubConstant.value();
287  unsigned lbInt = lbConstant.value();
288  auto minLatency = std::min(ubInt - lbInt - 1, getLatency(def));
289  earlyCycle = std::max(earlyCycle, opCycles[def] + minLatency);
290  } else {
291  earlyCycle = std::max(earlyCycle, opCycles[def] + getLatency(def));
292  }
293  }
294  opCycles[&op] = earlyCycle;
295  wrappedSchedule[earlyCycle % iterationInterval].push_back(&op);
296  }
297  for (const auto &it : wrappedSchedule) {
298  for (Operation *op : it.second) {
299  unsigned cycle = opCycles[op];
300  schedule.emplace_back(op, cycle / iterationInterval);
301  }
302  }
303 }
304 
306 transform::LoopPipelineOp::applyToOne(transform::TransformRewriter &rewriter,
307  scf::ForOp target,
309  transform::TransformState &state) {
311  options.getScheduleFn =
312  [this](scf::ForOp forOp,
313  std::vector<std::pair<Operation *, unsigned>> &schedule) mutable {
314  loopScheduling(forOp, schedule, getIterationInterval(),
315  getReadLatency());
316  };
317  scf::ForLoopPipeliningPattern pattern(options, target->getContext());
318  rewriter.setInsertionPoint(target);
319  FailureOr<scf::ForOp> patternResult =
320  scf::pipelineForLoop(rewriter, target, options);
321  if (succeeded(patternResult)) {
322  results.push_back(*patternResult);
324  }
325  return emitDefaultSilenceableFailure(target);
326 }
327 
328 //===----------------------------------------------------------------------===//
329 // LoopPromoteIfOneIterationOp
330 //===----------------------------------------------------------------------===//
331 
332 DiagnosedSilenceableFailure transform::LoopPromoteIfOneIterationOp::applyToOne(
333  transform::TransformRewriter &rewriter, LoopLikeOpInterface target,
335  transform::TransformState &state) {
336  (void)target.promoteIfSingleIteration(rewriter);
338 }
339 
340 void transform::LoopPromoteIfOneIterationOp::getEffects(
342  consumesHandle(getTargetMutable(), effects);
343  modifiesPayload(effects);
344 }
345 
346 //===----------------------------------------------------------------------===//
347 // LoopUnrollOp
348 //===----------------------------------------------------------------------===//
349 
351 transform::LoopUnrollOp::applyToOne(transform::TransformRewriter &rewriter,
352  Operation *op,
354  transform::TransformState &state) {
355  LogicalResult result(failure());
356  if (scf::ForOp scfFor = dyn_cast<scf::ForOp>(op))
357  result = loopUnrollByFactor(scfFor, getFactor());
358  else if (AffineForOp affineFor = dyn_cast<AffineForOp>(op))
359  result = loopUnrollByFactor(affineFor, getFactor());
360  else
361  return emitSilenceableError()
362  << "failed to unroll, incorrect type of payload";
363 
364  if (failed(result))
365  return emitSilenceableError() << "failed to unroll";
366 
368 }
369 
370 //===----------------------------------------------------------------------===//
371 // LoopUnrollAndJamOp
372 //===----------------------------------------------------------------------===//
373 
374 DiagnosedSilenceableFailure transform::LoopUnrollAndJamOp::applyToOne(
377  transform::TransformState &state) {
378  LogicalResult result(failure());
379  if (scf::ForOp scfFor = dyn_cast<scf::ForOp>(op))
380  result = loopUnrollJamByFactor(scfFor, getFactor());
381  else if (AffineForOp affineFor = dyn_cast<AffineForOp>(op))
382  result = loopUnrollJamByFactor(affineFor, getFactor());
383  else
384  return emitSilenceableError()
385  << "failed to unroll and jam, incorrect type of payload";
386 
387  if (failed(result))
388  return emitSilenceableError() << "failed to unroll and jam";
389 
391 }
392 
393 //===----------------------------------------------------------------------===//
394 // LoopCoalesceOp
395 //===----------------------------------------------------------------------===//
396 
398 transform::LoopCoalesceOp::applyToOne(transform::TransformRewriter &rewriter,
399  Operation *op,
401  transform::TransformState &state) {
402  LogicalResult result(failure());
403  if (scf::ForOp scfForOp = dyn_cast<scf::ForOp>(op))
404  result = coalescePerfectlyNestedSCFForLoops(scfForOp);
405  else if (AffineForOp affineForOp = dyn_cast<AffineForOp>(op))
406  result = coalescePerfectlyNestedAffineLoops(affineForOp);
407 
408  results.push_back(op);
409  if (failed(result)) {
410  DiagnosedSilenceableFailure diag = emitSilenceableError()
411  << "failed to coalesce";
412  return diag;
413  }
415 }
416 
417 //===----------------------------------------------------------------------===//
418 // TakeAssumedBranchOp
419 //===----------------------------------------------------------------------===//
420 /// Replaces the given op with the contents of the given single-block region,
421 /// using the operands of the block terminator to replace operation results.
422 static void replaceOpWithRegion(RewriterBase &rewriter, Operation *op,
423  Region &region) {
424  assert(llvm::hasSingleElement(region) && "expected single-region block");
425  Block *block = &region.front();
426  Operation *terminator = block->getTerminator();
427  ValueRange results = terminator->getOperands();
428  rewriter.inlineBlockBefore(block, op, /*blockArgs=*/{});
429  rewriter.replaceOp(op, results);
430  rewriter.eraseOp(terminator);
431 }
432 
433 DiagnosedSilenceableFailure transform::TakeAssumedBranchOp::applyToOne(
434  transform::TransformRewriter &rewriter, scf::IfOp ifOp,
436  transform::TransformState &state) {
437  rewriter.setInsertionPoint(ifOp);
438  Region &region =
439  getTakeElseBranch() ? ifOp.getElseRegion() : ifOp.getThenRegion();
440  if (!llvm::hasSingleElement(region)) {
441  return emitDefiniteFailure()
442  << "requires an scf.if op with a single-block "
443  << ((getTakeElseBranch()) ? "`else`" : "`then`") << " region";
444  }
445  replaceOpWithRegion(rewriter, ifOp, region);
447 }
448 
449 void transform::TakeAssumedBranchOp::getEffects(
451  onlyReadsHandle(getTargetMutable(), effects);
452  modifiesPayload(effects);
453 }
454 
455 //===----------------------------------------------------------------------===//
456 // LoopFuseSiblingOp
457 //===----------------------------------------------------------------------===//
458 
459 /// Check if `target` and `source` are siblings, in the context that `target`
460 /// is being fused into `source`.
461 ///
462 /// This is a simple check that just checks if both operations are in the same
463 /// block and some checks to ensure that the fused IR does not violate
464 /// dominance.
466  Operation *source) {
467  // Check if both operations are same.
468  if (target == source)
469  return emitSilenceableFailure(source)
470  << "target and source need to be different loops";
471 
472  // Check if both operations are in the same block.
473  if (target->getBlock() != source->getBlock())
474  return emitSilenceableFailure(source)
475  << "target and source are not in the same block";
476 
477  // Check if fusion will violate dominance.
478  DominanceInfo domInfo(source);
479  if (target->isBeforeInBlock(source)) {
480  // Since `target` is before `source`, all users of results of `target`
481  // need to be dominated by `source`.
482  for (Operation *user : target->getUsers()) {
483  if (!domInfo.properlyDominates(source, user, /*enclosingOpOk=*/false)) {
484  return emitSilenceableFailure(target)
485  << "user of results of target should be properly dominated by "
486  "source";
487  }
488  }
489  } else {
490  // Since `target` is after `source`, all values used by `target` need
491  // to dominate `source`.
492 
493  // Check if operands of `target` are dominated by `source`.
494  for (Value operand : target->getOperands()) {
495  Operation *operandOp = operand.getDefiningOp();
496  // Operands without defining operations are block arguments. When `target`
497  // and `source` occur in the same block, these operands dominate `source`.
498  if (!operandOp)
499  continue;
500 
501  // Operand's defining operation should properly dominate `source`.
502  if (!domInfo.properlyDominates(operandOp, source,
503  /*enclosingOpOk=*/false))
504  return emitSilenceableFailure(target)
505  << "operands of target should be properly dominated by source";
506  }
507 
508  // Check if values used by `target` are dominated by `source`.
509  bool failed = false;
510  OpOperand *failedValue = nullptr;
511  visitUsedValuesDefinedAbove(target->getRegions(), [&](OpOperand *operand) {
512  Operation *operandOp = operand->get().getDefiningOp();
513  if (operandOp && !domInfo.properlyDominates(operandOp, source,
514  /*enclosingOpOk=*/false)) {
515  // `operand` is not an argument of an enclosing block and the defining
516  // op of `operand` is outside `target` but does not dominate `source`.
517  failed = true;
518  failedValue = operand;
519  }
520  });
521 
522  if (failed)
523  return emitSilenceableFailure(failedValue->getOwner())
524  << "values used inside regions of target should be properly "
525  "dominated by source";
526  }
527 
529 }
530 
531 /// Check if `target` scf.forall can be fused into `source` scf.forall.
532 ///
533 /// This simply checks if both loops have the same bounds, steps and mapping.
534 /// No attempt is made at checking that the side effects of `target` and
535 /// `source` are independent of each other.
537  Operation *source) {
538  auto targetOp = dyn_cast<scf::ForallOp>(target);
539  auto sourceOp = dyn_cast<scf::ForallOp>(source);
540  if (!targetOp || !sourceOp)
541  return false;
542 
543  return targetOp.getMixedLowerBound() == sourceOp.getMixedLowerBound() &&
544  targetOp.getMixedUpperBound() == sourceOp.getMixedUpperBound() &&
545  targetOp.getMixedStep() == sourceOp.getMixedStep() &&
546  targetOp.getMapping() == sourceOp.getMapping();
547 }
548 
549 /// Check if `target` scf.for can be fused into `source` scf.for.
550 ///
551 /// This simply checks if both loops have the same bounds and steps. No attempt
552 /// is made at checking that the side effects of `target` and `source` are
553 /// independent of each other.
555  Operation *source) {
556  auto targetOp = dyn_cast<scf::ForOp>(target);
557  auto sourceOp = dyn_cast<scf::ForOp>(source);
558  if (!targetOp || !sourceOp)
559  return false;
560 
561  return targetOp.getLowerBound() == sourceOp.getLowerBound() &&
562  targetOp.getUpperBound() == sourceOp.getUpperBound() &&
563  targetOp.getStep() == sourceOp.getStep();
564 }
565 
567 transform::LoopFuseSiblingOp::apply(transform::TransformRewriter &rewriter,
569  transform::TransformState &state) {
570  auto targetOps = state.getPayloadOps(getTarget());
571  auto sourceOps = state.getPayloadOps(getSource());
572 
573  if (!llvm::hasSingleElement(targetOps) ||
574  !llvm::hasSingleElement(sourceOps)) {
575  return emitDefiniteFailure()
576  << "requires exactly one target handle (got "
577  << llvm::range_size(targetOps) << ") and exactly one "
578  << "source handle (got " << llvm::range_size(sourceOps) << ")";
579  }
580 
581  Operation *target = *targetOps.begin();
582  Operation *source = *sourceOps.begin();
583 
584  // Check if the target and source are siblings.
585  DiagnosedSilenceableFailure diag = isOpSibling(target, source);
586  if (!diag.succeeded())
587  return diag;
588 
589  Operation *fusedLoop;
590  /// TODO: Support fusion for loop-like ops besides scf.for and scf.forall.
591  if (isForWithIdenticalConfiguration(target, source)) {
592  fusedLoop = fuseIndependentSiblingForLoops(
593  cast<scf::ForOp>(target), cast<scf::ForOp>(source), rewriter);
594  } else if (isForallWithIdenticalConfiguration(target, source)) {
596  cast<scf::ForallOp>(target), cast<scf::ForallOp>(source), rewriter);
597  } else {
598  return emitSilenceableFailure(target->getLoc())
599  << "operations cannot be fused";
600  }
601 
602  assert(fusedLoop && "failed to fuse operations");
603 
604  results.set(cast<OpResult>(getFusedLoop()), {fusedLoop});
606 }
607 
608 //===----------------------------------------------------------------------===//
609 // Transform op registration
610 //===----------------------------------------------------------------------===//
611 
612 namespace {
613 class SCFTransformDialectExtension
615  SCFTransformDialectExtension> {
616 public:
617  MLIR_DEFINE_EXPLICIT_INTERNAL_INLINE_TYPE_ID(SCFTransformDialectExtension)
618 
619  using Base::Base;
620 
621  void init() {
622  declareGeneratedDialect<affine::AffineDialect>();
623  declareGeneratedDialect<func::FuncDialect>();
624 
625  registerTransformOps<
626 #define GET_OP_LIST
627 #include "mlir/Dialect/SCF/TransformOps/SCFTransformOps.cpp.inc"
628  >();
629  }
630 };
631 } // namespace
632 
633 #define GET_OP_CLASSES
634 #include "mlir/Dialect/SCF/TransformOps/SCFTransformOps.cpp.inc"
635 
637  registry.addExtensions<SCFTransformDialectExtension>();
638 }
static std::string diag(const llvm::Value &value)
static llvm::ManagedStatic< PassManagerOptions > options
static Value max(ImplicitLocOpBuilder &builder, Value value, Value bound)
static Value min(ImplicitLocOpBuilder &builder, Value value, Value bound)
static bool isForWithIdenticalConfiguration(Operation *target, Operation *source)
Check if target scf.for can be fused into source scf.for.
static DiagnosedSilenceableFailure isOpSibling(Operation *target, Operation *source)
Check if target and source are siblings, in the context that target is being fused into source.
static void loopScheduling(scf::ForOp forOp, std::vector< std::pair< Operation *, unsigned >> &schedule, unsigned iterationInterval, unsigned readLatency)
Callback for PipeliningOption.
static bool isForallWithIdenticalConfiguration(Operation *target, Operation *source)
Check if target scf.forall can be fused into source scf.forall.
static void replaceOpWithRegion(RewriterBase &rewriter, Operation *op, Region &region)
Replaces the given op with the contents of the given single-block region, using the operands of the b...
static scf::ExecuteRegionOp wrapInExecuteRegion(RewriterBase &b, Operation *op)
Wraps the given operation op into an scf.execute_region operation.
#define MLIR_DEFINE_EXPLICIT_INTERNAL_INLINE_TYPE_ID(CLASS_NAME)
Definition: TypeID.h:331
Block represents an ordered list of Operations.
Definition: Block.h:33
Operation * getTerminator()
Get the terminator operation of this block.
Definition: Block.cpp:244
This class describes a specific conversion target.
The result of a transform IR operation application.
static DiagnosedSilenceableFailure success()
Constructs a DiagnosedSilenceableFailure in the success state.
The DialectRegistry maps a dialect namespace to a constructor for the matching dialect.
void addExtensions()
Add the given extensions to the registry.
A class for computing basic dominance information.
Definition: Dominance.h:140
bool properlyDominates(Operation *a, Operation *b, bool enclosingOpOk=true) const
Return true if operation A properly dominates operation B, i.e.
Definition: Dominance.cpp:323
static FlatSymbolRefAttr get(StringAttr value)
Construct a symbol reference for the given value name.
This class defines the main interface for locations in MLIR and acts as a non-nullable wrapper around...
Definition: Location.h:76
RAII guard to reset the insertion point of the builder when destroyed.
Definition: Builders.h:346
void setInsertionPointToStart(Block *block)
Sets the insertion point to the start of the specified block.
Definition: Builders.h:429
void setInsertionPoint(Block *block, Block::iterator insertPoint)
Set the insertion point to the specified location.
Definition: Builders.h:396
Operation * create(const OperationState &state)
Creates an operation given the fields represented as an OperationState.
Definition: Builders.cpp:452
Operation * cloneWithoutRegions(Operation &op, IRMapping &mapper)
Creates a deep copy of this operation but keep the operation regions empty.
Definition: Builders.h:585
This class represents an operand of an operation.
Definition: Value.h:257
Operation is the basic unit of execution within MLIR.
Definition: Operation.h:88
bool isBeforeInBlock(Operation *other)
Given an operation 'other' that is within the same parent block, return whether the current operation...
Definition: Operation.cpp:385
unsigned getNumRegions()
Returns the number of regions held by this operation.
Definition: Operation.h:674
Location getLoc()
The source location the operation was defined or derived from.
Definition: Operation.h:223
Block * getBlock()
Returns the operation block that contains this operation.
Definition: Operation.h:213
MutableArrayRef< Region > getRegions()
Returns the regions held by this operation.
Definition: Operation.h:677
result_type_range getResultTypes()
Definition: Operation.h:428
operand_range getOperands()
Returns an iterator on the underlying Value's.
Definition: Operation.h:378
user_range getUsers()
Returns a range of all users.
Definition: Operation.h:873
result_range getResults()
Definition: Operation.h:415
This class contains a list of basic blocks and a link to the parent operation it is attached to.
Definition: Region.h:26
bool empty()
Definition: Region.h:60
iterator end()
Definition: Region.h:56
Block & front()
Definition: Region.h:65
This class coordinates the application of a rewrite on a set of IR, providing a way for clients to tr...
Definition: PatternMatch.h:358
virtual void replaceOp(Operation *op, ValueRange newValues)
Replace the results of the given (original) operation with the specified list of values (replacements...
virtual void eraseOp(Operation *op)
This method erases an operation that is known to have no uses.
virtual void inlineBlockBefore(Block *source, Block *dest, Block::iterator before, ValueRange argValues={})
Inline the operations of block 'source' into block 'dest' before the given position.
void inlineRegionBefore(Region &region, Region &parent, Region::iterator before)
Move the blocks that belong to "region" before the given position in another region "parent".
This class allows for representing and managing the symbol table used by operations with the 'SymbolT...
Definition: SymbolTable.h:24
StringAttr insert(Operation *symbol, Block::iterator insertPt={})
Insert a new symbol into the table, and rename it as necessary to avoid collisions.
static Operation * getNearestSymbolTable(Operation *from)
Returns the nearest symbol table from a given operation from.
Type conversion class.
This class provides an abstraction over the different types of ranges over Values.
Definition: ValueRange.h:387
This class represents an instance of an SSA value in the MLIR system, representing a computable value...
Definition: Value.h:96
A list of results of applying a transform op with ApplyEachOpTrait to a single payload operation,...
void push_back(Operation *op)
Appends an element to the list.
Base class for extensions of the Transform dialect that supports injecting operations into the Transf...
Local mapping between values defined by a specific op implementing the TransformOpInterface and the p...
void set(OpResult value, Range &&ops)
Indicates that the result of the transform IR op at the given position corresponds to the given list ...
This is a special rewriter to be used in transform op implementations, providing additional helper fu...
The state maintained across applications of various ops implementing the TransformOpInterface.
LogicalResult loopUnrollByFactor(AffineForOp forOp, uint64_t unrollFactor, function_ref< void(unsigned, Operation *, OpBuilder)> annotateFn=nullptr, bool cleanUpUnroll=false)
Unrolls this for operation by the specified unroll factor.
Definition: LoopUtils.cpp:1010
LogicalResult loopUnrollJamByFactor(AffineForOp forOp, uint64_t unrollJamFactor)
Unrolls and jams this loop by the specified factor.
Definition: LoopUtils.cpp:1099
LogicalResult coalescePerfectlyNestedAffineLoops(AffineForOp op)
Walk an affine.for to find a band to coalesce.
Definition: LoopUtils.cpp:2785
constexpr void enumerate(std::tuple< Tys... > &tuple, CallbackT &&callback)
Definition: Matchers.h:344
void registerTransformDialectExtension(DialectRegistry &registry)
LogicalResult peelForLoopAndSimplifyBounds(RewriterBase &rewriter, ForOp forOp, scf::ForOp &partialIteration)
Rewrite a for loop with bounds/step that potentially do not divide evenly into a for loop where the s...
LogicalResult forallToForLoop(RewriterBase &rewriter, ForallOp forallOp, SmallVectorImpl< Operation * > *results=nullptr)
Try converting scf.forall into a set of nested scf.for loops.
LogicalResult peelForLoopFirstIteration(RewriterBase &rewriter, ForOp forOp, scf::ForOp &partialIteration)
Peel the first iteration out of the scf.for loop.
LogicalResult forallToParallelLoop(RewriterBase &rewriter, ForallOp forallOp, ParallelOp *result=nullptr)
Try converting scf.forall into an scf.parallel loop.
void populateSCFStructuralTypeConversions(const TypeConverter &typeConverter, RewritePatternSet &patterns)
Similar to populateSCFStructuralTypeConversionsAndLegality but does not populate the conversion targe...
void populateSCFForLoopCanonicalizationPatterns(RewritePatternSet &patterns)
Populate patterns for canonicalizing operations inside SCF loop bodies.
FailureOr< ForOp > pipelineForLoop(RewriterBase &rewriter, ForOp forOp, const PipeliningOption &options, bool *modifiedIR=nullptr)
Generate a pipelined version of the scf.for loop based on the schedule given as option.
void populateSCFStructuralTypeConversionTarget(const TypeConverter &typeConverter, ConversionTarget &target)
Updates the ConversionTarget with dynamic legality of SCF operations based on the provided type conve...
void consumesHandle(MutableArrayRef< OpOperand > handles, SmallVectorImpl< MemoryEffects::EffectInstance > &effects)
Populates effects with the memory effects indicating the operation on the given handle value:
void onlyReadsHandle(MutableArrayRef< OpOperand > handles, SmallVectorImpl< MemoryEffects::EffectInstance > &effects)
void modifiesPayload(SmallVectorImpl< MemoryEffects::EffectInstance > &effects)
Populates effects with the memory effects indicating the access to payload IR resource.
Include the generated interface declarations.
std::optional< int64_t > getConstantIntValue(OpFoldResult ofr)
If ofr is a constant integer or an IntegerAttr, return the integer.
LogicalResult coalescePerfectlyNestedSCFForLoops(scf::ForOp op)
Walk an affine.for to find a band to coalesce.
Definition: Utils.cpp:990
DiagnosedSilenceableFailure emitSilenceableFailure(Location loc, const Twine &message={})
Emits a silenceable failure with the given message.
DiagnosedDefiniteFailure emitDefiniteFailure(Location loc, const Twine &message={})
Emits a definite failure with the given message.
const FrozenRewritePatternSet & patterns
void populateSCFToControlFlowConversionPatterns(RewritePatternSet &patterns)
Collect a set of patterns to convert SCF operations to CFG branch-based operations within the Control...
FailureOr< func::FuncOp > outlineSingleBlockRegion(RewriterBase &rewriter, Location loc, Region &region, StringRef funcName, func::CallOp *callOp=nullptr)
Outline a region with a single block into a new FuncOp.
Definition: Utils.cpp:119
scf::ForallOp fuseIndependentSiblingForallLoops(scf::ForallOp target, scf::ForallOp source, RewriterBase &rewriter)
Given two scf.forall loops, target and source, fuses target into source.
Definition: Utils.cpp:1366
scf::ForOp fuseIndependentSiblingForLoops(scf::ForOp target, scf::ForOp source, RewriterBase &rewriter)
Given two scf.for loops, target and source, fuses target into source.
Definition: Utils.cpp:1418
void visitUsedValuesDefinedAbove(Region &region, Region &limit, function_ref< void(OpOperand *)> callback)
Calls callback for each use of a value within region or its descendants that was defined at the ances...
Definition: RegionUtils.cpp:40
Options to dictate how loops should be pipelined.
Definition: Transforms.h:123