MLIR  18.0.0git
LoopPipelining.cpp
Go to the documentation of this file.
1 //===- LoopPipelining.cpp - Code to perform loop software pipelining-------===//
2 //
3 // Part of the LLVM Project, under the Apache License v2.0 with LLVM Exceptions.
4 // See https://llvm.org/LICENSE.txt for license information.
5 // SPDX-License-Identifier: Apache-2.0 WITH LLVM-exception
6 //
7 //===----------------------------------------------------------------------===//
8 //
9 // This file implements loop software pipelining
10 //
11 //===----------------------------------------------------------------------===//
12 
18 #include "mlir/IR/IRMapping.h"
19 #include "mlir/IR/PatternMatch.h"
22 #include "llvm/ADT/MapVector.h"
23 #include "llvm/Support/Debug.h"
24 
25 #define DEBUG_TYPE "scf-loop-pipelining"
26 #define DBGS() (llvm::dbgs() << "[" DEBUG_TYPE "]: ")
27 #define LDBG(X) LLVM_DEBUG(DBGS() << X << "\n")
28 
29 using namespace mlir;
30 using namespace mlir::scf;
31 
32 namespace {
33 
34 /// Helper to keep internal information during pipelining transformation.
35 struct LoopPipelinerInternal {
36  /// Coarse liverange information for ops used across stages.
37  struct LiverangeInfo {
38  unsigned lastUseStage = 0;
39  unsigned defStage = 0;
40  };
41 
42 protected:
43  ForOp forOp;
44  unsigned maxStage = 0;
46  std::vector<Operation *> opOrder;
47  int64_t ub;
48  int64_t lb;
49  int64_t step;
50  PipeliningOption::AnnotationlFnType annotateFn = nullptr;
51  bool peelEpilogue;
52  PipeliningOption::PredicateOpFn predicateFn = nullptr;
53 
54  // When peeling the kernel we generate several version of each value for
55  // different stage of the prologue. This map tracks the mapping between
56  // original Values in the loop and the different versions
57  // peeled from the loop.
59 
60  /// Assign a value to `valueMapping`, this means `val` represents the version
61  /// `idx` of `key` in the epilogue.
62  void setValueMapping(Value key, Value el, int64_t idx);
63 
64 public:
65  /// Initalize the information for the given `op`, return true if it
66  /// satisfies the pre-condition to apply pipelining.
67  bool initializeLoopInfo(ForOp op, const PipeliningOption &options);
68  /// Emits the prologue, this creates `maxStage - 1` part which will contain
69  /// operations from stages [0; i], where i is the part index.
70  void emitPrologue(RewriterBase &rewriter);
71  /// Gather liverange information for Values that are used in a different stage
72  /// than its definition.
73  llvm::MapVector<Value, LiverangeInfo> analyzeCrossStageValues();
74  scf::ForOp createKernelLoop(
75  const llvm::MapVector<Value, LiverangeInfo> &crossStageValues,
76  RewriterBase &rewriter,
77  llvm::DenseMap<std::pair<Value, unsigned>, unsigned> &loopArgMap);
78  /// Emits the pipelined kernel. This clones loop operations following user
79  /// order and remaps operands defined in a different stage as their use.
80  LogicalResult createKernel(
81  scf::ForOp newForOp,
82  const llvm::MapVector<Value, LiverangeInfo> &crossStageValues,
83  const llvm::DenseMap<std::pair<Value, unsigned>, unsigned> &loopArgMap,
84  RewriterBase &rewriter);
85  /// Emits the epilogue, this creates `maxStage - 1` part which will contain
86  /// operations from stages [i; maxStage], where i is the part index.
87  llvm::SmallVector<Value> emitEpilogue(RewriterBase &rewriter);
88 };
89 
90 bool LoopPipelinerInternal::initializeLoopInfo(
91  ForOp op, const PipeliningOption &options) {
92  LDBG("Start initializeLoopInfo");
93  forOp = op;
94  auto upperBoundCst =
95  forOp.getUpperBound().getDefiningOp<arith::ConstantIndexOp>();
96  auto lowerBoundCst =
97  forOp.getLowerBound().getDefiningOp<arith::ConstantIndexOp>();
98  auto stepCst = forOp.getStep().getDefiningOp<arith::ConstantIndexOp>();
99  if (!upperBoundCst || !lowerBoundCst || !stepCst) {
100  LDBG("--no constant bounds or step -> BAIL");
101  return false;
102  }
103  ub = upperBoundCst.value();
104  lb = lowerBoundCst.value();
105  step = stepCst.value();
106  peelEpilogue = options.peelEpilogue;
107  predicateFn = options.predicateFn;
108  if (!peelEpilogue && predicateFn == nullptr) {
109  LDBG("--no epilogue or predicate set -> BAIL");
110  return false;
111  }
112  int64_t numIteration = ceilDiv(ub - lb, step);
113  std::vector<std::pair<Operation *, unsigned>> schedule;
114  options.getScheduleFn(forOp, schedule);
115  if (schedule.empty()) {
116  LDBG("--empty schedule -> BAIL");
117  return false;
118  }
119 
120  opOrder.reserve(schedule.size());
121  for (auto &opSchedule : schedule) {
122  maxStage = std::max(maxStage, opSchedule.second);
123  stages[opSchedule.first] = opSchedule.second;
124  opOrder.push_back(opSchedule.first);
125  }
126  if (numIteration <= maxStage) {
127  LDBG("--fewer loop iterations than pipeline stages -> BAIL");
128  return false;
129  }
130 
131  // All operations need to have a stage.
132  for (Operation &op : forOp.getBody()->without_terminator()) {
133  if (!stages.contains(&op)) {
134  op.emitOpError("not assigned a pipeline stage");
135  LDBG("--op not assigned a pipeline stage: " << op << " -> BAIL");
136  return false;
137  }
138  }
139 
140  // Currently, we do not support assigning stages to ops in nested regions. The
141  // block of all operations assigned a stage should be the single `scf.for`
142  // body block.
143  for (const auto &[op, stageNum] : stages) {
144  (void)stageNum;
145  if (op == forOp.getBody()->getTerminator()) {
146  op->emitError("terminator should not be assigned a stage");
147  LDBG("--terminator should not be assigned stage: " << *op << " -> BAIL");
148  return false;
149  }
150  if (op->getBlock() != forOp.getBody()) {
151  op->emitOpError("the owning Block of all operations assigned a stage "
152  "should be the loop body block");
153  LDBG("--the owning Block of all operations assigned a stage "
154  "should be the loop body block: "
155  << *op << " -> BAIL");
156  return false;
157  }
158  }
159 
160  // Only support loop carried dependency with a distance of 1. This means the
161  // source of all the scf.yield operands needs to be defined by operations in
162  // the loop.
163  if (llvm::any_of(forOp.getBody()->getTerminator()->getOperands(),
164  [this](Value operand) {
165  Operation *def = operand.getDefiningOp();
166  return !def || !stages.contains(def);
167  })) {
168  LDBG("--only support loop carried dependency with a distance of 1 -> BAIL");
169  return false;
170  }
171  annotateFn = options.annotateFn;
172  return true;
173 }
174 
175 /// Clone `op` and call `callback` on the cloned op's oeprands as well as any
176 /// operands of nested ops that:
177 /// 1) aren't defined within the new op or
178 /// 2) are block arguments.
179 static Operation *
180 cloneAndUpdateOperands(RewriterBase &rewriter, Operation *op,
181  function_ref<void(OpOperand *newOperand)> callback) {
182  Operation *clone = rewriter.clone(*op);
183  for (OpOperand &operand : clone->getOpOperands())
184  callback(&operand);
185  clone->walk([&](Operation *nested) {
186  for (OpOperand &operand : nested->getOpOperands()) {
187  Operation *def = operand.get().getDefiningOp();
188  if ((def && !clone->isAncestor(def)) || isa<BlockArgument>(operand.get()))
189  callback(&operand);
190  }
191  });
192  return clone;
193 }
194 
195 void LoopPipelinerInternal::emitPrologue(RewriterBase &rewriter) {
196  // Initialize the iteration argument to the loop initiale values.
197  for (auto [arg, operand] :
198  llvm::zip(forOp.getRegionIterArgs(), forOp.getInitsMutable())) {
199  setValueMapping(arg, operand.get(), 0);
200  }
201  auto yield = cast<scf::YieldOp>(forOp.getBody()->getTerminator());
202  for (int64_t i = 0; i < maxStage; i++) {
203  // special handling for induction variable as the increment is implicit.
204  Value iv =
205  rewriter.create<arith::ConstantIndexOp>(forOp.getLoc(), lb + i * step);
206  setValueMapping(forOp.getInductionVar(), iv, i);
207  for (Operation *op : opOrder) {
208  if (stages[op] > i)
209  continue;
210  Operation *newOp =
211  cloneAndUpdateOperands(rewriter, op, [&](OpOperand *newOperand) {
212  auto it = valueMapping.find(newOperand->get());
213  if (it != valueMapping.end()) {
214  Value replacement = it->second[i - stages[op]];
215  newOperand->set(replacement);
216  }
217  });
218  if (annotateFn)
219  annotateFn(newOp, PipeliningOption::PipelinerPart::Prologue, i);
220  for (unsigned destId : llvm::seq(unsigned(0), op->getNumResults())) {
221  setValueMapping(op->getResult(destId), newOp->getResult(destId),
222  i - stages[op]);
223  // If the value is a loop carried dependency update the loop argument
224  // mapping.
225  for (OpOperand &operand : yield->getOpOperands()) {
226  if (operand.get() != op->getResult(destId))
227  continue;
228  setValueMapping(forOp.getRegionIterArgs()[operand.getOperandNumber()],
229  newOp->getResult(destId), i - stages[op] + 1);
230  }
231  }
232  }
233  }
234 }
235 
236 llvm::MapVector<Value, LoopPipelinerInternal::LiverangeInfo>
237 LoopPipelinerInternal::analyzeCrossStageValues() {
238  llvm::MapVector<Value, LoopPipelinerInternal::LiverangeInfo> crossStageValues;
239  for (Operation *op : opOrder) {
240  unsigned stage = stages[op];
241 
242  auto analyzeOperand = [&](OpOperand &operand) {
243  Operation *def = operand.get().getDefiningOp();
244  if (!def)
245  return;
246  auto defStage = stages.find(def);
247  if (defStage == stages.end() || defStage->second == stage)
248  return;
249  assert(stage > defStage->second);
250  LiverangeInfo &info = crossStageValues[operand.get()];
251  info.defStage = defStage->second;
252  info.lastUseStage = std::max(info.lastUseStage, stage);
253  };
254 
255  for (OpOperand &operand : op->getOpOperands())
256  analyzeOperand(operand);
257  visitUsedValuesDefinedAbove(op->getRegions(), [&](OpOperand *operand) {
258  analyzeOperand(*operand);
259  });
260  }
261  return crossStageValues;
262 }
263 
264 scf::ForOp LoopPipelinerInternal::createKernelLoop(
265  const llvm::MapVector<Value, LoopPipelinerInternal::LiverangeInfo>
266  &crossStageValues,
267  RewriterBase &rewriter,
268  llvm::DenseMap<std::pair<Value, unsigned>, unsigned> &loopArgMap) {
269  // Creates the list of initial values associated to values used across
270  // stages. The initial values come from the prologue created above.
271  // Keep track of the kernel argument associated to each version of the
272  // values passed to the kernel.
273  llvm::SmallVector<Value> newLoopArg;
274  // For existing loop argument initialize them with the right version from the
275  // prologue.
276  for (const auto &retVal :
277  llvm::enumerate(forOp.getBody()->getTerminator()->getOperands())) {
278  Operation *def = retVal.value().getDefiningOp();
279  assert(def && "Only support loop carried dependencies of distance 1");
280  unsigned defStage = stages[def];
281  Value valueVersion = valueMapping[forOp.getRegionIterArgs()[retVal.index()]]
282  [maxStage - defStage];
283  assert(valueVersion);
284  newLoopArg.push_back(valueVersion);
285  }
286  for (auto escape : crossStageValues) {
287  LiverangeInfo &info = escape.second;
288  Value value = escape.first;
289  for (unsigned stageIdx = 0; stageIdx < info.lastUseStage - info.defStage;
290  stageIdx++) {
291  Value valueVersion =
292  valueMapping[value][maxStage - info.lastUseStage + stageIdx];
293  assert(valueVersion);
294  newLoopArg.push_back(valueVersion);
295  loopArgMap[std::make_pair(value, info.lastUseStage - info.defStage -
296  stageIdx)] = newLoopArg.size() - 1;
297  }
298  }
299 
300  // Create the new kernel loop. When we peel the epilgue we need to peel
301  // `numStages - 1` iterations. Then we adjust the upper bound to remove those
302  // iterations.
303  Value newUb = forOp.getUpperBound();
304  if (peelEpilogue)
305  newUb = rewriter.create<arith::ConstantIndexOp>(forOp.getLoc(),
306  ub - maxStage * step);
307  auto newForOp =
308  rewriter.create<scf::ForOp>(forOp.getLoc(), forOp.getLowerBound(), newUb,
309  forOp.getStep(), newLoopArg);
310  // When there are no iter args, the loop body terminator will be created.
311  // Since we always create it below, remove the terminator if it was created.
312  if (!newForOp.getBody()->empty())
313  rewriter.eraseOp(newForOp.getBody()->getTerminator());
314  return newForOp;
315 }
316 
317 LogicalResult LoopPipelinerInternal::createKernel(
318  scf::ForOp newForOp,
319  const llvm::MapVector<Value, LoopPipelinerInternal::LiverangeInfo>
320  &crossStageValues,
321  const llvm::DenseMap<std::pair<Value, unsigned>, unsigned> &loopArgMap,
322  RewriterBase &rewriter) {
323  valueMapping.clear();
324 
325  // Create the kernel, we clone instruction based on the order given by
326  // user and remap operands coming from a previous stages.
327  rewriter.setInsertionPoint(newForOp.getBody(), newForOp.getBody()->begin());
328  IRMapping mapping;
329  mapping.map(forOp.getInductionVar(), newForOp.getInductionVar());
330  for (const auto &arg : llvm::enumerate(forOp.getRegionIterArgs())) {
331  mapping.map(arg.value(), newForOp.getRegionIterArgs()[arg.index()]);
332  }
333  SmallVector<Value> predicates(maxStage + 1, nullptr);
334  if (!peelEpilogue) {
335  // Create a predicate for each stage except the last stage.
336  for (unsigned i = 0; i < maxStage; i++) {
337  Value c = rewriter.create<arith::ConstantIndexOp>(
338  newForOp.getLoc(), ub - (maxStage - i) * step);
339  Value pred = rewriter.create<arith::CmpIOp>(
340  newForOp.getLoc(), arith::CmpIPredicate::slt,
341  newForOp.getInductionVar(), c);
342  predicates[i] = pred;
343  }
344  }
345  for (Operation *op : opOrder) {
346  int64_t useStage = stages[op];
347  auto *newOp = rewriter.clone(*op, mapping);
348  SmallVector<OpOperand *> operands;
349  // Collect all the operands for the cloned op and its nested ops.
350  op->walk([&operands](Operation *nestedOp) {
351  for (OpOperand &operand : nestedOp->getOpOperands()) {
352  operands.push_back(&operand);
353  }
354  });
355  for (OpOperand *operand : operands) {
356  Operation *nestedNewOp = mapping.lookup(operand->getOwner());
357  // Special case for the induction variable uses. We replace it with a
358  // version incremented based on the stage where it is used.
359  if (operand->get() == forOp.getInductionVar()) {
360  rewriter.setInsertionPoint(newOp);
361  Value offset = rewriter.create<arith::ConstantIndexOp>(
362  forOp.getLoc(), (maxStage - stages[op]) * step);
363  Value iv = rewriter.create<arith::AddIOp>(
364  forOp.getLoc(), newForOp.getInductionVar(), offset);
365  nestedNewOp->setOperand(operand->getOperandNumber(), iv);
366  rewriter.setInsertionPointAfter(newOp);
367  continue;
368  }
369  auto arg = dyn_cast<BlockArgument>(operand->get());
370  if (arg && arg.getOwner() == forOp.getBody()) {
371  // If the value is a loop carried value coming from stage N + 1 remap,
372  // it will become a direct use.
373  Value ret = forOp.getBody()->getTerminator()->getOperand(
374  arg.getArgNumber() - 1);
375  Operation *dep = ret.getDefiningOp();
376  if (!dep)
377  continue;
378  auto stageDep = stages.find(dep);
379  if (stageDep == stages.end() || stageDep->second == useStage)
380  continue;
381  assert(stageDep->second == useStage + 1);
382  nestedNewOp->setOperand(operand->getOperandNumber(),
383  mapping.lookupOrDefault(ret));
384  continue;
385  }
386  // For operands defined in a previous stage we need to remap it to use
387  // the correct region argument. We look for the right version of the
388  // Value based on the stage where it is used.
389  Operation *def = operand->get().getDefiningOp();
390  if (!def)
391  continue;
392  auto stageDef = stages.find(def);
393  if (stageDef == stages.end() || stageDef->second == useStage)
394  continue;
395  auto remap = loopArgMap.find(
396  std::make_pair(operand->get(), useStage - stageDef->second));
397  assert(remap != loopArgMap.end());
398  nestedNewOp->setOperand(operand->getOperandNumber(),
399  newForOp.getRegionIterArgs()[remap->second]);
400  }
401 
402  if (predicates[useStage]) {
403  newOp = predicateFn(rewriter, newOp, predicates[useStage]);
404  if (!newOp)
405  return failure();
406  // Remap the results to the new predicated one.
407  for (auto values : llvm::zip(op->getResults(), newOp->getResults()))
408  mapping.map(std::get<0>(values), std::get<1>(values));
409  }
410  rewriter.setInsertionPointAfter(newOp);
411  if (annotateFn)
412  annotateFn(newOp, PipeliningOption::PipelinerPart::Kernel, 0);
413  }
414 
415  // Collect the Values that need to be returned by the forOp. For each
416  // value we need to have `LastUseStage - DefStage` number of versions
417  // returned.
418  // We create a mapping between original values and the associated loop
419  // returned values that will be needed by the epilogue.
420  llvm::SmallVector<Value> yieldOperands;
421  for (Value retVal : forOp.getBody()->getTerminator()->getOperands()) {
422  yieldOperands.push_back(mapping.lookupOrDefault(retVal));
423  }
424  for (auto &it : crossStageValues) {
425  int64_t version = maxStage - it.second.lastUseStage + 1;
426  unsigned numVersionReturned = it.second.lastUseStage - it.second.defStage;
427  // add the original version to yield ops.
428  // If there is a live range spanning across more than 2 stages we need to
429  // add extra arg.
430  for (unsigned i = 1; i < numVersionReturned; i++) {
431  setValueMapping(it.first, newForOp->getResult(yieldOperands.size()),
432  version++);
433  yieldOperands.push_back(
434  newForOp.getBody()->getArguments()[yieldOperands.size() + 1 +
435  newForOp.getNumInductionVars()]);
436  }
437  setValueMapping(it.first, newForOp->getResult(yieldOperands.size()),
438  version++);
439  yieldOperands.push_back(mapping.lookupOrDefault(it.first));
440  }
441  // Map the yield operand to the forOp returned value.
442  for (const auto &retVal :
443  llvm::enumerate(forOp.getBody()->getTerminator()->getOperands())) {
444  Operation *def = retVal.value().getDefiningOp();
445  assert(def && "Only support loop carried dependencies of distance 1");
446  unsigned defStage = stages[def];
447  setValueMapping(forOp.getRegionIterArgs()[retVal.index()],
448  newForOp->getResult(retVal.index()),
449  maxStage - defStage + 1);
450  }
451  rewriter.create<scf::YieldOp>(forOp.getLoc(), yieldOperands);
452  return success();
453 }
454 
456 LoopPipelinerInternal::emitEpilogue(RewriterBase &rewriter) {
457  llvm::SmallVector<Value> returnValues(forOp->getNumResults());
458  // Emit different versions of the induction variable. They will be
459  // removed by dead code if not used.
460  for (int64_t i = 0; i < maxStage; i++) {
461  Value newlastIter = rewriter.create<arith::ConstantIndexOp>(
462  forOp.getLoc(), lb + step * ((((ub - 1) - lb) / step) - i));
463  setValueMapping(forOp.getInductionVar(), newlastIter, maxStage - i);
464  }
465  // Emit `maxStage - 1` epilogue part that includes operations from stages
466  // [i; maxStage].
467  for (int64_t i = 1; i <= maxStage; i++) {
468  for (Operation *op : opOrder) {
469  if (stages[op] < i)
470  continue;
471  Operation *newOp =
472  cloneAndUpdateOperands(rewriter, op, [&](OpOperand *newOperand) {
473  auto it = valueMapping.find(newOperand->get());
474  if (it != valueMapping.end()) {
475  Value replacement = it->second[maxStage - stages[op] + i];
476  newOperand->set(replacement);
477  }
478  });
479  if (annotateFn)
480  annotateFn(newOp, PipeliningOption::PipelinerPart::Epilogue, i - 1);
481  for (unsigned destId : llvm::seq(unsigned(0), op->getNumResults())) {
482  setValueMapping(op->getResult(destId), newOp->getResult(destId),
483  maxStage - stages[op] + i);
484  // If the value is a loop carried dependency update the loop argument
485  // mapping and keep track of the last version to replace the original
486  // forOp uses.
487  for (OpOperand &operand :
488  forOp.getBody()->getTerminator()->getOpOperands()) {
489  if (operand.get() != op->getResult(destId))
490  continue;
491  unsigned version = maxStage - stages[op] + i + 1;
492  // If the version is greater than maxStage it means it maps to the
493  // original forOp returned value.
494  if (version > maxStage) {
495  returnValues[operand.getOperandNumber()] = newOp->getResult(destId);
496  continue;
497  }
498  setValueMapping(forOp.getRegionIterArgs()[operand.getOperandNumber()],
499  newOp->getResult(destId), version);
500  }
501  }
502  }
503  }
504  return returnValues;
505 }
506 
507 void LoopPipelinerInternal::setValueMapping(Value key, Value el, int64_t idx) {
508  auto it = valueMapping.find(key);
509  // If the value is not in the map yet add a vector big enough to store all
510  // versions.
511  if (it == valueMapping.end())
512  it =
513  valueMapping
514  .insert(std::make_pair(key, llvm::SmallVector<Value>(maxStage + 1)))
515  .first;
516  it->second[idx] = el;
517 }
518 
519 } // namespace
520 
522  const PipeliningOption &options,
523  bool *modifiedIR) {
524  if (modifiedIR)
525  *modifiedIR = false;
526  LoopPipelinerInternal pipeliner;
527  if (!pipeliner.initializeLoopInfo(forOp, options))
528  return failure();
529 
530  if (modifiedIR)
531  *modifiedIR = true;
532 
533  // 1. Emit prologue.
534  pipeliner.emitPrologue(rewriter);
535 
536  // 2. Track values used across stages. When a value cross stages it will
537  // need to be passed as loop iteration arguments.
538  // We first collect the values that are used in a different stage than where
539  // they are defined.
540  llvm::MapVector<Value, LoopPipelinerInternal::LiverangeInfo>
541  crossStageValues = pipeliner.analyzeCrossStageValues();
542 
543  // Mapping between original loop values used cross stage and the block
544  // arguments associated after pipelining. A Value may map to several
545  // arguments if its liverange spans across more than 2 stages.
546  llvm::DenseMap<std::pair<Value, unsigned>, unsigned> loopArgMap;
547  // 3. Create the new kernel loop and return the block arguments mapping.
548  ForOp newForOp =
549  pipeliner.createKernelLoop(crossStageValues, rewriter, loopArgMap);
550  // Create the kernel block, order ops based on user choice and remap
551  // operands.
552  if (failed(pipeliner.createKernel(newForOp, crossStageValues, loopArgMap,
553  rewriter)))
554  return failure();
555 
556  llvm::SmallVector<Value> returnValues =
557  newForOp.getResults().take_front(forOp->getNumResults());
558  if (options.peelEpilogue) {
559  // 4. Emit the epilogue after the new forOp.
560  rewriter.setInsertionPointAfter(newForOp);
561  returnValues = pipeliner.emitEpilogue(rewriter);
562  }
563  // 5. Erase the original loop and replace the uses with the epilogue output.
564  if (forOp->getNumResults() > 0)
565  rewriter.replaceOp(forOp, returnValues);
566  else
567  rewriter.eraseOp(forOp);
568 
569  return newForOp;
570 }
571 
573  RewritePatternSet &patterns, const PipeliningOption &options) {
574  patterns.add<ForLoopPipeliningPattern>(options, patterns.getContext());
575 }
#define LDBG(X)
static llvm::ManagedStatic< PassManagerOptions > options
static Value max(ImplicitLocOpBuilder &builder, Value value, Value bound)
This class provides support for representing a failure result, or a valid value of type T.
Definition: LogicalResult.h:78
This is a utility class for mapping one set of IR entities to another.
Definition: IRMapping.h:26
auto lookupOrDefault(T from) const
Lookup a mapped value within the map.
Definition: IRMapping.h:65
auto lookup(T from) const
Lookup a mapped value within the map.
Definition: IRMapping.h:72
void map(Value from, Value to)
Inserts a new mapping for 'from' to 'to'.
Definition: IRMapping.h:30
IRValueT get() const
Return the current value being used by this operand.
Definition: UseDefLists.h:160
Operation * clone(Operation &op, IRMapping &mapper)
Creates a deep copy of the specified operation, remapping any operands that use values outside of the...
Definition: Builders.cpp:528
void setInsertionPoint(Block *block, Block::iterator insertPoint)
Set the insertion point to the specified location.
Definition: Builders.h:383
Operation * create(const OperationState &state)
Creates an operation given the fields represented as an OperationState.
Definition: Builders.cpp:446
void setInsertionPointAfter(Operation *op)
Sets the insertion point to the node after the specified operation, which will cause subsequent inser...
Definition: Builders.h:397
This class represents an operand of an operation.
Definition: Value.h:263
Operation is the basic unit of execution within MLIR.
Definition: Operation.h:88
void setOperand(unsigned idx, Value value)
Definition: Operation.h:346
OpResult getResult(unsigned idx)
Get the 'idx'th result of this operation.
Definition: Operation.h:402
std::enable_if_t< llvm::function_traits< std::decay_t< FnT > >::num_args==1, RetT > walk(FnT &&callback)
Walk the operation by calling the callback for each nested operation (including this one),...
Definition: Operation.h:776
InFlightDiagnostic emitError(const Twine &message={})
Emit an error about fatal conditions with this operation, reporting up to any diagnostic handlers tha...
Definition: Operation.cpp:267
Block * getBlock()
Returns the operation block that contains this operation.
Definition: Operation.h:213
MutableArrayRef< Region > getRegions()
Returns the regions held by this operation.
Definition: Operation.h:655
MutableArrayRef< OpOperand > getOpOperands()
Definition: Operation.h:378
result_range getResults()
Definition: Operation.h:410
InFlightDiagnostic emitOpError(const Twine &message={})
Emit an error with the op name prefixed, like "'dim' op " which is convenient for verifiers.
Definition: Operation.cpp:640
unsigned getNumResults()
Return the number of results held by this operation.
Definition: Operation.h:399
MLIRContext * getContext() const
RewritePatternSet & add(ConstructorArg &&arg, ConstructorArgs &&...args)
Add an instance of each of the pattern types 'Ts' to the pattern list with the given arguments.
This class coordinates the application of a rewrite on a set of IR, providing a way for clients to tr...
Definition: PatternMatch.h:399
virtual void replaceOp(Operation *op, ValueRange newValues)
This method replaces the results of the operation with the specified list of values.
virtual void eraseOp(Operation *op)
This method erases an operation that is known to have no uses.
This class represents an instance of an SSA value in the MLIR system, representing a computable value...
Definition: Value.h:96
Operation * getDefiningOp() const
If this value is the result of an operation, return the operation that defines it.
Definition: Value.cpp:20
constexpr void enumerate(std::tuple< Tys... > &tuple, CallbackT &&callback)
Definition: Matchers.h:285
void populateSCFLoopPipeliningPatterns(RewritePatternSet &patterns, const PipeliningOption &options)
Populate patterns for SCF software pipelining transformation.
FailureOr< ForOp > pipelineForLoop(RewriterBase &rewriter, ForOp forOp, const PipeliningOption &options, bool *modifiedIR=nullptr)
Generate a pipelined version of the scf.for loop based on the schedule given as option.
Include the generated interface declarations.
LogicalResult failure(bool isFailure=true)
Utility function to generate a LogicalResult.
Definition: LogicalResult.h:62
int64_t ceilDiv(int64_t lhs, int64_t rhs)
Returns the result of MLIR's ceildiv operation on constants.
Definition: MathExtras.h:23
LogicalResult success(bool isSuccess=true)
Utility function to generate a LogicalResult.
Definition: LogicalResult.h:56
Operation * clone(OpBuilder &b, Operation *op, TypeRange newResultTypes, ValueRange newOperands)
void visitUsedValuesDefinedAbove(Region &region, Region &limit, function_ref< void(OpOperand *)> callback)
Calls callback for each use of a value within region or its descendants that was defined at the ances...
Definition: RegionUtils.cpp:36
bool failed(LogicalResult result)
Utility function that returns true if the provided LogicalResult corresponds to a failure value.
Definition: LogicalResult.h:72
This class represents an efficient way to signal success or failure.
Definition: LogicalResult.h:26
Options to dictate how loops should be pipelined.
Definition: Transforms.h:104
std::function< void(Operation *, PipelinerPart, unsigned)> AnnotationlFnType
Lambda called by the pipeliner to allow the user to annotate the IR while it is generated.
Definition: Transforms.h:122
std::function< Operation *(RewriterBase &, Operation *, Value)> PredicateOpFn
Definition: Transforms.h:138