MLIR  16.0.0git
AffineOps.cpp
Go to the documentation of this file.
1 //===- AffineOps.cpp - MLIR Affine Operations -----------------------------===//
2 //
3 // Part of the LLVM Project, under the Apache License v2.0 with LLVM Exceptions.
4 // See https://llvm.org/LICENSE.txt for license information.
5 // SPDX-License-Identifier: Apache-2.0 WITH LLVM-exception
6 //
7 //===----------------------------------------------------------------------===//
8 
14 #include "mlir/IR/IntegerSet.h"
15 #include "mlir/IR/Matchers.h"
16 #include "mlir/IR/OpDefinition.h"
17 #include "mlir/IR/PatternMatch.h"
20 #include "llvm/ADT/ScopeExit.h"
21 #include "llvm/ADT/SmallBitVector.h"
22 #include "llvm/ADT/TypeSwitch.h"
23 #include "llvm/Support/Debug.h"
24 #include <numeric>
25 
26 using namespace mlir;
27 
28 #define DEBUG_TYPE "affine-analysis"
29 
30 #include "mlir/Dialect/Affine/IR/AffineOpsDialect.cpp.inc"
31 
32 /// A utility function to check if a value is defined at the top level of
33 /// `region` or is an argument of `region`. A value of index type defined at the
34 /// top level of a `AffineScope` region is always a valid symbol for all
35 /// uses in that region.
37  if (auto arg = value.dyn_cast<BlockArgument>())
38  return arg.getParentRegion() == region;
39  return value.getDefiningOp()->getParentRegion() == region;
40 }
41 
42 /// Checks if `value` known to be a legal affine dimension or symbol in `src`
43 /// region remains legal if the operation that uses it is inlined into `dest`
44 /// with the given value mapping. `legalityCheck` is either `isValidDim` or
45 /// `isValidSymbol`, depending on the value being required to remain a valid
46 /// dimension or symbol.
47 static bool
49  const BlockAndValueMapping &mapping,
50  function_ref<bool(Value, Region *)> legalityCheck) {
51  // If the value is a valid dimension for any other reason than being
52  // a top-level value, it will remain valid: constants get inlined
53  // with the function, transitive affine applies also get inlined and
54  // will be checked themselves, etc.
55  if (!isTopLevelValue(value, src))
56  return true;
57 
58  // If it's a top-level value because it's a block operand, i.e. a
59  // function argument, check whether the value replacing it after
60  // inlining is a valid dimension in the new region.
61  if (value.isa<BlockArgument>())
62  return legalityCheck(mapping.lookup(value), dest);
63 
64  // If it's a top-level value because it's defined in the region,
65  // it can only be inlined if the defining op is a constant or a
66  // `dim`, which can appear anywhere and be valid, since the defining
67  // op won't be top-level anymore after inlining.
68  Attribute operandCst;
69  bool isDimLikeOp = isa<ShapedDimOpInterface>(value.getDefiningOp());
70  return matchPattern(value.getDefiningOp(), m_Constant(&operandCst)) ||
71  isDimLikeOp;
72 }
73 
74 /// Checks if all values known to be legal affine dimensions or symbols in `src`
75 /// remain so if their respective users are inlined into `dest`.
76 static bool
78  const BlockAndValueMapping &mapping,
79  function_ref<bool(Value, Region *)> legalityCheck) {
80  return llvm::all_of(values, [&](Value v) {
81  return remainsLegalAfterInline(v, src, dest, mapping, legalityCheck);
82  });
83 }
84 
85 /// Checks if an affine read or write operation remains legal after inlining
86 /// from `src` to `dest`.
87 template <typename OpTy>
88 static bool remainsLegalAfterInline(OpTy op, Region *src, Region *dest,
89  const BlockAndValueMapping &mapping) {
90  static_assert(llvm::is_one_of<OpTy, AffineReadOpInterface,
91  AffineWriteOpInterface>::value,
92  "only ops with affine read/write interface are supported");
93 
94  AffineMap map = op.getAffineMap();
95  ValueRange dimOperands = op.getMapOperands().take_front(map.getNumDims());
96  ValueRange symbolOperands =
97  op.getMapOperands().take_back(map.getNumSymbols());
99  dimOperands, src, dest, mapping,
100  static_cast<bool (*)(Value, Region *)>(isValidDim)))
101  return false;
103  symbolOperands, src, dest, mapping,
104  static_cast<bool (*)(Value, Region *)>(isValidSymbol)))
105  return false;
106  return true;
107 }
108 
109 /// Checks if an affine apply operation remains legal after inlining from `src`
110 /// to `dest`.
111 // Use "unused attribute" marker to silence clang-tidy warning stemming from
112 // the inability to see through "llvm::TypeSwitch".
113 template <>
114 bool LLVM_ATTRIBUTE_UNUSED
115 remainsLegalAfterInline(AffineApplyOp op, Region *src, Region *dest,
116  const BlockAndValueMapping &mapping) {
117  // If it's a valid dimension, we need to check that it remains so.
118  if (isValidDim(op.getResult(), src))
120  op.getMapOperands(), src, dest, mapping,
121  static_cast<bool (*)(Value, Region *)>(isValidDim));
122 
123  // Otherwise it must be a valid symbol, check that it remains so.
125  op.getMapOperands(), src, dest, mapping,
126  static_cast<bool (*)(Value, Region *)>(isValidSymbol));
127 }
128 
129 //===----------------------------------------------------------------------===//
130 // AffineDialect Interfaces
131 //===----------------------------------------------------------------------===//
132 
133 namespace {
134 /// This class defines the interface for handling inlining with affine
135 /// operations.
136 struct AffineInlinerInterface : public DialectInlinerInterface {
138 
139  //===--------------------------------------------------------------------===//
140  // Analysis Hooks
141  //===--------------------------------------------------------------------===//
142 
143  /// Returns true if the given region 'src' can be inlined into the region
144  /// 'dest' that is attached to an operation registered to the current dialect.
145  /// 'wouldBeCloned' is set if the region is cloned into its new location
146  /// rather than moved, indicating there may be other users.
147  bool isLegalToInline(Region *dest, Region *src, bool wouldBeCloned,
148  BlockAndValueMapping &valueMapping) const final {
149  // We can inline into affine loops and conditionals if this doesn't break
150  // affine value categorization rules.
151  Operation *destOp = dest->getParentOp();
152  if (!isa<AffineParallelOp, AffineForOp, AffineIfOp>(destOp))
153  return false;
154 
155  // Multi-block regions cannot be inlined into affine constructs, all of
156  // which require single-block regions.
157  if (!llvm::hasSingleElement(*src))
158  return false;
159 
160  // Side-effecting operations that the affine dialect cannot understand
161  // should not be inlined.
162  Block &srcBlock = src->front();
163  for (Operation &op : srcBlock) {
164  // Ops with no side effects are fine,
165  if (auto iface = dyn_cast<MemoryEffectOpInterface>(op)) {
166  if (iface.hasNoEffect())
167  continue;
168  }
169 
170  // Assuming the inlined region is valid, we only need to check if the
171  // inlining would change it.
172  bool remainsValid =
174  .Case<AffineApplyOp, AffineReadOpInterface,
175  AffineWriteOpInterface>([&](auto op) {
176  return remainsLegalAfterInline(op, src, dest, valueMapping);
177  })
178  .Default([](Operation *) {
179  // Conservatively disallow inlining ops we cannot reason about.
180  return false;
181  });
182 
183  if (!remainsValid)
184  return false;
185  }
186 
187  return true;
188  }
189 
190  /// Returns true if the given operation 'op', that is registered to this
191  /// dialect, can be inlined into the given region, false otherwise.
192  bool isLegalToInline(Operation *op, Region *region, bool wouldBeCloned,
193  BlockAndValueMapping &valueMapping) const final {
194  // Always allow inlining affine operations into a region that is marked as
195  // affine scope, or into affine loops and conditionals. There are some edge
196  // cases when inlining *into* affine structures, but that is handled in the
197  // other 'isLegalToInline' hook above.
198  Operation *parentOp = region->getParentOp();
199  return parentOp->hasTrait<OpTrait::AffineScope>() ||
200  isa<AffineForOp, AffineParallelOp, AffineIfOp>(parentOp);
201  }
202 
203  /// Affine regions should be analyzed recursively.
204  bool shouldAnalyzeRecursively(Operation *op) const final { return true; }
205 };
206 } // namespace
207 
208 //===----------------------------------------------------------------------===//
209 // AffineDialect
210 //===----------------------------------------------------------------------===//
211 
212 void AffineDialect::initialize() {
213  addOperations<AffineDmaStartOp, AffineDmaWaitOp,
214 #define GET_OP_LIST
215 #include "mlir/Dialect/Affine/IR/AffineOps.cpp.inc"
216  >();
217  addInterfaces<AffineInlinerInterface>();
218 }
219 
220 /// Materialize a single constant operation from a given attribute value with
221 /// the desired resultant type.
223  Attribute value, Type type,
224  Location loc) {
225  return builder.create<arith::ConstantOp>(loc, type, value);
226 }
227 
228 /// A utility function to check if a value is defined at the top level of an
229 /// op with trait `AffineScope`. If the value is defined in an unlinked region,
230 /// conservatively assume it is not top-level. A value of index type defined at
231 /// the top level is always a valid symbol.
233  if (auto arg = value.dyn_cast<BlockArgument>()) {
234  // The block owning the argument may be unlinked, e.g. when the surrounding
235  // region has not yet been attached to an Op, at which point the parent Op
236  // is null.
237  Operation *parentOp = arg.getOwner()->getParentOp();
238  return parentOp && parentOp->hasTrait<OpTrait::AffineScope>();
239  }
240  // The defining Op may live in an unlinked block so its parent Op may be null.
241  Operation *parentOp = value.getDefiningOp()->getParentOp();
242  return parentOp && parentOp->hasTrait<OpTrait::AffineScope>();
243 }
244 
245 /// Returns the closest region enclosing `op` that is held by an operation with
246 /// trait `AffineScope`; `nullptr` if there is no such region.
248  auto *curOp = op;
249  while (auto *parentOp = curOp->getParentOp()) {
250  if (parentOp->hasTrait<OpTrait::AffineScope>())
251  return curOp->getParentRegion();
252  curOp = parentOp;
253  }
254  return nullptr;
255 }
256 
257 // A Value can be used as a dimension id iff it meets one of the following
258 // conditions:
259 // *) It is valid as a symbol.
260 // *) It is an induction variable.
261 // *) It is the result of affine apply operation with dimension id arguments.
263  // The value must be an index type.
264  if (!value.getType().isIndex())
265  return false;
266 
267  if (auto *defOp = value.getDefiningOp())
268  return isValidDim(value, getAffineScope(defOp));
269 
270  // This value has to be a block argument for an op that has the
271  // `AffineScope` trait or for an affine.for or affine.parallel.
272  auto *parentOp = value.cast<BlockArgument>().getOwner()->getParentOp();
273  return parentOp && (parentOp->hasTrait<OpTrait::AffineScope>() ||
274  isa<AffineForOp, AffineParallelOp>(parentOp));
275 }
276 
277 // Value can be used as a dimension id iff it meets one of the following
278 // conditions:
279 // *) It is valid as a symbol.
280 // *) It is an induction variable.
281 // *) It is the result of an affine apply operation with dimension id operands.
283  // The value must be an index type.
284  if (!value.getType().isIndex())
285  return false;
286 
287  // All valid symbols are okay.
288  if (isValidSymbol(value, region))
289  return true;
290 
291  auto *op = value.getDefiningOp();
292  if (!op) {
293  // This value has to be a block argument for an affine.for or an
294  // affine.parallel.
295  auto *parentOp = value.cast<BlockArgument>().getOwner()->getParentOp();
296  return isa<AffineForOp, AffineParallelOp>(parentOp);
297  }
298 
299  // Affine apply operation is ok if all of its operands are ok.
300  if (auto applyOp = dyn_cast<AffineApplyOp>(op))
301  return applyOp.isValidDim(region);
302  // The dim op is okay if its operand memref/tensor is defined at the top
303  // level.
304  if (auto dimOp = dyn_cast<ShapedDimOpInterface>(op))
305  return isTopLevelValue(dimOp.getShapedValue());
306  return false;
307 }
308 
309 /// Returns true if the 'index' dimension of the `memref` defined by
310 /// `memrefDefOp` is a statically shaped one or defined using a valid symbol
311 /// for `region`.
312 template <typename AnyMemRefDefOp>
313 static bool isMemRefSizeValidSymbol(AnyMemRefDefOp memrefDefOp, unsigned index,
314  Region *region) {
315  auto memRefType = memrefDefOp.getType();
316  // Statically shaped.
317  if (!memRefType.isDynamicDim(index))
318  return true;
319  // Get the position of the dimension among dynamic dimensions;
320  unsigned dynamicDimPos = memRefType.getDynamicDimIndex(index);
321  return isValidSymbol(*(memrefDefOp.getDynamicSizes().begin() + dynamicDimPos),
322  region);
323 }
324 
325 /// Returns true if the result of the dim op is a valid symbol for `region`.
326 static bool isDimOpValidSymbol(ShapedDimOpInterface dimOp, Region *region) {
327  // The dim op is okay if its source is defined at the top level.
328  if (isTopLevelValue(dimOp.getShapedValue()))
329  return true;
330 
331  // Conservatively handle remaining BlockArguments as non-valid symbols.
332  // E.g. scf.for iterArgs.
333  if (dimOp.getShapedValue().template isa<BlockArgument>())
334  return false;
335 
336  // The dim op is also okay if its operand memref is a view/subview whose
337  // corresponding size is a valid symbol.
338  Optional<int64_t> index = getConstantIntValue(dimOp.getDimension());
339  assert(index.has_value() &&
340  "expect only `dim` operations with a constant index");
341  int64_t i = index.value();
342  return TypeSwitch<Operation *, bool>(dimOp.getShapedValue().getDefiningOp())
343  .Case<memref::ViewOp, memref::SubViewOp, memref::AllocOp>(
344  [&](auto op) { return isMemRefSizeValidSymbol(op, i, region); })
345  .Default([](Operation *) { return false; });
346 }
347 
348 // A value can be used as a symbol (at all its use sites) iff it meets one of
349 // the following conditions:
350 // *) It is a constant.
351 // *) Its defining op or block arg appearance is immediately enclosed by an op
352 // with `AffineScope` trait.
353 // *) It is the result of an affine.apply operation with symbol operands.
354 // *) It is a result of the dim op on a memref whose corresponding size is a
355 // valid symbol.
357  if (!value)
358  return false;
359 
360  // The value must be an index type.
361  if (!value.getType().isIndex())
362  return false;
363 
364  // Check that the value is a top level value.
365  if (isTopLevelValue(value))
366  return true;
367 
368  if (auto *defOp = value.getDefiningOp())
369  return isValidSymbol(value, getAffineScope(defOp));
370 
371  return false;
372 }
373 
374 /// A value can be used as a symbol for `region` iff it meets one of the
375 /// following conditions:
376 /// *) It is a constant.
377 /// *) It is the result of an affine apply operation with symbol arguments.
378 /// *) It is a result of the dim op on a memref whose corresponding size is
379 /// a valid symbol.
380 /// *) It is defined at the top level of 'region' or is its argument.
381 /// *) It dominates `region`'s parent op.
382 /// If `region` is null, conservatively assume the symbol definition scope does
383 /// not exist and only accept the values that would be symbols regardless of
384 /// the surrounding region structure, i.e. the first three cases above.
386  // The value must be an index type.
387  if (!value.getType().isIndex())
388  return false;
389 
390  // A top-level value is a valid symbol.
391  if (region && ::isTopLevelValue(value, region))
392  return true;
393 
394  auto *defOp = value.getDefiningOp();
395  if (!defOp) {
396  // A block argument that is not a top-level value is a valid symbol if it
397  // dominates region's parent op.
398  Operation *regionOp = region ? region->getParentOp() : nullptr;
399  if (regionOp && !regionOp->hasTrait<OpTrait::IsIsolatedFromAbove>())
400  if (auto *parentOpRegion = region->getParentOp()->getParentRegion())
401  return isValidSymbol(value, parentOpRegion);
402  return false;
403  }
404 
405  // Constant operation is ok.
406  Attribute operandCst;
407  if (matchPattern(defOp, m_Constant(&operandCst)))
408  return true;
409 
410  // Affine apply operation is ok if all of its operands are ok.
411  if (auto applyOp = dyn_cast<AffineApplyOp>(defOp))
412  return applyOp.isValidSymbol(region);
413 
414  // Dim op results could be valid symbols at any level.
415  if (auto dimOp = dyn_cast<ShapedDimOpInterface>(defOp))
416  return isDimOpValidSymbol(dimOp, region);
417 
418  // Check for values dominating `region`'s parent op.
419  Operation *regionOp = region ? region->getParentOp() : nullptr;
420  if (regionOp && !regionOp->hasTrait<OpTrait::IsIsolatedFromAbove>())
421  if (auto *parentRegion = region->getParentOp()->getParentRegion())
422  return isValidSymbol(value, parentRegion);
423 
424  return false;
425 }
426 
427 // Returns true if 'value' is a valid index to an affine operation (e.g.
428 // affine.load, affine.store, affine.dma_start, affine.dma_wait) where
429 // `region` provides the polyhedral symbol scope. Returns false otherwise.
431  return isValidDim(value, region) || isValidSymbol(value, region);
432 }
433 
434 /// Prints dimension and symbol list.
437  unsigned numDims, OpAsmPrinter &printer) {
438  OperandRange operands(begin, end);
439  printer << '(' << operands.take_front(numDims) << ')';
440  if (operands.size() > numDims)
441  printer << '[' << operands.drop_front(numDims) << ']';
442 }
443 
444 /// Parses dimension and symbol list and returns true if parsing failed.
446  SmallVectorImpl<Value> &operands,
447  unsigned &numDims) {
449  if (parser.parseOperandList(opInfos, OpAsmParser::Delimiter::Paren))
450  return failure();
451  // Store number of dimensions for validation by caller.
452  numDims = opInfos.size();
453 
454  // Parse the optional symbol operands.
455  auto indexTy = parser.getBuilder().getIndexType();
456  return failure(parser.parseOperandList(
458  parser.resolveOperands(opInfos, indexTy, operands));
459 }
460 
461 /// Utility function to verify that a set of operands are valid dimension and
462 /// symbol identifiers. The operands should be laid out such that the dimension
463 /// operands are before the symbol operands. This function returns failure if
464 /// there was an invalid operand. An operation is provided to emit any necessary
465 /// errors.
466 template <typename OpTy>
467 static LogicalResult
469  unsigned numDims) {
470  unsigned opIt = 0;
471  for (auto operand : operands) {
472  if (opIt++ < numDims) {
473  if (!isValidDim(operand, getAffineScope(op)))
474  return op.emitOpError("operand cannot be used as a dimension id");
475  } else if (!isValidSymbol(operand, getAffineScope(op))) {
476  return op.emitOpError("operand cannot be used as a symbol");
477  }
478  }
479  return success();
480 }
481 
482 //===----------------------------------------------------------------------===//
483 // AffineApplyOp
484 //===----------------------------------------------------------------------===//
485 
486 AffineValueMap AffineApplyOp::getAffineValueMap() {
487  return AffineValueMap(getAffineMap(), getOperands(), getResult());
488 }
489 
490 ParseResult AffineApplyOp::parse(OpAsmParser &parser, OperationState &result) {
491  auto &builder = parser.getBuilder();
492  auto indexTy = builder.getIndexType();
493 
494  AffineMapAttr mapAttr;
495  unsigned numDims;
496  if (parser.parseAttribute(mapAttr, "map", result.attributes) ||
497  parseDimAndSymbolList(parser, result.operands, numDims) ||
498  parser.parseOptionalAttrDict(result.attributes))
499  return failure();
500  auto map = mapAttr.getValue();
501 
502  if (map.getNumDims() != numDims ||
503  numDims + map.getNumSymbols() != result.operands.size()) {
504  return parser.emitError(parser.getNameLoc(),
505  "dimension or symbol index mismatch");
506  }
507 
508  result.types.append(map.getNumResults(), indexTy);
509  return success();
510 }
511 
513  p << " " << getMapAttr();
514  printDimAndSymbolList(operand_begin(), operand_end(),
515  getAffineMap().getNumDims(), p);
516  p.printOptionalAttrDict((*this)->getAttrs(), /*elidedAttrs=*/{"map"});
517 }
518 
520  // Check input and output dimensions match.
521  AffineMap affineMap = getMap();
522 
523  // Verify that operand count matches affine map dimension and symbol count.
524  if (getNumOperands() != affineMap.getNumDims() + affineMap.getNumSymbols())
525  return emitOpError(
526  "operand count and affine map dimension and symbol count must match");
527 
528  // Verify that the map only produces one result.
529  if (affineMap.getNumResults() != 1)
530  return emitOpError("mapping must produce one value");
531 
532  return success();
533 }
534 
535 // The result of the affine apply operation can be used as a dimension id if all
536 // its operands are valid dimension ids.
538  return llvm::all_of(getOperands(),
539  [](Value op) { return mlir::isValidDim(op); });
540 }
541 
542 // The result of the affine apply operation can be used as a dimension id if all
543 // its operands are valid dimension ids with the parent operation of `region`
544 // defining the polyhedral scope for symbols.
545 bool AffineApplyOp::isValidDim(Region *region) {
546  return llvm::all_of(getOperands(),
547  [&](Value op) { return ::isValidDim(op, region); });
548 }
549 
550 // The result of the affine apply operation can be used as a symbol if all its
551 // operands are symbols.
553  return llvm::all_of(getOperands(),
554  [](Value op) { return mlir::isValidSymbol(op); });
555 }
556 
557 // The result of the affine apply operation can be used as a symbol in `region`
558 // if all its operands are symbols in `region`.
559 bool AffineApplyOp::isValidSymbol(Region *region) {
560  return llvm::all_of(getOperands(), [&](Value operand) {
561  return mlir::isValidSymbol(operand, region);
562  });
563 }
564 
565 OpFoldResult AffineApplyOp::fold(ArrayRef<Attribute> operands) {
566  auto map = getAffineMap();
567 
568  // Fold dims and symbols to existing values.
569  auto expr = map.getResult(0);
570  if (auto dim = expr.dyn_cast<AffineDimExpr>())
571  return getOperand(dim.getPosition());
572  if (auto sym = expr.dyn_cast<AffineSymbolExpr>())
573  return getOperand(map.getNumDims() + sym.getPosition());
574 
575  // Otherwise, default to folding the map.
577  if (failed(map.constantFold(operands, result)))
578  return {};
579  return result[0];
580 }
581 
582 /// Returns the largest known divisor of `e`. Exploits information from the
583 /// values in `operands`.
584 static int64_t getLargestKnownDivisor(AffineExpr e, ArrayRef<Value> operands) {
585  // This method isn't aware of `operands`.
586  int64_t div = e.getLargestKnownDivisor();
587 
588  // We now make use of operands for the case `e` is a dim expression.
589  // TODO: More powerful simplification would have to modify
590  // getLargestKnownDivisor to take `operands` and exploit that information as
591  // well for dim/sym expressions, but in that case, getLargestKnownDivisor
592  // can't be part of the IR library but of the `Analysis` library. The IR
593  // library can only really depend on simple O(1) checks.
594  auto dimExpr = e.dyn_cast<AffineDimExpr>();
595  // If it's not a dim expr, `div` is the best we have.
596  if (!dimExpr)
597  return div;
598 
599  // We simply exploit information from loop IVs.
600  // We don't need to use mlir::getLargestKnownDivisorOfValue since the other
601  // desired simplifications are expected to be part of other
602  // canonicalizations. Also, mlir::getLargestKnownDivisorOfValue is part of the
603  // LoopAnalysis library.
604  Value operand = operands[dimExpr.getPosition()];
605  int64_t operandDivisor = 1;
606  // TODO: With the right accessors, this can be extended to
607  // LoopLikeOpInterface.
608  if (AffineForOp forOp = getForInductionVarOwner(operand)) {
609  if (forOp.hasConstantLowerBound() && forOp.getConstantLowerBound() == 0) {
610  operandDivisor = forOp.getStep();
611  } else {
612  uint64_t lbLargestKnownDivisor =
613  forOp.getLowerBoundMap().getLargestKnownDivisorOfMapExprs();
614  operandDivisor = std::gcd(lbLargestKnownDivisor, forOp.getStep());
615  }
616  }
617  return operandDivisor;
618 }
619 
620 /// Check if `e` is known to be: 0 <= `e` < `k`. Handles the simple cases of `e`
621 /// being an affine dim expression or a constant.
623  int64_t k) {
624  if (auto constExpr = e.dyn_cast<AffineConstantExpr>()) {
625  int64_t constVal = constExpr.getValue();
626  return constVal >= 0 && constVal < k;
627  }
628  auto dimExpr = e.dyn_cast<AffineDimExpr>();
629  if (!dimExpr)
630  return false;
631  Value operand = operands[dimExpr.getPosition()];
632  // TODO: With the right accessors, this can be extended to
633  // LoopLikeOpInterface.
634  if (AffineForOp forOp = getForInductionVarOwner(operand)) {
635  if (forOp.hasConstantLowerBound() && forOp.getConstantLowerBound() >= 0 &&
636  forOp.hasConstantUpperBound() && forOp.getConstantUpperBound() <= k) {
637  return true;
638  }
639  }
640 
641  // We don't consider other cases like `operand` being defined by a constant or
642  // an affine.apply op since such cases will already be handled by other
643  // patterns and propagation of loop IVs or constant would happen.
644  return false;
645 }
646 
647 /// Check if expression `e` is of the form d*e_1 + e_2 where 0 <= e_2 < d.
648 /// Set `div` to `d`, `quotientTimesDiv` to e_1 and `rem` to e_2 if the
649 /// expression is in that form.
650 static bool isQTimesDPlusR(AffineExpr e, ArrayRef<Value> operands, int64_t &div,
651  AffineExpr &quotientTimesDiv, AffineExpr &rem) {
652  auto bin = e.dyn_cast<AffineBinaryOpExpr>();
653  if (!bin || bin.getKind() != AffineExprKind::Add)
654  return false;
655 
656  AffineExpr llhs = bin.getLHS();
657  AffineExpr rlhs = bin.getRHS();
658  div = getLargestKnownDivisor(llhs, operands);
659  if (isNonNegativeBoundedBy(rlhs, operands, div)) {
660  quotientTimesDiv = llhs;
661  rem = rlhs;
662  return true;
663  }
664  div = getLargestKnownDivisor(rlhs, operands);
665  if (isNonNegativeBoundedBy(llhs, operands, div)) {
666  quotientTimesDiv = rlhs;
667  rem = llhs;
668  return true;
669  }
670  return false;
671 }
672 
673 /// Simplify `expr` while exploiting information from the values in `operands`.
675  ArrayRef<Value> operands) {
676  // We do this only for certain floordiv/mod expressions.
677  auto binExpr = expr.dyn_cast<AffineBinaryOpExpr>();
678  if (!binExpr)
679  return;
680 
681  // Simplify the child expressions first.
682  AffineExpr lhs = binExpr.getLHS();
683  AffineExpr rhs = binExpr.getRHS();
684  simplifyExprAndOperands(lhs, operands);
685  simplifyExprAndOperands(rhs, operands);
686  expr = getAffineBinaryOpExpr(binExpr.getKind(), lhs, rhs);
687 
688  binExpr = expr.dyn_cast<AffineBinaryOpExpr>();
689  if (!binExpr || (binExpr.getKind() != AffineExprKind::FloorDiv &&
690  binExpr.getKind() != AffineExprKind::Mod)) {
691  return;
692  }
693 
694  // The `lhs` and `rhs` may be different post construction of simplified expr.
695  lhs = binExpr.getLHS();
696  rhs = binExpr.getRHS();
697  auto rhsConst = rhs.dyn_cast<AffineConstantExpr>();
698  if (!rhsConst)
699  return;
700 
701  int64_t rhsConstVal = rhsConst.getValue();
702  // Undefined exprsessions aren't touched; IR can still be valid with them.
703  if (rhsConstVal == 0)
704  return;
705 
706  AffineExpr quotientTimesDiv, rem;
707  int64_t divisor;
708 
709  // Simplify expressions of the form e = (e_1 + e_2) floordiv c or (e_1 + e_2)
710  // mod c, where e_1 is a multiple of `k` and 0 <= e_2 < k. In such cases, if
711  // `c` % `k` == 0, (e_1 + e_2) floordiv c can be simplified to e_1 floordiv c.
712  // And when k % c == 0, (e_1 + e_2) mod c can be simplified to e_2 mod c.
713  if (isQTimesDPlusR(lhs, operands, divisor, quotientTimesDiv, rem)) {
714  if (rhsConstVal % divisor == 0 &&
715  binExpr.getKind() == AffineExprKind::FloorDiv) {
716  expr = quotientTimesDiv.floorDiv(rhsConst);
717  } else if (divisor % rhsConstVal == 0 &&
718  binExpr.getKind() == AffineExprKind::Mod) {
719  expr = rem % rhsConst;
720  }
721  return;
722  }
723 
724  // Handle the simple case when the LHS expression can be either upper
725  // bounded or is a known multiple of RHS constant.
726  // lhs floordiv c -> 0 if 0 <= lhs < c,
727  // lhs mod c -> 0 if lhs % c = 0.
728  if ((isNonNegativeBoundedBy(lhs, operands, rhsConstVal) &&
729  binExpr.getKind() == AffineExprKind::FloorDiv) ||
730  (getLargestKnownDivisor(lhs, operands) % rhsConstVal == 0 &&
731  binExpr.getKind() == AffineExprKind::Mod)) {
732  expr = getAffineConstantExpr(0, expr.getContext());
733  }
734 }
735 
736 /// Simplify the map while exploiting information on the values in `operands`.
737 // Use "unused attribute" marker to silence warning stemming from the inability
738 // to see through the template expansion.
739 static void LLVM_ATTRIBUTE_UNUSED
741  assert(map.getNumInputs() == operands.size() && "invalid operands for map");
742  SmallVector<AffineExpr> newResults;
743  newResults.reserve(map.getNumResults());
744  for (AffineExpr expr : map.getResults()) {
745  simplifyExprAndOperands(expr, operands);
746  newResults.push_back(expr);
747  }
748  map = AffineMap::get(map.getNumDims(), map.getNumSymbols(), newResults,
749  map.getContext());
750 }
751 
752 /// Replace all occurrences of AffineExpr at position `pos` in `map` by the
753 /// defining AffineApplyOp expression and operands.
754 /// When `dimOrSymbolPosition < dims.size()`, AffineDimExpr@[pos] is replaced.
755 /// When `dimOrSymbolPosition >= dims.size()`,
756 /// AffineSymbolExpr@[pos - dims.size()] is replaced.
757 /// Mutate `map`,`dims` and `syms` in place as follows:
758 /// 1. `dims` and `syms` are only appended to.
759 /// 2. `map` dim and symbols are gradually shifted to higher positions.
760 /// 3. Old `dim` and `sym` entries are replaced by nullptr
761 /// This avoids the need for any bookkeeping.
763  unsigned dimOrSymbolPosition,
765  SmallVectorImpl<Value> &syms) {
766  bool isDimReplacement = (dimOrSymbolPosition < dims.size());
767  unsigned pos = isDimReplacement ? dimOrSymbolPosition
768  : dimOrSymbolPosition - dims.size();
769  Value &v = isDimReplacement ? dims[pos] : syms[pos];
770  if (!v)
771  return failure();
772 
773  auto affineApply = v.getDefiningOp<AffineApplyOp>();
774  if (!affineApply)
775  return failure();
776 
777  // At this point we will perform a replacement of `v`, set the entry in `dim`
778  // or `sym` to nullptr immediately.
779  v = nullptr;
780 
781  // Compute the map, dims and symbols coming from the AffineApplyOp.
782  AffineMap composeMap = affineApply.getAffineMap();
783  assert(composeMap.getNumResults() == 1 && "affine.apply with >1 results");
784  AffineExpr composeExpr =
785  composeMap.shiftDims(dims.size()).shiftSymbols(syms.size()).getResult(0);
786  ValueRange composeDims =
787  affineApply.getMapOperands().take_front(composeMap.getNumDims());
788  ValueRange composeSyms =
789  affineApply.getMapOperands().take_back(composeMap.getNumSymbols());
790 
791  // Append the dims and symbols where relevant and perform the replacement.
792  MLIRContext *ctx = map->getContext();
793  AffineExpr toReplace = isDimReplacement ? getAffineDimExpr(pos, ctx)
794  : getAffineSymbolExpr(pos, ctx);
795  dims.append(composeDims.begin(), composeDims.end());
796  syms.append(composeSyms.begin(), composeSyms.end());
797  *map = map->replace(toReplace, composeExpr, dims.size(), syms.size());
798 
799  return success();
800 }
801 
802 /// Iterate over `operands` and fold away all those produced by an AffineApplyOp
803 /// iteratively. Perform canonicalization of map and operands as well as
804 /// AffineMap simplification. `map` and `operands` are mutated in place.
806  SmallVectorImpl<Value> *operands) {
807  if (map->getNumResults() == 0) {
808  canonicalizeMapAndOperands(map, operands);
809  *map = simplifyAffineMap(*map);
810  return;
811  }
812 
813  MLIRContext *ctx = map->getContext();
814  SmallVector<Value, 4> dims(operands->begin(),
815  operands->begin() + map->getNumDims());
816  SmallVector<Value, 4> syms(operands->begin() + map->getNumDims(),
817  operands->end());
818 
819  // Iterate over dims and symbols coming from AffineApplyOp and replace until
820  // exhaustion. This iteratively mutates `map`, `dims` and `syms`. Both `dims`
821  // and `syms` can only increase by construction.
822  // The implementation uses a `while` loop to support the case of symbols
823  // that may be constructed from dims ;this may be overkill.
824  while (true) {
825  bool changed = false;
826  for (unsigned pos = 0; pos != dims.size() + syms.size(); ++pos)
827  if ((changed |= succeeded(replaceDimOrSym(map, pos, dims, syms))))
828  break;
829  if (!changed)
830  break;
831  }
832 
833  // Clear operands so we can fill them anew.
834  operands->clear();
835 
836  // At this point we may have introduced null operands, prune them out before
837  // canonicalizing map and operands.
838  unsigned nDims = 0, nSyms = 0;
839  SmallVector<AffineExpr, 4> dimReplacements, symReplacements;
840  dimReplacements.reserve(dims.size());
841  symReplacements.reserve(syms.size());
842  for (auto *container : {&dims, &syms}) {
843  bool isDim = (container == &dims);
844  auto &repls = isDim ? dimReplacements : symReplacements;
845  for (const auto &en : llvm::enumerate(*container)) {
846  Value v = en.value();
847  if (!v) {
848  assert(isDim ? !map->isFunctionOfDim(en.index())
849  : !map->isFunctionOfSymbol(en.index()) &&
850  "map is function of unexpected expr@pos");
851  repls.push_back(getAffineConstantExpr(0, ctx));
852  continue;
853  }
854  repls.push_back(isDim ? getAffineDimExpr(nDims++, ctx)
855  : getAffineSymbolExpr(nSyms++, ctx));
856  operands->push_back(v);
857  }
858  }
859  *map = map->replaceDimsAndSymbols(dimReplacements, symReplacements, nDims,
860  nSyms);
861 
862  // Canonicalize and simplify before returning.
863  canonicalizeMapAndOperands(map, operands);
864  *map = simplifyAffineMap(*map);
865 }
866 
868  SmallVectorImpl<Value> *operands) {
869  while (llvm::any_of(*operands, [](Value v) {
870  return isa_and_nonnull<AffineApplyOp>(v.getDefiningOp());
871  })) {
872  composeAffineMapAndOperands(map, operands);
873  }
874 }
875 
876 /// Given a list of `OpFoldResult`, build the necessary operations to populate
877 /// `actualValues` with values produced by operations. In particular, for any
878 /// attribute-typed element in `values`, call the constant materializer
879 /// associated with the Affine dialect to produce an operation. Do NOT notify
880 /// the builder listener about the constant ops being created as they are
881 /// intended to be removed after being folded into affine constructs; this is
882 /// not suitable for use beyond the Affine dialect.
884  ArrayRef<OpFoldResult> values,
885  SmallVectorImpl<Operation *> &constants,
886  SmallVectorImpl<Value> &actualValues) {
887  OpBuilder::Listener *listener = b.getListener();
888  b.setListener(nullptr);
889  auto listenerResetter =
890  llvm::make_scope_exit([listener, &b] { b.setListener(listener); });
891 
892  actualValues.reserve(values.size());
893  auto *dialect = b.getContext()->getLoadedDialect<AffineDialect>();
894  for (OpFoldResult ofr : values) {
895  if (auto value = ofr.dyn_cast<Value>()) {
896  actualValues.push_back(value);
897  continue;
898  }
899  // Since we are directly specifying `index` as the result type, we need to
900  // ensure the provided attribute is also an index type. Otherwise, the
901  // AffineDialect materializer will create invalid `arith.constant`
902  // operations if the provided Attribute is any other kind of integer.
903  constants.push_back(dialect->materializeConstant(
904  b, b.getIndexAttr(ofr.get<Attribute>().cast<IntegerAttr>().getInt()),
905  b.getIndexType(), loc));
906  actualValues.push_back(constants.back()->getResult(0));
907  }
908 }
909 
910 /// Create an operation of the type provided as template argument and attempt to
911 /// fold it immediately. The operation is expected to have a builder taking
912 /// arbitrary `leadingArguments`, followed by a list of Value-typed `operands`.
913 /// The operation is also expected to always produce a single result. Return an
914 /// `OpFoldResult` containing the Attribute representing the folded constant if
915 /// complete folding was possible and a Value produced by the created operation
916 /// otherwise.
917 template <typename OpTy, typename... Args>
918 static std::enable_if_t<OpTy::template hasTrait<OpTrait::OneResult>(),
919  OpFoldResult>
921  Args &&... leadingArguments) {
922  // Identify the constant operands and extract their values as attributes.
923  // Note that we cannot use the original values directly because the list of
924  // operands may have changed due to canonicalization and composition.
925  SmallVector<Attribute> constantOperands;
926  constantOperands.reserve(operands.size());
927  for (Value operand : operands) {
928  IntegerAttr attr;
929  if (matchPattern(operand, m_Constant(&attr)))
930  constantOperands.push_back(attr);
931  else
932  constantOperands.push_back(nullptr);
933  }
934 
935  // Create the operation and immediately attempt to fold it. On success,
936  // delete the operation and prepare the (unmaterialized) value for being
937  // returned. On failure, return the operation result value. Temporarily remove
938  // the listener to avoid notifying it when the op is created as it may be
939  // removed immediately and there is no way of notifying the caller about that
940  // without resorting to RewriterBase.
941  //
942  // TODO: arguably, the main folder (createOrFold) API should support this use
943  // case instead of indiscriminately materializing constants.
944  OpBuilder::Listener *listener = b.getListener();
945  b.setListener(nullptr);
946  auto listenerResetter =
947  llvm::make_scope_exit([listener, &b] { b.setListener(listener); });
948  OpTy op =
949  b.create<OpTy>(loc, std::forward<Args>(leadingArguments)..., operands);
950  SmallVector<OpFoldResult, 1> foldResults;
951  if (succeeded(op->fold(constantOperands, foldResults)) &&
952  !foldResults.empty()) {
953  op->erase();
954  return foldResults.front();
955  }
956 
957  // Notify the listener now that we definitely know that the operation will
958  // persist. Use the original listener stored in the variable.
959  if (listener)
960  listener->notifyOperationInserted(op);
961  return op->getResult(0);
962 }
963 
965  AffineMap map,
966  ValueRange operands) {
967  AffineMap normalizedMap = map;
968  SmallVector<Value, 8> normalizedOperands(operands.begin(), operands.end());
969  composeAffineMapAndOperands(&normalizedMap, &normalizedOperands);
970  assert(normalizedMap);
971  return b.create<AffineApplyOp>(loc, normalizedMap, normalizedOperands);
972 }
973 
975  AffineExpr e, ValueRange values) {
978  values);
979 }
980 
981 /// Composes the given affine map with the given list of operands, pulling in
982 /// the maps from any affine.apply operations that supply the operands.
984  SmallVectorImpl<Value> &operands) {
985  // Compose and canonicalize each expression in the map individually because
986  // composition only applies to single-result maps, collecting potentially
987  // duplicate operands in a single list with shifted dimensions and symbols.
988  SmallVector<Value> dims, symbols;
990  for (unsigned i : llvm::seq<unsigned>(0, map.getNumResults())) {
991  SmallVector<Value> submapOperands(operands.begin(), operands.end());
992  AffineMap submap = map.getSubMap({i});
993  fullyComposeAffineMapAndOperands(&submap, &submapOperands);
994  canonicalizeMapAndOperands(&submap, &submapOperands);
995  unsigned numNewDims = submap.getNumDims();
996  submap = submap.shiftDims(dims.size()).shiftSymbols(symbols.size());
997  llvm::append_range(dims,
998  ArrayRef<Value>(submapOperands).take_front(numNewDims));
999  llvm::append_range(symbols,
1000  ArrayRef<Value>(submapOperands).drop_front(numNewDims));
1001  exprs.push_back(submap.getResult(0));
1002  }
1003 
1004  // Canonicalize the map created from composed expressions to deduplicate the
1005  // dimension and symbol operands.
1006  operands = llvm::to_vector(llvm::concat<Value>(dims, symbols));
1007  map = AffineMap::get(dims.size(), symbols.size(), exprs, map.getContext());
1008  canonicalizeMapAndOperands(&map, &operands);
1009 }
1010 
1013  ArrayRef<OpFoldResult> operands) {
1014  assert(map.getNumResults() == 1 && "building affine.apply with !=1 result");
1015 
1016  SmallVector<Operation *> constants;
1017  SmallVector<Value> actualValues;
1018  materializeConstants(b, loc, operands, constants, actualValues);
1019  composeAffineMapAndOperands(&map, &actualValues);
1020  OpFoldResult result = createOrFold<AffineApplyOp>(b, loc, actualValues, map);
1021 
1022  // Constants are always folded into affine min/max because they can be
1023  // represented as constant expressions, so delete them.
1024  for (Operation *op : constants)
1025  op->erase();
1026  return result;
1027 }
1028 
1031  ArrayRef<OpFoldResult> operands) {
1033  b, loc, AffineMap::inferFromExprList(ArrayRef<AffineExpr>{expr}).front(),
1034  operands);
1035 }
1036 
1038  OpBuilder &b, Location loc, AffineMap map,
1039  ArrayRef<OpFoldResult> operands) {
1040  return llvm::to_vector(llvm::map_range(
1041  llvm::seq<unsigned>(0, map.getNumResults()), [&](unsigned i) {
1042  return makeComposedFoldedAffineApply(b, loc, map.getSubMap({i}),
1043  operands);
1044  }));
1045 }
1046 
1048  ValueRange operands) {
1049  SmallVector<Value> allOperands = llvm::to_vector(operands);
1050  composeMultiResultAffineMap(map, allOperands);
1051  return b.createOrFold<AffineMinOp>(loc, b.getIndexType(), map, allOperands);
1052 }
1053 
1054 template <typename OpTy>
1056  AffineMap map,
1057  ArrayRef<OpFoldResult> operands) {
1058  SmallVector<Operation *> constants;
1059  SmallVector<Value> actualValues;
1060  materializeConstants(b, loc, operands, constants, actualValues);
1061  composeMultiResultAffineMap(map, actualValues);
1062  OpFoldResult result =
1063  createOrFold<OpTy>(b, loc, actualValues, b.getIndexType(), map);
1064 
1065  // Constants are always folded into affine min/max because they can be
1066  // represented as constant expressions, so delete them.
1067  for (Operation *op : constants)
1068  op->erase();
1069  return result;
1070 }
1071 
1074  ArrayRef<OpFoldResult> operands) {
1075  return makeComposedFoldedMinMax<AffineMinOp>(b, loc, map, operands);
1076 }
1077 
1080  ArrayRef<OpFoldResult> operands) {
1081  return makeComposedFoldedMinMax<AffineMaxOp>(b, loc, map, operands);
1082 }
1083 
1084 /// Fully compose map with operands and canonicalize the result.
1085 /// Return the `createOrFold`'ed AffineApply op.
1087  AffineMap map,
1088  ValueRange operandsRef) {
1089  SmallVector<Value, 4> operands(operandsRef.begin(), operandsRef.end());
1090  fullyComposeAffineMapAndOperands(&map, &operands);
1091  canonicalizeMapAndOperands(&map, &operands);
1092  return b.createOrFold<AffineApplyOp>(loc, map, operands);
1093 }
1094 
1096  AffineMap map, ValueRange values) {
1098  res.reserve(map.getNumResults());
1099  unsigned numDims = map.getNumDims(), numSym = map.getNumSymbols();
1100  // For each `expr` in `map`, applies the `expr` to the values extracted from
1101  // ranges. If the resulting application can be folded into a Value, the
1102  // folding occurs eagerly.
1103  for (auto expr : map.getResults()) {
1104  AffineMap map = AffineMap::get(numDims, numSym, expr);
1105  res.push_back(createFoldedComposedAffineApply(b, loc, map, values));
1106  }
1107  return res;
1108 }
1109 
1110 // A symbol may appear as a dim in affine.apply operations. This function
1111 // canonicalizes dims that are valid symbols into actual symbols.
1112 template <class MapOrSet>
1113 static void canonicalizePromotedSymbols(MapOrSet *mapOrSet,
1114  SmallVectorImpl<Value> *operands) {
1115  if (!mapOrSet || operands->empty())
1116  return;
1117 
1118  assert(mapOrSet->getNumInputs() == operands->size() &&
1119  "map/set inputs must match number of operands");
1120 
1121  auto *context = mapOrSet->getContext();
1122  SmallVector<Value, 8> resultOperands;
1123  resultOperands.reserve(operands->size());
1124  SmallVector<Value, 8> remappedSymbols;
1125  remappedSymbols.reserve(operands->size());
1126  unsigned nextDim = 0;
1127  unsigned nextSym = 0;
1128  unsigned oldNumSyms = mapOrSet->getNumSymbols();
1129  SmallVector<AffineExpr, 8> dimRemapping(mapOrSet->getNumDims());
1130  for (unsigned i = 0, e = mapOrSet->getNumInputs(); i != e; ++i) {
1131  if (i < mapOrSet->getNumDims()) {
1132  if (isValidSymbol((*operands)[i])) {
1133  // This is a valid symbol that appears as a dim, canonicalize it.
1134  dimRemapping[i] = getAffineSymbolExpr(oldNumSyms + nextSym++, context);
1135  remappedSymbols.push_back((*operands)[i]);
1136  } else {
1137  dimRemapping[i] = getAffineDimExpr(nextDim++, context);
1138  resultOperands.push_back((*operands)[i]);
1139  }
1140  } else {
1141  resultOperands.push_back((*operands)[i]);
1142  }
1143  }
1144 
1145  resultOperands.append(remappedSymbols.begin(), remappedSymbols.end());
1146  *operands = resultOperands;
1147  *mapOrSet = mapOrSet->replaceDimsAndSymbols(dimRemapping, {}, nextDim,
1148  oldNumSyms + nextSym);
1149 
1150  assert(mapOrSet->getNumInputs() == operands->size() &&
1151  "map/set inputs must match number of operands");
1152 }
1153 
1154 // Works for either an affine map or an integer set.
1155 template <class MapOrSet>
1156 static void canonicalizeMapOrSetAndOperands(MapOrSet *mapOrSet,
1157  SmallVectorImpl<Value> *operands) {
1159  "Argument must be either of AffineMap or IntegerSet type");
1160 
1161  if (!mapOrSet || operands->empty())
1162  return;
1163 
1164  assert(mapOrSet->getNumInputs() == operands->size() &&
1165  "map/set inputs must match number of operands");
1166 
1167  canonicalizePromotedSymbols<MapOrSet>(mapOrSet, operands);
1168 
1169  // Check to see what dims are used.
1170  llvm::SmallBitVector usedDims(mapOrSet->getNumDims());
1171  llvm::SmallBitVector usedSyms(mapOrSet->getNumSymbols());
1172  mapOrSet->walkExprs([&](AffineExpr expr) {
1173  if (auto dimExpr = expr.dyn_cast<AffineDimExpr>())
1174  usedDims[dimExpr.getPosition()] = true;
1175  else if (auto symExpr = expr.dyn_cast<AffineSymbolExpr>())
1176  usedSyms[symExpr.getPosition()] = true;
1177  });
1178 
1179  auto *context = mapOrSet->getContext();
1180 
1181  SmallVector<Value, 8> resultOperands;
1182  resultOperands.reserve(operands->size());
1183 
1184  llvm::SmallDenseMap<Value, AffineExpr, 8> seenDims;
1185  SmallVector<AffineExpr, 8> dimRemapping(mapOrSet->getNumDims());
1186  unsigned nextDim = 0;
1187  for (unsigned i = 0, e = mapOrSet->getNumDims(); i != e; ++i) {
1188  if (usedDims[i]) {
1189  // Remap dim positions for duplicate operands.
1190  auto it = seenDims.find((*operands)[i]);
1191  if (it == seenDims.end()) {
1192  dimRemapping[i] = getAffineDimExpr(nextDim++, context);
1193  resultOperands.push_back((*operands)[i]);
1194  seenDims.insert(std::make_pair((*operands)[i], dimRemapping[i]));
1195  } else {
1196  dimRemapping[i] = it->second;
1197  }
1198  }
1199  }
1200  llvm::SmallDenseMap<Value, AffineExpr, 8> seenSymbols;
1201  SmallVector<AffineExpr, 8> symRemapping(mapOrSet->getNumSymbols());
1202  unsigned nextSym = 0;
1203  for (unsigned i = 0, e = mapOrSet->getNumSymbols(); i != e; ++i) {
1204  if (!usedSyms[i])
1205  continue;
1206  // Handle constant operands (only needed for symbolic operands since
1207  // constant operands in dimensional positions would have already been
1208  // promoted to symbolic positions above).
1209  IntegerAttr operandCst;
1210  if (matchPattern((*operands)[i + mapOrSet->getNumDims()],
1211  m_Constant(&operandCst))) {
1212  symRemapping[i] =
1213  getAffineConstantExpr(operandCst.getValue().getSExtValue(), context);
1214  continue;
1215  }
1216  // Remap symbol positions for duplicate operands.
1217  auto it = seenSymbols.find((*operands)[i + mapOrSet->getNumDims()]);
1218  if (it == seenSymbols.end()) {
1219  symRemapping[i] = getAffineSymbolExpr(nextSym++, context);
1220  resultOperands.push_back((*operands)[i + mapOrSet->getNumDims()]);
1221  seenSymbols.insert(std::make_pair((*operands)[i + mapOrSet->getNumDims()],
1222  symRemapping[i]));
1223  } else {
1224  symRemapping[i] = it->second;
1225  }
1226  }
1227  *mapOrSet = mapOrSet->replaceDimsAndSymbols(dimRemapping, symRemapping,
1228  nextDim, nextSym);
1229  *operands = resultOperands;
1230 }
1231 
1233  SmallVectorImpl<Value> *operands) {
1234  canonicalizeMapOrSetAndOperands<AffineMap>(map, operands);
1235 }
1236 
1238  SmallVectorImpl<Value> *operands) {
1239  canonicalizeMapOrSetAndOperands<IntegerSet>(set, operands);
1240 }
1241 
1242 namespace {
1243 /// Simplify AffineApply, AffineLoad, and AffineStore operations by composing
1244 /// maps that supply results into them.
1245 ///
1246 template <typename AffineOpTy>
1247 struct SimplifyAffineOp : public OpRewritePattern<AffineOpTy> {
1249 
1250  /// Replace the affine op with another instance of it with the supplied
1251  /// map and mapOperands.
1252  void replaceAffineOp(PatternRewriter &rewriter, AffineOpTy affineOp,
1253  AffineMap map, ArrayRef<Value> mapOperands) const;
1254 
1255  LogicalResult matchAndRewrite(AffineOpTy affineOp,
1256  PatternRewriter &rewriter) const override {
1257  static_assert(
1258  llvm::is_one_of<AffineOpTy, AffineLoadOp, AffinePrefetchOp,
1259  AffineStoreOp, AffineApplyOp, AffineMinOp, AffineMaxOp,
1260  AffineVectorStoreOp, AffineVectorLoadOp>::value,
1261  "affine load/store/vectorstore/vectorload/apply/prefetch/min/max op "
1262  "expected");
1263  auto map = affineOp.getAffineMap();
1264  AffineMap oldMap = map;
1265  auto oldOperands = affineOp.getMapOperands();
1266  SmallVector<Value, 8> resultOperands(oldOperands);
1267  composeAffineMapAndOperands(&map, &resultOperands);
1268  canonicalizeMapAndOperands(&map, &resultOperands);
1269  simplifyMapWithOperands(map, resultOperands);
1270  if (map == oldMap && std::equal(oldOperands.begin(), oldOperands.end(),
1271  resultOperands.begin()))
1272  return failure();
1273 
1274  replaceAffineOp(rewriter, affineOp, map, resultOperands);
1275  return success();
1276  }
1277 };
1278 
1279 // Specialize the template to account for the different build signatures for
1280 // affine load, store, and apply ops.
1281 template <>
1282 void SimplifyAffineOp<AffineLoadOp>::replaceAffineOp(
1283  PatternRewriter &rewriter, AffineLoadOp load, AffineMap map,
1284  ArrayRef<Value> mapOperands) const {
1285  rewriter.replaceOpWithNewOp<AffineLoadOp>(load, load.getMemRef(), map,
1286  mapOperands);
1287 }
1288 template <>
1289 void SimplifyAffineOp<AffinePrefetchOp>::replaceAffineOp(
1290  PatternRewriter &rewriter, AffinePrefetchOp prefetch, AffineMap map,
1291  ArrayRef<Value> mapOperands) const {
1292  rewriter.replaceOpWithNewOp<AffinePrefetchOp>(
1293  prefetch, prefetch.getMemref(), map, mapOperands,
1294  prefetch.getLocalityHint(), prefetch.getIsWrite(),
1295  prefetch.getIsDataCache());
1296 }
1297 template <>
1298 void SimplifyAffineOp<AffineStoreOp>::replaceAffineOp(
1299  PatternRewriter &rewriter, AffineStoreOp store, AffineMap map,
1300  ArrayRef<Value> mapOperands) const {
1301  rewriter.replaceOpWithNewOp<AffineStoreOp>(
1302  store, store.getValueToStore(), store.getMemRef(), map, mapOperands);
1303 }
1304 template <>
1305 void SimplifyAffineOp<AffineVectorLoadOp>::replaceAffineOp(
1306  PatternRewriter &rewriter, AffineVectorLoadOp vectorload, AffineMap map,
1307  ArrayRef<Value> mapOperands) const {
1308  rewriter.replaceOpWithNewOp<AffineVectorLoadOp>(
1309  vectorload, vectorload.getVectorType(), vectorload.getMemRef(), map,
1310  mapOperands);
1311 }
1312 template <>
1313 void SimplifyAffineOp<AffineVectorStoreOp>::replaceAffineOp(
1314  PatternRewriter &rewriter, AffineVectorStoreOp vectorstore, AffineMap map,
1315  ArrayRef<Value> mapOperands) const {
1316  rewriter.replaceOpWithNewOp<AffineVectorStoreOp>(
1317  vectorstore, vectorstore.getValueToStore(), vectorstore.getMemRef(), map,
1318  mapOperands);
1319 }
1320 
1321 // Generic version for ops that don't have extra operands.
1322 template <typename AffineOpTy>
1323 void SimplifyAffineOp<AffineOpTy>::replaceAffineOp(
1324  PatternRewriter &rewriter, AffineOpTy op, AffineMap map,
1325  ArrayRef<Value> mapOperands) const {
1326  rewriter.replaceOpWithNewOp<AffineOpTy>(op, map, mapOperands);
1327 }
1328 } // namespace
1329 
1330 void AffineApplyOp::getCanonicalizationPatterns(RewritePatternSet &results,
1331  MLIRContext *context) {
1332  results.add<SimplifyAffineOp<AffineApplyOp>>(context);
1333 }
1334 
1335 //===----------------------------------------------------------------------===//
1336 // AffineDmaStartOp
1337 //===----------------------------------------------------------------------===//
1338 
1339 // TODO: Check that map operands are loop IVs or symbols.
1340 void AffineDmaStartOp::build(OpBuilder &builder, OperationState &result,
1341  Value srcMemRef, AffineMap srcMap,
1342  ValueRange srcIndices, Value destMemRef,
1343  AffineMap dstMap, ValueRange destIndices,
1344  Value tagMemRef, AffineMap tagMap,
1345  ValueRange tagIndices, Value numElements,
1346  Value stride, Value elementsPerStride) {
1347  result.addOperands(srcMemRef);
1348  result.addAttribute(getSrcMapAttrStrName(), AffineMapAttr::get(srcMap));
1349  result.addOperands(srcIndices);
1350  result.addOperands(destMemRef);
1351  result.addAttribute(getDstMapAttrStrName(), AffineMapAttr::get(dstMap));
1352  result.addOperands(destIndices);
1353  result.addOperands(tagMemRef);
1354  result.addAttribute(getTagMapAttrStrName(), AffineMapAttr::get(tagMap));
1355  result.addOperands(tagIndices);
1356  result.addOperands(numElements);
1357  if (stride) {
1358  result.addOperands({stride, elementsPerStride});
1359  }
1360 }
1361 
1363  p << " " << getSrcMemRef() << '[';
1365  p << "], " << getDstMemRef() << '[';
1367  p << "], " << getTagMemRef() << '[';
1369  p << "], " << getNumElements();
1370  if (isStrided()) {
1371  p << ", " << getStride();
1372  p << ", " << getNumElementsPerStride();
1373  }
1374  p << " : " << getSrcMemRefType() << ", " << getDstMemRefType() << ", "
1375  << getTagMemRefType();
1376 }
1377 
1378 // Parse AffineDmaStartOp.
1379 // Ex:
1380 // affine.dma_start %src[%i, %j], %dst[%k, %l], %tag[%index], %size,
1381 // %stride, %num_elt_per_stride
1382 // : memref<3076 x f32, 0>, memref<1024 x f32, 2>, memref<1 x i32>
1383 //
1385  OperationState &result) {
1386  OpAsmParser::UnresolvedOperand srcMemRefInfo;
1387  AffineMapAttr srcMapAttr;
1389  OpAsmParser::UnresolvedOperand dstMemRefInfo;
1390  AffineMapAttr dstMapAttr;
1392  OpAsmParser::UnresolvedOperand tagMemRefInfo;
1393  AffineMapAttr tagMapAttr;
1395  OpAsmParser::UnresolvedOperand numElementsInfo;
1397 
1398  SmallVector<Type, 3> types;
1399  auto indexType = parser.getBuilder().getIndexType();
1400 
1401  // Parse and resolve the following list of operands:
1402  // *) dst memref followed by its affine maps operands (in square brackets).
1403  // *) src memref followed by its affine map operands (in square brackets).
1404  // *) tag memref followed by its affine map operands (in square brackets).
1405  // *) number of elements transferred by DMA operation.
1406  if (parser.parseOperand(srcMemRefInfo) ||
1407  parser.parseAffineMapOfSSAIds(srcMapOperands, srcMapAttr,
1409  result.attributes) ||
1410  parser.parseComma() || parser.parseOperand(dstMemRefInfo) ||
1411  parser.parseAffineMapOfSSAIds(dstMapOperands, dstMapAttr,
1413  result.attributes) ||
1414  parser.parseComma() || parser.parseOperand(tagMemRefInfo) ||
1415  parser.parseAffineMapOfSSAIds(tagMapOperands, tagMapAttr,
1417  result.attributes) ||
1418  parser.parseComma() || parser.parseOperand(numElementsInfo))
1419  return failure();
1420 
1421  // Parse optional stride and elements per stride.
1422  if (parser.parseTrailingOperandList(strideInfo))
1423  return failure();
1424 
1425  if (!strideInfo.empty() && strideInfo.size() != 2) {
1426  return parser.emitError(parser.getNameLoc(),
1427  "expected two stride related operands");
1428  }
1429  bool isStrided = strideInfo.size() == 2;
1430 
1431  if (parser.parseColonTypeList(types))
1432  return failure();
1433 
1434  if (types.size() != 3)
1435  return parser.emitError(parser.getNameLoc(), "expected three types");
1436 
1437  if (parser.resolveOperand(srcMemRefInfo, types[0], result.operands) ||
1438  parser.resolveOperands(srcMapOperands, indexType, result.operands) ||
1439  parser.resolveOperand(dstMemRefInfo, types[1], result.operands) ||
1440  parser.resolveOperands(dstMapOperands, indexType, result.operands) ||
1441  parser.resolveOperand(tagMemRefInfo, types[2], result.operands) ||
1442  parser.resolveOperands(tagMapOperands, indexType, result.operands) ||
1443  parser.resolveOperand(numElementsInfo, indexType, result.operands))
1444  return failure();
1445 
1446  if (isStrided) {
1447  if (parser.resolveOperands(strideInfo, indexType, result.operands))
1448  return failure();
1449  }
1450 
1451  // Check that src/dst/tag operand counts match their map.numInputs.
1452  if (srcMapOperands.size() != srcMapAttr.getValue().getNumInputs() ||
1453  dstMapOperands.size() != dstMapAttr.getValue().getNumInputs() ||
1454  tagMapOperands.size() != tagMapAttr.getValue().getNumInputs())
1455  return parser.emitError(parser.getNameLoc(),
1456  "memref operand count not equal to map.numInputs");
1457  return success();
1458 }
1459 
1461  if (!getOperand(getSrcMemRefOperandIndex()).getType().isa<MemRefType>())
1462  return emitOpError("expected DMA source to be of memref type");
1463  if (!getOperand(getDstMemRefOperandIndex()).getType().isa<MemRefType>())
1464  return emitOpError("expected DMA destination to be of memref type");
1465  if (!getOperand(getTagMemRefOperandIndex()).getType().isa<MemRefType>())
1466  return emitOpError("expected DMA tag to be of memref type");
1467 
1468  unsigned numInputsAllMaps = getSrcMap().getNumInputs() +
1469  getDstMap().getNumInputs() +
1470  getTagMap().getNumInputs();
1471  if (getNumOperands() != numInputsAllMaps + 3 + 1 &&
1472  getNumOperands() != numInputsAllMaps + 3 + 1 + 2) {
1473  return emitOpError("incorrect number of operands");
1474  }
1475 
1476  Region *scope = getAffineScope(*this);
1477  for (auto idx : getSrcIndices()) {
1478  if (!idx.getType().isIndex())
1479  return emitOpError("src index to dma_start must have 'index' type");
1480  if (!isValidAffineIndexOperand(idx, scope))
1481  return emitOpError("src index must be a dimension or symbol identifier");
1482  }
1483  for (auto idx : getDstIndices()) {
1484  if (!idx.getType().isIndex())
1485  return emitOpError("dst index to dma_start must have 'index' type");
1486  if (!isValidAffineIndexOperand(idx, scope))
1487  return emitOpError("dst index must be a dimension or symbol identifier");
1488  }
1489  for (auto idx : getTagIndices()) {
1490  if (!idx.getType().isIndex())
1491  return emitOpError("tag index to dma_start must have 'index' type");
1492  if (!isValidAffineIndexOperand(idx, scope))
1493  return emitOpError("tag index must be a dimension or symbol identifier");
1494  }
1495  return success();
1496 }
1497 
1499  SmallVectorImpl<OpFoldResult> &results) {
1500  /// dma_start(memrefcast) -> dma_start
1501  return memref::foldMemRefCast(*this);
1502 }
1503 
1504 //===----------------------------------------------------------------------===//
1505 // AffineDmaWaitOp
1506 //===----------------------------------------------------------------------===//
1507 
1508 // TODO: Check that map operands are loop IVs or symbols.
1510  Value tagMemRef, AffineMap tagMap,
1511  ValueRange tagIndices, Value numElements) {
1512  result.addOperands(tagMemRef);
1513  result.addAttribute(getTagMapAttrStrName(), AffineMapAttr::get(tagMap));
1514  result.addOperands(tagIndices);
1515  result.addOperands(numElements);
1516 }
1517 
1519  p << " " << getTagMemRef() << '[';
1521  p.printAffineMapOfSSAIds(getTagMapAttr(), operands);
1522  p << "], ";
1524  p << " : " << getTagMemRef().getType();
1525 }
1526 
1527 // Parse AffineDmaWaitOp.
1528 // Eg:
1529 // affine.dma_wait %tag[%index], %num_elements
1530 // : memref<1 x i32, (d0) -> (d0), 4>
1531 //
1533  OperationState &result) {
1534  OpAsmParser::UnresolvedOperand tagMemRefInfo;
1535  AffineMapAttr tagMapAttr;
1537  Type type;
1538  auto indexType = parser.getBuilder().getIndexType();
1539  OpAsmParser::UnresolvedOperand numElementsInfo;
1540 
1541  // Parse tag memref, its map operands, and dma size.
1542  if (parser.parseOperand(tagMemRefInfo) ||
1543  parser.parseAffineMapOfSSAIds(tagMapOperands, tagMapAttr,
1545  result.attributes) ||
1546  parser.parseComma() || parser.parseOperand(numElementsInfo) ||
1547  parser.parseColonType(type) ||
1548  parser.resolveOperand(tagMemRefInfo, type, result.operands) ||
1549  parser.resolveOperands(tagMapOperands, indexType, result.operands) ||
1550  parser.resolveOperand(numElementsInfo, indexType, result.operands))
1551  return failure();
1552 
1553  if (!type.isa<MemRefType>())
1554  return parser.emitError(parser.getNameLoc(),
1555  "expected tag to be of memref type");
1556 
1557  if (tagMapOperands.size() != tagMapAttr.getValue().getNumInputs())
1558  return parser.emitError(parser.getNameLoc(),
1559  "tag memref operand count != to map.numInputs");
1560  return success();
1561 }
1562 
1564  if (!getOperand(0).getType().isa<MemRefType>())
1565  return emitOpError("expected DMA tag to be of memref type");
1566  Region *scope = getAffineScope(*this);
1567  for (auto idx : getTagIndices()) {
1568  if (!idx.getType().isIndex())
1569  return emitOpError("index to dma_wait must have 'index' type");
1570  if (!isValidAffineIndexOperand(idx, scope))
1571  return emitOpError("index must be a dimension or symbol identifier");
1572  }
1573  return success();
1574 }
1575 
1577  SmallVectorImpl<OpFoldResult> &results) {
1578  /// dma_wait(memrefcast) -> dma_wait
1579  return memref::foldMemRefCast(*this);
1580 }
1581 
1582 //===----------------------------------------------------------------------===//
1583 // AffineForOp
1584 //===----------------------------------------------------------------------===//
1585 
1586 /// 'bodyBuilder' is used to build the body of affine.for. If iterArgs and
1587 /// bodyBuilder are empty/null, we include default terminator op.
1588 void AffineForOp::build(OpBuilder &builder, OperationState &result,
1589  ValueRange lbOperands, AffineMap lbMap,
1590  ValueRange ubOperands, AffineMap ubMap, int64_t step,
1591  ValueRange iterArgs, BodyBuilderFn bodyBuilder) {
1592  assert(((!lbMap && lbOperands.empty()) ||
1593  lbOperands.size() == lbMap.getNumInputs()) &&
1594  "lower bound operand count does not match the affine map");
1595  assert(((!ubMap && ubOperands.empty()) ||
1596  ubOperands.size() == ubMap.getNumInputs()) &&
1597  "upper bound operand count does not match the affine map");
1598  assert(step > 0 && "step has to be a positive integer constant");
1599 
1600  for (Value val : iterArgs)
1601  result.addTypes(val.getType());
1602 
1603  // Add an attribute for the step.
1604  result.addAttribute(getStepAttrStrName(),
1605  builder.getIntegerAttr(builder.getIndexType(), step));
1606 
1607  // Add the lower bound.
1608  result.addAttribute(getLowerBoundAttrStrName(), AffineMapAttr::get(lbMap));
1609  result.addOperands(lbOperands);
1610 
1611  // Add the upper bound.
1612  result.addAttribute(getUpperBoundAttrStrName(), AffineMapAttr::get(ubMap));
1613  result.addOperands(ubOperands);
1614 
1615  result.addOperands(iterArgs);
1616  // Create a region and a block for the body. The argument of the region is
1617  // the loop induction variable.
1618  Region *bodyRegion = result.addRegion();
1619  bodyRegion->push_back(new Block);
1620  Block &bodyBlock = bodyRegion->front();
1621  Value inductionVar =
1622  bodyBlock.addArgument(builder.getIndexType(), result.location);
1623  for (Value val : iterArgs)
1624  bodyBlock.addArgument(val.getType(), val.getLoc());
1625 
1626  // Create the default terminator if the builder is not provided and if the
1627  // iteration arguments are not provided. Otherwise, leave this to the caller
1628  // because we don't know which values to return from the loop.
1629  if (iterArgs.empty() && !bodyBuilder) {
1630  ensureTerminator(*bodyRegion, builder, result.location);
1631  } else if (bodyBuilder) {
1632  OpBuilder::InsertionGuard guard(builder);
1633  builder.setInsertionPointToStart(&bodyBlock);
1634  bodyBuilder(builder, result.location, inductionVar,
1635  bodyBlock.getArguments().drop_front());
1636  }
1637 }
1638 
1639 void AffineForOp::build(OpBuilder &builder, OperationState &result, int64_t lb,
1640  int64_t ub, int64_t step, ValueRange iterArgs,
1641  BodyBuilderFn bodyBuilder) {
1642  auto lbMap = AffineMap::getConstantMap(lb, builder.getContext());
1643  auto ubMap = AffineMap::getConstantMap(ub, builder.getContext());
1644  return build(builder, result, {}, lbMap, {}, ubMap, step, iterArgs,
1645  bodyBuilder);
1646 }
1647 
1648 LogicalResult AffineForOp::verifyRegions() {
1649  // Check that the body defines as single block argument for the induction
1650  // variable.
1651  auto *body = getBody();
1652  if (body->getNumArguments() == 0 || !body->getArgument(0).getType().isIndex())
1653  return emitOpError("expected body to have a single index argument for the "
1654  "induction variable");
1655 
1656  // Verify that the bound operands are valid dimension/symbols.
1657  /// Lower bound.
1658  if (getLowerBoundMap().getNumInputs() > 0)
1660  getLowerBoundMap().getNumDims())))
1661  return failure();
1662  /// Upper bound.
1663  if (getUpperBoundMap().getNumInputs() > 0)
1665  getUpperBoundMap().getNumDims())))
1666  return failure();
1667 
1668  unsigned opNumResults = getNumResults();
1669  if (opNumResults == 0)
1670  return success();
1671 
1672  // If ForOp defines values, check that the number and types of the defined
1673  // values match ForOp initial iter operands and backedge basic block
1674  // arguments.
1675  if (getNumIterOperands() != opNumResults)
1676  return emitOpError(
1677  "mismatch between the number of loop-carried values and results");
1678  if (getNumRegionIterArgs() != opNumResults)
1679  return emitOpError(
1680  "mismatch between the number of basic block args and results");
1681 
1682  return success();
1683 }
1684 
1685 /// Parse a for operation loop bounds.
1686 static ParseResult parseBound(bool isLower, OperationState &result,
1687  OpAsmParser &p) {
1688  // 'min' / 'max' prefixes are generally syntactic sugar, but are required if
1689  // the map has multiple results.
1690  bool failedToParsedMinMax =
1691  failed(p.parseOptionalKeyword(isLower ? "max" : "min"));
1692 
1693  auto &builder = p.getBuilder();
1694  auto boundAttrStrName = isLower ? AffineForOp::getLowerBoundAttrStrName()
1695  : AffineForOp::getUpperBoundAttrStrName();
1696 
1697  // Parse ssa-id as identity map.
1699  if (p.parseOperandList(boundOpInfos))
1700  return failure();
1701 
1702  if (!boundOpInfos.empty()) {
1703  // Check that only one operand was parsed.
1704  if (boundOpInfos.size() > 1)
1705  return p.emitError(p.getNameLoc(),
1706  "expected only one loop bound operand");
1707 
1708  // TODO: improve error message when SSA value is not of index type.
1709  // Currently it is 'use of value ... expects different type than prior uses'
1710  if (p.resolveOperand(boundOpInfos.front(), builder.getIndexType(),
1711  result.operands))
1712  return failure();
1713 
1714  // Create an identity map using symbol id. This representation is optimized
1715  // for storage. Analysis passes may expand it into a multi-dimensional map
1716  // if desired.
1717  AffineMap map = builder.getSymbolIdentityMap();
1718  result.addAttribute(boundAttrStrName, AffineMapAttr::get(map));
1719  return success();
1720  }
1721 
1722  // Get the attribute location.
1723  SMLoc attrLoc = p.getCurrentLocation();
1724 
1725  Attribute boundAttr;
1726  if (p.parseAttribute(boundAttr, builder.getIndexType(), boundAttrStrName,
1727  result.attributes))
1728  return failure();
1729 
1730  // Parse full form - affine map followed by dim and symbol list.
1731  if (auto affineMapAttr = boundAttr.dyn_cast<AffineMapAttr>()) {
1732  unsigned currentNumOperands = result.operands.size();
1733  unsigned numDims;
1734  if (parseDimAndSymbolList(p, result.operands, numDims))
1735  return failure();
1736 
1737  auto map = affineMapAttr.getValue();
1738  if (map.getNumDims() != numDims)
1739  return p.emitError(
1740  p.getNameLoc(),
1741  "dim operand count and affine map dim count must match");
1742 
1743  unsigned numDimAndSymbolOperands =
1744  result.operands.size() - currentNumOperands;
1745  if (numDims + map.getNumSymbols() != numDimAndSymbolOperands)
1746  return p.emitError(
1747  p.getNameLoc(),
1748  "symbol operand count and affine map symbol count must match");
1749 
1750  // If the map has multiple results, make sure that we parsed the min/max
1751  // prefix.
1752  if (map.getNumResults() > 1 && failedToParsedMinMax) {
1753  if (isLower) {
1754  return p.emitError(attrLoc, "lower loop bound affine map with "
1755  "multiple results requires 'max' prefix");
1756  }
1757  return p.emitError(attrLoc, "upper loop bound affine map with multiple "
1758  "results requires 'min' prefix");
1759  }
1760  return success();
1761  }
1762 
1763  // Parse custom assembly form.
1764  if (auto integerAttr = boundAttr.dyn_cast<IntegerAttr>()) {
1765  result.attributes.pop_back();
1766  result.addAttribute(
1767  boundAttrStrName,
1768  AffineMapAttr::get(builder.getConstantAffineMap(integerAttr.getInt())));
1769  return success();
1770  }
1771 
1772  return p.emitError(
1773  p.getNameLoc(),
1774  "expected valid affine map representation for loop bounds");
1775 }
1776 
1777 ParseResult AffineForOp::parse(OpAsmParser &parser, OperationState &result) {
1778  auto &builder = parser.getBuilder();
1779  OpAsmParser::Argument inductionVariable;
1780  inductionVariable.type = builder.getIndexType();
1781  // Parse the induction variable followed by '='.
1782  if (parser.parseArgument(inductionVariable) || parser.parseEqual())
1783  return failure();
1784 
1785  // Parse loop bounds.
1786  if (parseBound(/*isLower=*/true, result, parser) ||
1787  parser.parseKeyword("to", " between bounds") ||
1788  parseBound(/*isLower=*/false, result, parser))
1789  return failure();
1790 
1791  // Parse the optional loop step, we default to 1 if one is not present.
1792  if (parser.parseOptionalKeyword("step")) {
1793  result.addAttribute(
1794  AffineForOp::getStepAttrStrName(),
1795  builder.getIntegerAttr(builder.getIndexType(), /*value=*/1));
1796  } else {
1797  SMLoc stepLoc = parser.getCurrentLocation();
1798  IntegerAttr stepAttr;
1799  if (parser.parseAttribute(stepAttr, builder.getIndexType(),
1800  AffineForOp::getStepAttrStrName().data(),
1801  result.attributes))
1802  return failure();
1803 
1804  if (stepAttr.getValue().getSExtValue() < 0)
1805  return parser.emitError(
1806  stepLoc,
1807  "expected step to be representable as a positive signed integer");
1808  }
1809 
1810  // Parse the optional initial iteration arguments.
1813 
1814  // Induction variable.
1815  regionArgs.push_back(inductionVariable);
1816 
1817  if (succeeded(parser.parseOptionalKeyword("iter_args"))) {
1818  // Parse assignment list and results type list.
1819  if (parser.parseAssignmentList(regionArgs, operands) ||
1820  parser.parseArrowTypeList(result.types))
1821  return failure();
1822  // Resolve input operands.
1823  for (auto argOperandType :
1824  llvm::zip(llvm::drop_begin(regionArgs), operands, result.types)) {
1825  Type type = std::get<2>(argOperandType);
1826  std::get<0>(argOperandType).type = type;
1827  if (parser.resolveOperand(std::get<1>(argOperandType), type,
1828  result.operands))
1829  return failure();
1830  }
1831  }
1832 
1833  // Parse the body region.
1834  Region *body = result.addRegion();
1835  if (regionArgs.size() != result.types.size() + 1)
1836  return parser.emitError(
1837  parser.getNameLoc(),
1838  "mismatch between the number of loop-carried values and results");
1839  if (parser.parseRegion(*body, regionArgs))
1840  return failure();
1841 
1842  AffineForOp::ensureTerminator(*body, builder, result.location);
1843 
1844  // Parse the optional attribute list.
1845  return parser.parseOptionalAttrDict(result.attributes);
1846 }
1847 
1848 static void printBound(AffineMapAttr boundMap,
1849  Operation::operand_range boundOperands,
1850  const char *prefix, OpAsmPrinter &p) {
1851  AffineMap map = boundMap.getValue();
1852 
1853  // Check if this bound should be printed using custom assembly form.
1854  // The decision to restrict printing custom assembly form to trivial cases
1855  // comes from the will to roundtrip MLIR binary -> text -> binary in a
1856  // lossless way.
1857  // Therefore, custom assembly form parsing and printing is only supported for
1858  // zero-operand constant maps and single symbol operand identity maps.
1859  if (map.getNumResults() == 1) {
1860  AffineExpr expr = map.getResult(0);
1861 
1862  // Print constant bound.
1863  if (map.getNumDims() == 0 && map.getNumSymbols() == 0) {
1864  if (auto constExpr = expr.dyn_cast<AffineConstantExpr>()) {
1865  p << constExpr.getValue();
1866  return;
1867  }
1868  }
1869 
1870  // Print bound that consists of a single SSA symbol if the map is over a
1871  // single symbol.
1872  if (map.getNumDims() == 0 && map.getNumSymbols() == 1) {
1873  if (auto symExpr = expr.dyn_cast<AffineSymbolExpr>()) {
1874  p.printOperand(*boundOperands.begin());
1875  return;
1876  }
1877  }
1878  } else {
1879  // Map has multiple results. Print 'min' or 'max' prefix.
1880  p << prefix << ' ';
1881  }
1882 
1883  // Print the map and its operands.
1884  p << boundMap;
1885  printDimAndSymbolList(boundOperands.begin(), boundOperands.end(),
1886  map.getNumDims(), p);
1887 }
1888 
1889 unsigned AffineForOp::getNumIterOperands() {
1890  AffineMap lbMap = getLowerBoundMapAttr().getValue();
1891  AffineMap ubMap = getUpperBoundMapAttr().getValue();
1892 
1893  return getNumOperands() - lbMap.getNumInputs() - ubMap.getNumInputs();
1894 }
1895 
1897  p << ' ';
1898  p.printRegionArgument(getBody()->getArgument(0), /*argAttrs=*/{},
1899  /*omitType=*/true);
1900  p << " = ";
1901  printBound(getLowerBoundMapAttr(), getLowerBoundOperands(), "max", p);
1902  p << " to ";
1903  printBound(getUpperBoundMapAttr(), getUpperBoundOperands(), "min", p);
1904 
1905  if (getStep() != 1)
1906  p << " step " << getStep();
1907 
1908  bool printBlockTerminators = false;
1909  if (getNumIterOperands() > 0) {
1910  p << " iter_args(";
1911  auto regionArgs = getRegionIterArgs();
1912  auto operands = getIterOperands();
1913 
1914  llvm::interleaveComma(llvm::zip(regionArgs, operands), p, [&](auto it) {
1915  p << std::get<0>(it) << " = " << std::get<1>(it);
1916  });
1917  p << ") -> (" << getResultTypes() << ")";
1918  printBlockTerminators = true;
1919  }
1920 
1921  p << ' ';
1922  p.printRegion(getRegion(), /*printEntryBlockArgs=*/false,
1923  printBlockTerminators);
1924  p.printOptionalAttrDict((*this)->getAttrs(),
1925  /*elidedAttrs=*/{getLowerBoundAttrStrName(),
1926  getUpperBoundAttrStrName(),
1927  getStepAttrStrName()});
1928 }
1929 
1930 /// Fold the constant bounds of a loop.
1931 static LogicalResult foldLoopBounds(AffineForOp forOp) {
1932  auto foldLowerOrUpperBound = [&forOp](bool lower) {
1933  // Check to see if each of the operands is the result of a constant. If
1934  // so, get the value. If not, ignore it.
1935  SmallVector<Attribute, 8> operandConstants;
1936  auto boundOperands =
1937  lower ? forOp.getLowerBoundOperands() : forOp.getUpperBoundOperands();
1938  for (auto operand : boundOperands) {
1939  Attribute operandCst;
1940  matchPattern(operand, m_Constant(&operandCst));
1941  operandConstants.push_back(operandCst);
1942  }
1943 
1944  AffineMap boundMap =
1945  lower ? forOp.getLowerBoundMap() : forOp.getUpperBoundMap();
1946  assert(boundMap.getNumResults() >= 1 &&
1947  "bound maps should have at least one result");
1948  SmallVector<Attribute, 4> foldedResults;
1949  if (failed(boundMap.constantFold(operandConstants, foldedResults)))
1950  return failure();
1951 
1952  // Compute the max or min as applicable over the results.
1953  assert(!foldedResults.empty() && "bounds should have at least one result");
1954  auto maxOrMin = foldedResults[0].cast<IntegerAttr>().getValue();
1955  for (unsigned i = 1, e = foldedResults.size(); i < e; i++) {
1956  auto foldedResult = foldedResults[i].cast<IntegerAttr>().getValue();
1957  maxOrMin = lower ? llvm::APIntOps::smax(maxOrMin, foldedResult)
1958  : llvm::APIntOps::smin(maxOrMin, foldedResult);
1959  }
1960  lower ? forOp.setConstantLowerBound(maxOrMin.getSExtValue())
1961  : forOp.setConstantUpperBound(maxOrMin.getSExtValue());
1962  return success();
1963  };
1964 
1965  // Try to fold the lower bound.
1966  bool folded = false;
1967  if (!forOp.hasConstantLowerBound())
1968  folded |= succeeded(foldLowerOrUpperBound(/*lower=*/true));
1969 
1970  // Try to fold the upper bound.
1971  if (!forOp.hasConstantUpperBound())
1972  folded |= succeeded(foldLowerOrUpperBound(/*lower=*/false));
1973  return success(folded);
1974 }
1975 
1976 /// Canonicalize the bounds of the given loop.
1977 static LogicalResult canonicalizeLoopBounds(AffineForOp forOp) {
1978  SmallVector<Value, 4> lbOperands(forOp.getLowerBoundOperands());
1979  SmallVector<Value, 4> ubOperands(forOp.getUpperBoundOperands());
1980 
1981  auto lbMap = forOp.getLowerBoundMap();
1982  auto ubMap = forOp.getUpperBoundMap();
1983  auto prevLbMap = lbMap;
1984  auto prevUbMap = ubMap;
1985 
1986  composeAffineMapAndOperands(&lbMap, &lbOperands);
1987  canonicalizeMapAndOperands(&lbMap, &lbOperands);
1988  lbMap = removeDuplicateExprs(lbMap);
1989 
1990  composeAffineMapAndOperands(&ubMap, &ubOperands);
1991  canonicalizeMapAndOperands(&ubMap, &ubOperands);
1992  ubMap = removeDuplicateExprs(ubMap);
1993 
1994  // Any canonicalization change always leads to updated map(s).
1995  if (lbMap == prevLbMap && ubMap == prevUbMap)
1996  return failure();
1997 
1998  if (lbMap != prevLbMap)
1999  forOp.setLowerBound(lbOperands, lbMap);
2000  if (ubMap != prevUbMap)
2001  forOp.setUpperBound(ubOperands, ubMap);
2002  return success();
2003 }
2004 
2005 namespace {
2006 /// Returns constant trip count in trivial cases.
2007 static Optional<uint64_t> getTrivialConstantTripCount(AffineForOp forOp) {
2008  int64_t step = forOp.getStep();
2009  if (!forOp.hasConstantBounds() || step <= 0)
2010  return None;
2011  int64_t lb = forOp.getConstantLowerBound();
2012  int64_t ub = forOp.getConstantUpperBound();
2013  return ub - lb <= 0 ? 0 : (ub - lb + step - 1) / step;
2014 }
2015 
2016 /// This is a pattern to fold trivially empty loop bodies.
2017 /// TODO: This should be moved into the folding hook.
2018 struct AffineForEmptyLoopFolder : public OpRewritePattern<AffineForOp> {
2020 
2021  LogicalResult matchAndRewrite(AffineForOp forOp,
2022  PatternRewriter &rewriter) const override {
2023  // Check that the body only contains a yield.
2024  if (!llvm::hasSingleElement(*forOp.getBody()))
2025  return failure();
2026  if (forOp.getNumResults() == 0)
2027  return success();
2028  Optional<uint64_t> tripCount = getTrivialConstantTripCount(forOp);
2029  if (tripCount && *tripCount == 0) {
2030  // The initial values of the iteration arguments would be the op's
2031  // results.
2032  rewriter.replaceOp(forOp, forOp.getIterOperands());
2033  return success();
2034  }
2035  SmallVector<Value, 4> replacements;
2036  auto yieldOp = cast<AffineYieldOp>(forOp.getBody()->getTerminator());
2037  auto iterArgs = forOp.getRegionIterArgs();
2038  bool hasValDefinedOutsideLoop = false;
2039  bool iterArgsNotInOrder = false;
2040  for (unsigned i = 0, e = yieldOp->getNumOperands(); i < e; ++i) {
2041  Value val = yieldOp.getOperand(i);
2042  auto *iterArgIt = llvm::find(iterArgs, val);
2043  if (iterArgIt == iterArgs.end()) {
2044  // `val` is defined outside of the loop.
2045  assert(forOp.isDefinedOutsideOfLoop(val) &&
2046  "must be defined outside of the loop");
2047  hasValDefinedOutsideLoop = true;
2048  replacements.push_back(val);
2049  } else {
2050  unsigned pos = std::distance(iterArgs.begin(), iterArgIt);
2051  if (pos != i)
2052  iterArgsNotInOrder = true;
2053  replacements.push_back(forOp.getIterOperands()[pos]);
2054  }
2055  }
2056  // Bail out when the trip count is unknown and the loop returns any value
2057  // defined outside of the loop or any iterArg out of order.
2058  if (!tripCount.has_value() &&
2059  (hasValDefinedOutsideLoop || iterArgsNotInOrder))
2060  return failure();
2061  // Bail out when the loop iterates more than once and it returns any iterArg
2062  // out of order.
2063  if (tripCount.has_value() && tripCount.value() >= 2 && iterArgsNotInOrder)
2064  return failure();
2065  rewriter.replaceOp(forOp, replacements);
2066  return success();
2067  }
2068 };
2069 } // namespace
2070 
2071 void AffineForOp::getCanonicalizationPatterns(RewritePatternSet &results,
2072  MLIRContext *context) {
2073  results.add<AffineForEmptyLoopFolder>(context);
2074 }
2075 
2076 /// Return operands used when entering the region at 'index'. These operands
2077 /// correspond to the loop iterator operands, i.e., those excluding the
2078 /// induction variable. AffineForOp only has one region, so zero is the only
2079 /// valid value for `index`.
2080 OperandRange AffineForOp::getSuccessorEntryOperands(Optional<unsigned> index) {
2081  assert((!index || *index == 0) && "invalid region index");
2082 
2083  // The initial operands map to the loop arguments after the induction
2084  // variable or are forwarded to the results when the trip count is zero.
2085  return getIterOperands();
2086 }
2087 
2088 /// Given the region at `index`, or the parent operation if `index` is None,
2089 /// return the successor regions. These are the regions that may be selected
2090 /// during the flow of control. `operands` is a set of optional attributes that
2091 /// correspond to a constant value for each operand, or null if that operand is
2092 /// not a constant.
2093 void AffineForOp::getSuccessorRegions(
2094  Optional<unsigned> index, ArrayRef<Attribute> operands,
2096  assert((!index.has_value() || index.value() == 0) && "expected loop region");
2097  // The loop may typically branch back to its body or to the parent operation.
2098  // If the predecessor is the parent op and the trip count is known to be at
2099  // least one, branch into the body using the iterator arguments. And in cases
2100  // we know the trip count is zero, it can only branch back to its parent.
2101  Optional<uint64_t> tripCount = getTrivialConstantTripCount(*this);
2102  if (!index.has_value() && tripCount.has_value()) {
2103  if (tripCount.value() > 0) {
2104  regions.push_back(RegionSuccessor(&getLoopBody(), getRegionIterArgs()));
2105  return;
2106  }
2107  if (tripCount.value() == 0) {
2108  regions.push_back(RegionSuccessor(getResults()));
2109  return;
2110  }
2111  }
2112 
2113  // From the loop body, if the trip count is one, we can only branch back to
2114  // the parent.
2115  if (index && tripCount && *tripCount == 1) {
2116  regions.push_back(RegionSuccessor(getResults()));
2117  return;
2118  }
2119 
2120  // In all other cases, the loop may branch back to itself or the parent
2121  // operation.
2122  regions.push_back(RegionSuccessor(&getLoopBody(), getRegionIterArgs()));
2123  regions.push_back(RegionSuccessor(getResults()));
2124 }
2125 
2126 /// Returns true if the affine.for has zero iterations in trivial cases.
2127 static bool hasTrivialZeroTripCount(AffineForOp op) {
2128  Optional<uint64_t> tripCount = getTrivialConstantTripCount(op);
2129  return tripCount && *tripCount == 0;
2130 }
2131 
2132 LogicalResult AffineForOp::fold(ArrayRef<Attribute> operands,
2133  SmallVectorImpl<OpFoldResult> &results) {
2134  bool folded = succeeded(foldLoopBounds(*this));
2135  folded |= succeeded(canonicalizeLoopBounds(*this));
2136  if (hasTrivialZeroTripCount(*this)) {
2137  // The initial values of the loop-carried variables (iter_args) are the
2138  // results of the op.
2139  results.assign(getIterOperands().begin(), getIterOperands().end());
2140  folded = true;
2141  }
2142  return success(folded);
2143 }
2144 
2145 AffineBound AffineForOp::getLowerBound() {
2146  auto lbMap = getLowerBoundMap();
2147  return AffineBound(AffineForOp(*this), 0, lbMap.getNumInputs(), lbMap);
2148 }
2149 
2150 AffineBound AffineForOp::getUpperBound() {
2151  auto lbMap = getLowerBoundMap();
2152  auto ubMap = getUpperBoundMap();
2153  return AffineBound(AffineForOp(*this), lbMap.getNumInputs(),
2154  lbMap.getNumInputs() + ubMap.getNumInputs(), ubMap);
2155 }
2156 
2157 void AffineForOp::setLowerBound(ValueRange lbOperands, AffineMap map) {
2158  assert(lbOperands.size() == map.getNumInputs());
2159  assert(map.getNumResults() >= 1 && "bound map has at least one result");
2160 
2161  SmallVector<Value, 4> newOperands(lbOperands.begin(), lbOperands.end());
2162 
2163  auto ubOperands = getUpperBoundOperands();
2164  newOperands.append(ubOperands.begin(), ubOperands.end());
2165  auto iterOperands = getIterOperands();
2166  newOperands.append(iterOperands.begin(), iterOperands.end());
2167  (*this)->setOperands(newOperands);
2168 
2169  (*this)->setAttr(getLowerBoundAttrStrName(), AffineMapAttr::get(map));
2170 }
2171 
2172 void AffineForOp::setUpperBound(ValueRange ubOperands, AffineMap map) {
2173  assert(ubOperands.size() == map.getNumInputs());
2174  assert(map.getNumResults() >= 1 && "bound map has at least one result");
2175 
2177  newOperands.append(ubOperands.begin(), ubOperands.end());
2178  auto iterOperands = getIterOperands();
2179  newOperands.append(iterOperands.begin(), iterOperands.end());
2180  (*this)->setOperands(newOperands);
2181 
2182  (*this)->setAttr(getUpperBoundAttrStrName(), AffineMapAttr::get(map));
2183 }
2184 
2185 void AffineForOp::setLowerBoundMap(AffineMap map) {
2186  auto lbMap = getLowerBoundMap();
2187  assert(lbMap.getNumDims() == map.getNumDims() &&
2188  lbMap.getNumSymbols() == map.getNumSymbols());
2189  assert(map.getNumResults() >= 1 && "bound map has at least one result");
2190  (void)lbMap;
2191  (*this)->setAttr(getLowerBoundAttrStrName(), AffineMapAttr::get(map));
2192 }
2193 
2194 void AffineForOp::setUpperBoundMap(AffineMap map) {
2195  auto ubMap = getUpperBoundMap();
2196  assert(ubMap.getNumDims() == map.getNumDims() &&
2197  ubMap.getNumSymbols() == map.getNumSymbols());
2198  assert(map.getNumResults() >= 1 && "bound map has at least one result");
2199  (void)ubMap;
2200  (*this)->setAttr(getUpperBoundAttrStrName(), AffineMapAttr::get(map));
2201 }
2202 
2203 bool AffineForOp::hasConstantLowerBound() {
2204  return getLowerBoundMap().isSingleConstant();
2205 }
2206 
2207 bool AffineForOp::hasConstantUpperBound() {
2208  return getUpperBoundMap().isSingleConstant();
2209 }
2210 
2211 int64_t AffineForOp::getConstantLowerBound() {
2212  return getLowerBoundMap().getSingleConstantResult();
2213 }
2214 
2215 int64_t AffineForOp::getConstantUpperBound() {
2216  return getUpperBoundMap().getSingleConstantResult();
2217 }
2218 
2219 void AffineForOp::setConstantLowerBound(int64_t value) {
2220  setLowerBound({}, AffineMap::getConstantMap(value, getContext()));
2221 }
2222 
2223 void AffineForOp::setConstantUpperBound(int64_t value) {
2224  setUpperBound({}, AffineMap::getConstantMap(value, getContext()));
2225 }
2226 
2227 AffineForOp::operand_range AffineForOp::getLowerBoundOperands() {
2228  return {operand_begin(), operand_begin() + getLowerBoundMap().getNumInputs()};
2229 }
2230 
2231 AffineForOp::operand_range AffineForOp::getUpperBoundOperands() {
2232  return {operand_begin() + getLowerBoundMap().getNumInputs(),
2233  operand_begin() + getLowerBoundMap().getNumInputs() +
2234  getUpperBoundMap().getNumInputs()};
2235 }
2236 
2237 AffineForOp::operand_range AffineForOp::getControlOperands() {
2238  return {operand_begin(), operand_begin() + getLowerBoundMap().getNumInputs() +
2239  getUpperBoundMap().getNumInputs()};
2240 }
2241 
2242 bool AffineForOp::matchingBoundOperandList() {
2243  auto lbMap = getLowerBoundMap();
2244  auto ubMap = getUpperBoundMap();
2245  if (lbMap.getNumDims() != ubMap.getNumDims() ||
2246  lbMap.getNumSymbols() != ubMap.getNumSymbols())
2247  return false;
2248 
2249  unsigned numOperands = lbMap.getNumInputs();
2250  for (unsigned i = 0, e = lbMap.getNumInputs(); i < e; i++) {
2251  // Compare Value 's.
2252  if (getOperand(i) != getOperand(numOperands + i))
2253  return false;
2254  }
2255  return true;
2256 }
2257 
2258 Region &AffineForOp::getLoopBody() { return getRegion(); }
2259 
2260 Optional<Value> AffineForOp::getSingleInductionVar() {
2261  return getInductionVar();
2262 }
2263 
2264 Optional<OpFoldResult> AffineForOp::getSingleLowerBound() {
2265  if (!hasConstantLowerBound())
2266  return llvm::None;
2267  OpBuilder b(getContext());
2268  return OpFoldResult(b.getI64IntegerAttr(getConstantLowerBound()));
2269 }
2270 
2271 Optional<OpFoldResult> AffineForOp::getSingleStep() {
2272  OpBuilder b(getContext());
2273  return OpFoldResult(b.getI64IntegerAttr(getStep()));
2274 }
2275 
2276 Optional<OpFoldResult> AffineForOp::getSingleUpperBound() {
2277  if (!hasConstantUpperBound())
2278  return llvm::None;
2279  OpBuilder b(getContext());
2280  return OpFoldResult(b.getI64IntegerAttr(getConstantUpperBound()));
2281 }
2282 
2283 Speculation::Speculatability AffineForOp::getSpeculatability() {
2284  // `affine.for (I = Start; I < End; I += 1)` terminates for all values of
2285  // Start and End.
2286  //
2287  // For Step != 1, the loop may not terminate. We can add more smarts here if
2288  // needed.
2289  return getStep() == 1 ? Speculation::RecursivelySpeculatable
2291 }
2292 
2293 /// Returns true if the provided value is the induction variable of a
2294 /// AffineForOp.
2296  return getForInductionVarOwner(val) != AffineForOp();
2297 }
2298 
2299 /// Returns the loop parent of an induction variable. If the provided value is
2300 /// not an induction variable, then return nullptr.
2302  auto ivArg = val.dyn_cast<BlockArgument>();
2303  if (!ivArg || !ivArg.getOwner())
2304  return AffineForOp();
2305  auto *containingInst = ivArg.getOwner()->getParent()->getParentOp();
2306  if (auto forOp = dyn_cast<AffineForOp>(containingInst))
2307  // Check to make sure `val` is the induction variable, not an iter_arg.
2308  return forOp.getInductionVar() == val ? forOp : AffineForOp();
2309  return AffineForOp();
2310 }
2311 
2312 /// Extracts the induction variables from a list of AffineForOps and returns
2313 /// them.
2315  SmallVectorImpl<Value> *ivs) {
2316  ivs->reserve(forInsts.size());
2317  for (auto forInst : forInsts)
2318  ivs->push_back(forInst.getInductionVar());
2319 }
2320 
2321 /// Builds an affine loop nest, using "loopCreatorFn" to create individual loop
2322 /// operations.
2323 template <typename BoundListTy, typename LoopCreatorTy>
2325  OpBuilder &builder, Location loc, BoundListTy lbs, BoundListTy ubs,
2326  ArrayRef<int64_t> steps,
2327  function_ref<void(OpBuilder &, Location, ValueRange)> bodyBuilderFn,
2328  LoopCreatorTy &&loopCreatorFn) {
2329  assert(lbs.size() == ubs.size() && "Mismatch in number of arguments");
2330  assert(lbs.size() == steps.size() && "Mismatch in number of arguments");
2331 
2332  // If there are no loops to be constructed, construct the body anyway.
2333  OpBuilder::InsertionGuard guard(builder);
2334  if (lbs.empty()) {
2335  if (bodyBuilderFn)
2336  bodyBuilderFn(builder, loc, ValueRange());
2337  return;
2338  }
2339 
2340  // Create the loops iteratively and store the induction variables.
2342  ivs.reserve(lbs.size());
2343  for (unsigned i = 0, e = lbs.size(); i < e; ++i) {
2344  // Callback for creating the loop body, always creates the terminator.
2345  auto loopBody = [&](OpBuilder &nestedBuilder, Location nestedLoc, Value iv,
2346  ValueRange iterArgs) {
2347  ivs.push_back(iv);
2348  // In the innermost loop, call the body builder.
2349  if (i == e - 1 && bodyBuilderFn) {
2350  OpBuilder::InsertionGuard nestedGuard(nestedBuilder);
2351  bodyBuilderFn(nestedBuilder, nestedLoc, ivs);
2352  }
2353  nestedBuilder.create<AffineYieldOp>(nestedLoc);
2354  };
2355 
2356  // Delegate actual loop creation to the callback in order to dispatch
2357  // between constant- and variable-bound loops.
2358  auto loop = loopCreatorFn(builder, loc, lbs[i], ubs[i], steps[i], loopBody);
2359  builder.setInsertionPointToStart(loop.getBody());
2360  }
2361 }
2362 
2363 /// Creates an affine loop from the bounds known to be constants.
2364 static AffineForOp
2366  int64_t ub, int64_t step,
2367  AffineForOp::BodyBuilderFn bodyBuilderFn) {
2368  return builder.create<AffineForOp>(loc, lb, ub, step, /*iterArgs=*/llvm::None,
2369  bodyBuilderFn);
2370 }
2371 
2372 /// Creates an affine loop from the bounds that may or may not be constants.
2373 static AffineForOp
2375  int64_t step,
2376  AffineForOp::BodyBuilderFn bodyBuilderFn) {
2377  auto lbConst = lb.getDefiningOp<arith::ConstantIndexOp>();
2378  auto ubConst = ub.getDefiningOp<arith::ConstantIndexOp>();
2379  if (lbConst && ubConst)
2380  return buildAffineLoopFromConstants(builder, loc, lbConst.value(),
2381  ubConst.value(), step, bodyBuilderFn);
2382  return builder.create<AffineForOp>(loc, lb, builder.getDimIdentityMap(), ub,
2383  builder.getDimIdentityMap(), step,
2384  /*iterArgs=*/llvm::None, bodyBuilderFn);
2385 }
2386 
2388  OpBuilder &builder, Location loc, ArrayRef<int64_t> lbs,
2390  function_ref<void(OpBuilder &, Location, ValueRange)> bodyBuilderFn) {
2391  buildAffineLoopNestImpl(builder, loc, lbs, ubs, steps, bodyBuilderFn,
2393 }
2394 
2396  OpBuilder &builder, Location loc, ValueRange lbs, ValueRange ubs,
2397  ArrayRef<int64_t> steps,
2398  function_ref<void(OpBuilder &, Location, ValueRange)> bodyBuilderFn) {
2399  buildAffineLoopNestImpl(builder, loc, lbs, ubs, steps, bodyBuilderFn,
2401 }
2402 
2403 AffineForOp mlir::replaceForOpWithNewYields(OpBuilder &b, AffineForOp loop,
2404  ValueRange newIterOperands,
2405  ValueRange newYieldedValues,
2406  ValueRange newIterArgs,
2407  bool replaceLoopResults) {
2408  assert(newIterOperands.size() == newYieldedValues.size() &&
2409  "newIterOperands must be of the same size as newYieldedValues");
2410  // Create a new loop before the existing one, with the extra operands.
2412  b.setInsertionPoint(loop);
2413  auto operands = llvm::to_vector<4>(loop.getIterOperands());
2414  operands.append(newIterOperands.begin(), newIterOperands.end());
2415  SmallVector<Value, 4> lbOperands(loop.getLowerBoundOperands());
2416  SmallVector<Value, 4> ubOperands(loop.getUpperBoundOperands());
2417  SmallVector<Value, 4> steps(loop.getStep());
2418  auto lbMap = loop.getLowerBoundMap();
2419  auto ubMap = loop.getUpperBoundMap();
2420  AffineForOp newLoop =
2421  b.create<AffineForOp>(loop.getLoc(), lbOperands, lbMap, ubOperands, ubMap,
2422  loop.getStep(), operands);
2423  // Take the body of the original parent loop.
2424  newLoop.getLoopBody().takeBody(loop.getLoopBody());
2425  for (Value val : newIterArgs)
2426  newLoop.getLoopBody().addArgument(val.getType(), val.getLoc());
2427 
2428  // Update yield operation with new values to be added.
2429  if (!newYieldedValues.empty()) {
2430  auto yield = cast<AffineYieldOp>(newLoop.getBody()->getTerminator());
2431  b.setInsertionPoint(yield);
2432  auto yieldOperands = llvm::to_vector<4>(yield.getOperands());
2433  yieldOperands.append(newYieldedValues.begin(), newYieldedValues.end());
2434  b.create<AffineYieldOp>(yield.getLoc(), yieldOperands);
2435  yield.erase();
2436  }
2437  if (replaceLoopResults) {
2438  for (auto it : llvm::zip(loop.getResults(), newLoop.getResults().take_front(
2439  loop.getNumResults()))) {
2440  std::get<0>(it).replaceAllUsesWith(std::get<1>(it));
2441  }
2442  }
2443  return newLoop;
2444 }
2445 
2446 //===----------------------------------------------------------------------===//
2447 // AffineIfOp
2448 //===----------------------------------------------------------------------===//
2449 
2450 namespace {
2451 /// Remove else blocks that have nothing other than a zero value yield.
2452 struct SimplifyDeadElse : public OpRewritePattern<AffineIfOp> {
2454 
2455  LogicalResult matchAndRewrite(AffineIfOp ifOp,
2456  PatternRewriter &rewriter) const override {
2457  if (ifOp.getElseRegion().empty() ||
2458  !llvm::hasSingleElement(*ifOp.getElseBlock()) || ifOp.getNumResults())
2459  return failure();
2460 
2461  rewriter.startRootUpdate(ifOp);
2462  rewriter.eraseBlock(ifOp.getElseBlock());
2463  rewriter.finalizeRootUpdate(ifOp);
2464  return success();
2465  }
2466 };
2467 
2468 /// Removes affine.if cond if the condition is always true or false in certain
2469 /// trivial cases. Promotes the then/else block in the parent operation block.
2470 struct AlwaysTrueOrFalseIf : public OpRewritePattern<AffineIfOp> {
2472 
2473  LogicalResult matchAndRewrite(AffineIfOp op,
2474  PatternRewriter &rewriter) const override {
2475 
2476  auto isTriviallyFalse = [](IntegerSet iSet) {
2477  return iSet.isEmptyIntegerSet();
2478  };
2479 
2480  auto isTriviallyTrue = [](IntegerSet iSet) {
2481  return (iSet.getNumEqualities() == 1 && iSet.getNumInequalities() == 0 &&
2482  iSet.getConstraint(0) == 0);
2483  };
2484 
2485  IntegerSet affineIfConditions = op.getIntegerSet();
2486  Block *blockToMove;
2487  if (isTriviallyFalse(affineIfConditions)) {
2488  // The absence, or equivalently, the emptiness of the else region need not
2489  // be checked when affine.if is returning results because if an affine.if
2490  // operation is returning results, it always has a non-empty else region.
2491  if (op.getNumResults() == 0 && !op.hasElse()) {
2492  // If the else region is absent, or equivalently, empty, remove the
2493  // affine.if operation (which is not returning any results).
2494  rewriter.eraseOp(op);
2495  return success();
2496  }
2497  blockToMove = op.getElseBlock();
2498  } else if (isTriviallyTrue(affineIfConditions)) {
2499  blockToMove = op.getThenBlock();
2500  } else {
2501  return failure();
2502  }
2503  Operation *blockToMoveTerminator = blockToMove->getTerminator();
2504  // Promote the "blockToMove" block to the parent operation block between the
2505  // prologue and epilogue of "op".
2506  rewriter.mergeBlockBefore(blockToMove, op);
2507  // Replace the "op" operation with the operands of the
2508  // "blockToMoveTerminator" operation. Note that "blockToMoveTerminator" is
2509  // the affine.yield operation present in the "blockToMove" block. It has no
2510  // operands when affine.if is not returning results and therefore, in that
2511  // case, replaceOp just erases "op". When affine.if is not returning
2512  // results, the affine.yield operation can be omitted. It gets inserted
2513  // implicitly.
2514  rewriter.replaceOp(op, blockToMoveTerminator->getOperands());
2515  // Erase the "blockToMoveTerminator" operation since it is now in the parent
2516  // operation block, which already has its own terminator.
2517  rewriter.eraseOp(blockToMoveTerminator);
2518  return success();
2519  }
2520 };
2521 } // namespace
2522 
2523 /// AffineIfOp has two regions -- `then` and `else`. The flow of data should be
2524 /// as follows: AffineIfOp -> `then`/`else` -> AffineIfOp
2525 void AffineIfOp::getSuccessorRegions(
2526  Optional<unsigned> index, ArrayRef<Attribute> operands,
2528  // If the predecessor is an AffineIfOp, then branching into both `then` and
2529  // `else` region is valid.
2530  if (!index.has_value()) {
2531  regions.reserve(2);
2532  regions.push_back(
2533  RegionSuccessor(&getThenRegion(), getThenRegion().getArguments()));
2534  // Don't consider the else region if it is empty.
2535  if (!getElseRegion().empty())
2536  regions.push_back(
2537  RegionSuccessor(&getElseRegion(), getElseRegion().getArguments()));
2538  return;
2539  }
2540 
2541  // If the predecessor is the `else`/`then` region, then branching into parent
2542  // op is valid.
2543  regions.push_back(RegionSuccessor(getResults()));
2544 }
2545 
2547  // Verify that we have a condition attribute.
2548  // FIXME: This should be specified in the arguments list in ODS.
2549  auto conditionAttr =
2550  (*this)->getAttrOfType<IntegerSetAttr>(getConditionAttrStrName());
2551  if (!conditionAttr)
2552  return emitOpError("requires an integer set attribute named 'condition'");
2553 
2554  // Verify that there are enough operands for the condition.
2555  IntegerSet condition = conditionAttr.getValue();
2556  if (getNumOperands() != condition.getNumInputs())
2557  return emitOpError("operand count and condition integer set dimension and "
2558  "symbol count must match");
2559 
2560  // Verify that the operands are valid dimension/symbols.
2561  if (failed(verifyDimAndSymbolIdentifiers(*this, getOperands(),
2562  condition.getNumDims())))
2563  return failure();
2564 
2565  return success();
2566 }
2567 
2568 ParseResult AffineIfOp::parse(OpAsmParser &parser, OperationState &result) {
2569  // Parse the condition attribute set.
2570  IntegerSetAttr conditionAttr;
2571  unsigned numDims;
2572  if (parser.parseAttribute(conditionAttr,
2573  AffineIfOp::getConditionAttrStrName(),
2574  result.attributes) ||
2575  parseDimAndSymbolList(parser, result.operands, numDims))
2576  return failure();
2577 
2578  // Verify the condition operands.
2579  auto set = conditionAttr.getValue();
2580  if (set.getNumDims() != numDims)
2581  return parser.emitError(
2582  parser.getNameLoc(),
2583  "dim operand count and integer set dim count must match");
2584  if (numDims + set.getNumSymbols() != result.operands.size())
2585  return parser.emitError(
2586  parser.getNameLoc(),
2587  "symbol operand count and integer set symbol count must match");
2588 
2589  if (parser.parseOptionalArrowTypeList(result.types))
2590  return failure();
2591 
2592  // Create the regions for 'then' and 'else'. The latter must be created even
2593  // if it remains empty for the validity of the operation.
2594  result.regions.reserve(2);
2595  Region *thenRegion = result.addRegion();
2596  Region *elseRegion = result.addRegion();
2597 
2598  // Parse the 'then' region.
2599  if (parser.parseRegion(*thenRegion, {}, {}))
2600  return failure();
2601  AffineIfOp::ensureTerminator(*thenRegion, parser.getBuilder(),
2602  result.location);
2603 
2604  // If we find an 'else' keyword then parse the 'else' region.
2605  if (!parser.parseOptionalKeyword("else")) {
2606  if (parser.parseRegion(*elseRegion, {}, {}))
2607  return failure();
2608  AffineIfOp::ensureTerminator(*elseRegion, parser.getBuilder(),
2609  result.location);
2610  }
2611 
2612  // Parse the optional attribute list.
2613  if (parser.parseOptionalAttrDict(result.attributes))
2614  return failure();
2615 
2616  return success();
2617 }
2618 
2620  auto conditionAttr =
2621  (*this)->getAttrOfType<IntegerSetAttr>(getConditionAttrStrName());
2622  p << " " << conditionAttr;
2623  printDimAndSymbolList(operand_begin(), operand_end(),
2624  conditionAttr.getValue().getNumDims(), p);
2625  p.printOptionalArrowTypeList(getResultTypes());
2626  p << ' ';
2627  p.printRegion(getThenRegion(), /*printEntryBlockArgs=*/false,
2628  /*printBlockTerminators=*/getNumResults());
2629 
2630  // Print the 'else' regions if it has any blocks.
2631  auto &elseRegion = this->getElseRegion();
2632  if (!elseRegion.empty()) {
2633  p << " else ";
2634  p.printRegion(elseRegion,
2635  /*printEntryBlockArgs=*/false,
2636  /*printBlockTerminators=*/getNumResults());
2637  }
2638 
2639  // Print the attribute list.
2640  p.printOptionalAttrDict((*this)->getAttrs(),
2641  /*elidedAttrs=*/getConditionAttrStrName());
2642 }
2643 
2644 IntegerSet AffineIfOp::getIntegerSet() {
2645  return (*this)
2646  ->getAttrOfType<IntegerSetAttr>(getConditionAttrStrName())
2647  .getValue();
2648 }
2649 
2650 void AffineIfOp::setIntegerSet(IntegerSet newSet) {
2651  (*this)->setAttr(getConditionAttrStrName(), IntegerSetAttr::get(newSet));
2652 }
2653 
2654 void AffineIfOp::setConditional(IntegerSet set, ValueRange operands) {
2655  setIntegerSet(set);
2656  (*this)->setOperands(operands);
2657 }
2658 
2659 void AffineIfOp::build(OpBuilder &builder, OperationState &result,
2660  TypeRange resultTypes, IntegerSet set, ValueRange args,
2661  bool withElseRegion) {
2662  assert(resultTypes.empty() || withElseRegion);
2663  result.addTypes(resultTypes);
2664  result.addOperands(args);
2665  result.addAttribute(getConditionAttrStrName(), IntegerSetAttr::get(set));
2666 
2667  Region *thenRegion = result.addRegion();
2668  thenRegion->push_back(new Block());
2669  if (resultTypes.empty())
2670  AffineIfOp::ensureTerminator(*thenRegion, builder, result.location);
2671 
2672  Region *elseRegion = result.addRegion();
2673  if (withElseRegion) {
2674  elseRegion->push_back(new Block());
2675  if (resultTypes.empty())
2676  AffineIfOp::ensureTerminator(*elseRegion, builder, result.location);
2677  }
2678 }
2679 
2680 void AffineIfOp::build(OpBuilder &builder, OperationState &result,
2681  IntegerSet set, ValueRange args, bool withElseRegion) {
2682  AffineIfOp::build(builder, result, /*resultTypes=*/{}, set, args,
2683  withElseRegion);
2684 }
2685 
2686 /// Compose any affine.apply ops feeding into `operands` of the integer set
2687 /// `set` by composing the maps of such affine.apply ops with the integer
2688 /// set constraints.
2690  SmallVectorImpl<Value> &operands) {
2691  // We will simply reuse the API of the map composition by viewing the LHSs of
2692  // the equalities and inequalities of `set` as the affine exprs of an affine
2693  // map. Convert to equivalent map, compose, and convert back to set.
2694  auto map = AffineMap::get(set.getNumDims(), set.getNumSymbols(),
2695  set.getConstraints(), set.getContext());
2696  // Check if any composition is possible.
2697  if (llvm::none_of(operands,
2698  [](Value v) { return v.getDefiningOp<AffineApplyOp>(); }))
2699  return;
2700 
2701  composeAffineMapAndOperands(&map, &operands);
2702  set = IntegerSet::get(map.getNumDims(), map.getNumSymbols(), map.getResults(),
2703  set.getEqFlags());
2704 }
2705 
2706 /// Canonicalize an affine if op's conditional (integer set + operands).
2707 LogicalResult AffineIfOp::fold(ArrayRef<Attribute>,
2709  auto set = getIntegerSet();
2710  SmallVector<Value, 4> operands(getOperands());
2711  composeSetAndOperands(set, operands);
2712  canonicalizeSetAndOperands(&set, &operands);
2713 
2714  // Check if the canonicalization or composition led to any change.
2715  if (getIntegerSet() == set && llvm::equal(operands, getOperands()))
2716  return failure();
2717 
2718  setConditional(set, operands);
2719  return success();
2720 }
2721 
2722 void AffineIfOp::getCanonicalizationPatterns(RewritePatternSet &results,
2723  MLIRContext *context) {
2724  results.add<SimplifyDeadElse, AlwaysTrueOrFalseIf>(context);
2725 }
2726 
2727 //===----------------------------------------------------------------------===//
2728 // AffineLoadOp
2729 //===----------------------------------------------------------------------===//
2730 
2731 void AffineLoadOp::build(OpBuilder &builder, OperationState &result,
2732  AffineMap map, ValueRange operands) {
2733  assert(operands.size() == 1 + map.getNumInputs() && "inconsistent operands");
2734  result.addOperands(operands);
2735  if (map)
2736  result.addAttribute(getMapAttrStrName(), AffineMapAttr::get(map));
2737  auto memrefType = operands[0].getType().cast<MemRefType>();
2738  result.types.push_back(memrefType.getElementType());
2739 }
2740 
2741 void AffineLoadOp::build(OpBuilder &builder, OperationState &result,
2742  Value memref, AffineMap map, ValueRange mapOperands) {
2743  assert(map.getNumInputs() == mapOperands.size() && "inconsistent index info");
2744  result.addOperands(memref);
2745  result.addOperands(mapOperands);
2746  auto memrefType = memref.getType().cast<MemRefType>();
2747  result.addAttribute(getMapAttrStrName(), AffineMapAttr::get(map));
2748  result.types.push_back(memrefType.getElementType());
2749 }
2750 
2751 void AffineLoadOp::build(OpBuilder &builder, OperationState &result,
2752  Value memref, ValueRange indices) {
2753  auto memrefType = memref.getType().cast<MemRefType>();
2754  int64_t rank = memrefType.getRank();
2755  // Create identity map for memrefs with at least one dimension or () -> ()
2756  // for zero-dimensional memrefs.
2757  auto map =
2758  rank ? builder.getMultiDimIdentityMap(rank) : builder.getEmptyAffineMap();
2759  build(builder, result, memref, map, indices);
2760 }
2761 
2762 ParseResult AffineLoadOp::parse(OpAsmParser &parser, OperationState &result) {
2763  auto &builder = parser.getBuilder();
2764  auto indexTy = builder.getIndexType();
2765 
2766  MemRefType type;
2767  OpAsmParser::UnresolvedOperand memrefInfo;
2768  AffineMapAttr mapAttr;
2770  return failure(
2771  parser.parseOperand(memrefInfo) ||
2772  parser.parseAffineMapOfSSAIds(mapOperands, mapAttr,
2773  AffineLoadOp::getMapAttrStrName(),
2774  result.attributes) ||
2775  parser.parseOptionalAttrDict(result.attributes) ||
2776  parser.parseColonType(type) ||
2777  parser.resolveOperand(memrefInfo, type, result.operands) ||
2778  parser.resolveOperands(mapOperands, indexTy, result.operands) ||
2779  parser.addTypeToList(type.getElementType(), result.types));
2780 }
2781 
2783  p << " " << getMemRef() << '[';
2784  if (AffineMapAttr mapAttr =
2785  (*this)->getAttrOfType<AffineMapAttr>(getMapAttrStrName()))
2786  p.printAffineMapOfSSAIds(mapAttr, getMapOperands());
2787  p << ']';
2788  p.printOptionalAttrDict((*this)->getAttrs(),
2789  /*elidedAttrs=*/{getMapAttrStrName()});
2790  p << " : " << getMemRefType();
2791 }
2792 
2793 /// Verify common indexing invariants of affine.load, affine.store,
2794 /// affine.vector_load and affine.vector_store.
2795 static LogicalResult
2796 verifyMemoryOpIndexing(Operation *op, AffineMapAttr mapAttr,
2797  Operation::operand_range mapOperands,
2798  MemRefType memrefType, unsigned numIndexOperands) {
2799  if (mapAttr) {
2800  AffineMap map = mapAttr.getValue();
2801  if (map.getNumResults() != memrefType.getRank())
2802  return op->emitOpError("affine map num results must equal memref rank");
2803  if (map.getNumInputs() != numIndexOperands)
2804  return op->emitOpError("expects as many subscripts as affine map inputs");
2805  } else {
2806  if (memrefType.getRank() != numIndexOperands)
2807  return op->emitOpError(
2808  "expects the number of subscripts to be equal to memref rank");
2809  }
2810 
2811  Region *scope = getAffineScope(op);
2812  for (auto idx : mapOperands) {
2813  if (!idx.getType().isIndex())
2814  return op->emitOpError("index to load must have 'index' type");
2815  if (!isValidAffineIndexOperand(idx, scope))
2816  return op->emitOpError("index must be a dimension or symbol identifier");
2817  }
2818 
2819  return success();
2820 }
2821 
2823  auto memrefType = getMemRefType();
2824  if (getType() != memrefType.getElementType())
2825  return emitOpError("result type must match element type of memref");
2826 
2828  getOperation(),
2829  (*this)->getAttrOfType<AffineMapAttr>(getMapAttrStrName()),
2830  getMapOperands(), memrefType,
2831  /*numIndexOperands=*/getNumOperands() - 1)))
2832  return failure();
2833 
2834  return success();
2835 }
2836 
2837 void AffineLoadOp::getCanonicalizationPatterns(RewritePatternSet &results,
2838  MLIRContext *context) {
2839  results.add<SimplifyAffineOp<AffineLoadOp>>(context);
2840 }
2841 
2842 OpFoldResult AffineLoadOp::fold(ArrayRef<Attribute> cstOperands) {
2843  /// load(memrefcast) -> load
2844  if (succeeded(memref::foldMemRefCast(*this)))
2845  return getResult();
2846 
2847  // Fold load from a global constant memref.
2848  auto getGlobalOp = getMemref().getDefiningOp<memref::GetGlobalOp>();
2849  if (!getGlobalOp)
2850  return {};
2851  // Get to the memref.global defining the symbol.
2852  auto *symbolTableOp = getGlobalOp->getParentWithTrait<OpTrait::SymbolTable>();
2853  if (!symbolTableOp)
2854  return {};
2855  auto global = dyn_cast_or_null<memref::GlobalOp>(
2856  SymbolTable::lookupSymbolIn(symbolTableOp, getGlobalOp.getNameAttr()));
2857  if (!global)
2858  return {};
2859 
2860  // Check if the global memref is a constant.
2861  auto cstAttr =
2862  global.getConstantInitValue().dyn_cast_or_null<DenseElementsAttr>();
2863  if (!cstAttr)
2864  return {};
2865  // If it's a splat constant, we can fold irrespective of indices.
2866  if (auto splatAttr = cstAttr.dyn_cast<SplatElementsAttr>())
2867  return splatAttr.getSplatValue<Attribute>();
2868  // Otherwise, we can fold only if we know the indices.
2869  if (!getAffineMap().isConstant())
2870  return {};
2871  auto indices = llvm::to_vector<4>(
2872  llvm::map_range(getAffineMap().getConstantResults(),
2873  [](int64_t v) -> uint64_t { return v; }));
2874  return cstAttr.getValues<Attribute>()[indices];
2875 }
2876 
2877 //===----------------------------------------------------------------------===//
2878 // AffineStoreOp
2879 //===----------------------------------------------------------------------===//
2880 
2881 void AffineStoreOp::build(OpBuilder &builder, OperationState &result,
2882  Value valueToStore, Value memref, AffineMap map,
2883  ValueRange mapOperands) {
2884  assert(map.getNumInputs() == mapOperands.size() && "inconsistent index info");
2885  result.addOperands(valueToStore);
2886  result.addOperands(memref);
2887  result.addOperands(mapOperands);
2888  result.addAttribute(getMapAttrStrName(), AffineMapAttr::get(map));
2889 }
2890 
2891 // Use identity map.
2892 void AffineStoreOp::build(OpBuilder &builder, OperationState &result,
2893  Value valueToStore, Value memref,
2894  ValueRange indices) {
2895  auto memrefType = memref.getType().cast<MemRefType>();
2896  int64_t rank = memrefType.getRank();
2897  // Create identity map for memrefs with at least one dimension or () -> ()
2898  // for zero-dimensional memrefs.
2899  auto map =
2900  rank ? builder.getMultiDimIdentityMap(rank) : builder.getEmptyAffineMap();
2901  build(builder, result, valueToStore, memref, map, indices);
2902 }
2903 
2904 ParseResult AffineStoreOp::parse(OpAsmParser &parser, OperationState &result) {
2905  auto indexTy = parser.getBuilder().getIndexType();
2906 
2907  MemRefType type;
2908  OpAsmParser::UnresolvedOperand storeValueInfo;
2909  OpAsmParser::UnresolvedOperand memrefInfo;
2910  AffineMapAttr mapAttr;
2912  return failure(parser.parseOperand(storeValueInfo) || parser.parseComma() ||
2913  parser.parseOperand(memrefInfo) ||
2914  parser.parseAffineMapOfSSAIds(
2915  mapOperands, mapAttr, AffineStoreOp::getMapAttrStrName(),
2916  result.attributes) ||
2917  parser.parseOptionalAttrDict(result.attributes) ||
2918  parser.parseColonType(type) ||
2919  parser.resolveOperand(storeValueInfo, type.getElementType(),
2920  result.operands) ||
2921  parser.resolveOperand(memrefInfo, type, result.operands) ||
2922  parser.resolveOperands(mapOperands, indexTy, result.operands));
2923 }
2924 
2926  p << " " << getValueToStore();
2927  p << ", " << getMemRef() << '[';
2928  if (AffineMapAttr mapAttr =
2929  (*this)->getAttrOfType<AffineMapAttr>(getMapAttrStrName()))
2930  p.printAffineMapOfSSAIds(mapAttr, getMapOperands());
2931  p << ']';
2932  p.printOptionalAttrDict((*this)->getAttrs(),
2933  /*elidedAttrs=*/{getMapAttrStrName()});
2934  p << " : " << getMemRefType();
2935 }
2936 
2938  // The value to store must have the same type as memref element type.
2939  auto memrefType = getMemRefType();
2940  if (getValueToStore().getType() != memrefType.getElementType())
2941  return emitOpError(
2942  "value to store must have the same type as memref element type");
2943 
2945  getOperation(),
2946  (*this)->getAttrOfType<AffineMapAttr>(getMapAttrStrName()),
2947  getMapOperands(), memrefType,
2948  /*numIndexOperands=*/getNumOperands() - 2)))
2949  return failure();
2950 
2951  return success();
2952 }
2953 
2954 void AffineStoreOp::getCanonicalizationPatterns(RewritePatternSet &results,
2955  MLIRContext *context) {
2956  results.add<SimplifyAffineOp<AffineStoreOp>>(context);
2957 }
2958 
2959 LogicalResult AffineStoreOp::fold(ArrayRef<Attribute> cstOperands,
2960  SmallVectorImpl<OpFoldResult> &results) {
2961  /// store(memrefcast) -> store
2962  return memref::foldMemRefCast(*this, getValueToStore());
2963 }
2964 
2965 //===----------------------------------------------------------------------===//
2966 // AffineMinMaxOpBase
2967 //===----------------------------------------------------------------------===//
2968 
2969 template <typename T>
2971  // Verify that operand count matches affine map dimension and symbol count.
2972  if (op.getNumOperands() !=
2973  op.getMap().getNumDims() + op.getMap().getNumSymbols())
2974  return op.emitOpError(
2975  "operand count and affine map dimension and symbol count must match");
2976  return success();
2977 }
2978 
2979 template <typename T>
2980 static void printAffineMinMaxOp(OpAsmPrinter &p, T op) {
2981  p << ' ' << op->getAttr(T::getMapAttrStrName());
2982  auto operands = op.getOperands();
2983  unsigned numDims = op.getMap().getNumDims();
2984  p << '(' << operands.take_front(numDims) << ')';
2985 
2986  if (operands.size() != numDims)
2987  p << '[' << operands.drop_front(numDims) << ']';
2988  p.printOptionalAttrDict(op->getAttrs(),
2989  /*elidedAttrs=*/{T::getMapAttrStrName()});
2990 }
2991 
2992 template <typename T>
2994  OperationState &result) {
2995  auto &builder = parser.getBuilder();
2996  auto indexType = builder.getIndexType();
2999  AffineMapAttr mapAttr;
3000  return failure(
3001  parser.parseAttribute(mapAttr, T::getMapAttrStrName(),
3002  result.attributes) ||
3003  parser.parseOperandList(dimInfos, OpAsmParser::Delimiter::Paren) ||
3004  parser.parseOperandList(symInfos,
3006  parser.parseOptionalAttrDict(result.attributes) ||
3007  parser.resolveOperands(dimInfos, indexType, result.operands) ||
3008  parser.resolveOperands(symInfos, indexType, result.operands) ||
3009  parser.addTypeToList(indexType, result.types));
3010 }
3011 
3012 /// Fold an affine min or max operation with the given operands. The operand
3013 /// list may contain nulls, which are interpreted as the operand not being a
3014 /// constant.
3015 template <typename T>
3018  "expected affine min or max op");
3019 
3020  // Fold the affine map.
3021  // TODO: Fold more cases:
3022  // min(some_affine, some_affine + constant, ...), etc.
3023  SmallVector<int64_t, 2> results;
3024  auto foldedMap = op.getMap().partialConstantFold(operands, &results);
3025 
3026  // If some of the map results are not constant, try changing the map in-place.
3027  if (results.empty()) {
3028  // If the map is the same, report that folding did not happen.
3029  if (foldedMap == op.getMap())
3030  return {};
3031  op->setAttr("map", AffineMapAttr::get(foldedMap));
3032  return op.getResult();
3033  }
3034 
3035  // Otherwise, completely fold the op into a constant.
3036  auto resultIt = std::is_same<T, AffineMinOp>::value
3037  ? std::min_element(results.begin(), results.end())
3038  : std::max_element(results.begin(), results.end());
3039  if (resultIt == results.end())
3040  return {};
3041  return IntegerAttr::get(IndexType::get(op.getContext()), *resultIt);
3042 }
3043 
3044 /// Remove duplicated expressions in affine min/max ops.
3045 template <typename T>
3048 
3050  PatternRewriter &rewriter) const override {
3051  AffineMap oldMap = affineOp.getAffineMap();
3052 
3053  SmallVector<AffineExpr, 4> newExprs;
3054  for (AffineExpr expr : oldMap.getResults()) {
3055  // This is a linear scan over newExprs, but it should be fine given that
3056  // we typically just have a few expressions per op.
3057  if (!llvm::is_contained(newExprs, expr))
3058  newExprs.push_back(expr);
3059  }
3060 
3061  if (newExprs.size() == oldMap.getNumResults())
3062  return failure();
3063 
3064  auto newMap = AffineMap::get(oldMap.getNumDims(), oldMap.getNumSymbols(),
3065  newExprs, rewriter.getContext());
3066  rewriter.replaceOpWithNewOp<T>(affineOp, newMap, affineOp.getMapOperands());
3067 
3068  return success();
3069  }
3070 };
3071 
3072 /// Merge an affine min/max op to its consumers if its consumer is also an
3073 /// affine min/max op.
3074 ///
3075 /// This pattern requires the producer affine min/max op is bound to a
3076 /// dimension/symbol that is used as a standalone expression in the consumer
3077 /// affine op's map.
3078 ///
3079 /// For example, a pattern like the following:
3080 ///
3081 /// %0 = affine.min affine_map<()[s0] -> (s0 + 16, s0 * 8)> ()[%sym1]
3082 /// %1 = affine.min affine_map<(d0)[s0] -> (s0 + 4, d0)> (%0)[%sym2]
3083 ///
3084 /// Can be turned into:
3085 ///
3086 /// %1 = affine.min affine_map<
3087 /// ()[s0, s1] -> (s0 + 4, s1 + 16, s1 * 8)> ()[%sym2, %sym1]
3088 template <typename T>
3091 
3093  PatternRewriter &rewriter) const override {
3094  AffineMap oldMap = affineOp.getAffineMap();
3095  ValueRange dimOperands =
3096  affineOp.getMapOperands().take_front(oldMap.getNumDims());
3097  ValueRange symOperands =
3098  affineOp.getMapOperands().take_back(oldMap.getNumSymbols());
3099 
3100  auto newDimOperands = llvm::to_vector<8>(dimOperands);
3101  auto newSymOperands = llvm::to_vector<8>(symOperands);
3102  SmallVector<AffineExpr, 4> newExprs;
3103  SmallVector<T, 4> producerOps;
3104 
3105  // Go over each expression to see whether it's a single dimension/symbol
3106  // with the corresponding operand which is the result of another affine
3107  // min/max op. If So it can be merged into this affine op.
3108  for (AffineExpr expr : oldMap.getResults()) {
3109  if (auto symExpr = expr.dyn_cast<AffineSymbolExpr>()) {
3110  Value symValue = symOperands[symExpr.getPosition()];
3111  if (auto producerOp = symValue.getDefiningOp<T>()) {
3112  producerOps.push_back(producerOp);
3113  continue;
3114  }
3115  } else if (auto dimExpr = expr.dyn_cast<AffineDimExpr>()) {
3116  Value dimValue = dimOperands[dimExpr.getPosition()];
3117  if (auto producerOp = dimValue.getDefiningOp<T>()) {
3118  producerOps.push_back(producerOp);
3119  continue;
3120  }
3121  }
3122  // For the above cases we will remove the expression by merging the
3123  // producer affine min/max's affine expressions. Otherwise we need to
3124  // keep the existing expression.
3125  newExprs.push_back(expr);
3126  }
3127 
3128  if (producerOps.empty())
3129  return failure();
3130 
3131  unsigned numUsedDims = oldMap.getNumDims();
3132  unsigned numUsedSyms = oldMap.getNumSymbols();
3133 
3134  // Now go over all producer affine ops and merge their expressions.
3135  for (T producerOp : producerOps) {
3136  AffineMap producerMap = producerOp.getAffineMap();
3137  unsigned numProducerDims = producerMap.getNumDims();
3138  unsigned numProducerSyms = producerMap.getNumSymbols();
3139 
3140  // Collect all dimension/symbol values.
3141  ValueRange dimValues =
3142  producerOp.getMapOperands().take_front(numProducerDims);
3143  ValueRange symValues =
3144  producerOp.getMapOperands().take_back(numProducerSyms);
3145  newDimOperands.append(dimValues.begin(), dimValues.end());
3146  newSymOperands.append(symValues.begin(), symValues.end());
3147 
3148  // For expressions we need to shift to avoid overlap.
3149  for (AffineExpr expr : producerMap.getResults()) {
3150  newExprs.push_back(expr.shiftDims(numProducerDims, numUsedDims)
3151  .shiftSymbols(numProducerSyms, numUsedSyms));
3152  }
3153 
3154  numUsedDims += numProducerDims;
3155  numUsedSyms += numProducerSyms;
3156  }
3157 
3158  auto newMap = AffineMap::get(numUsedDims, numUsedSyms, newExprs,
3159  rewriter.getContext());
3160  auto newOperands =
3161  llvm::to_vector<8>(llvm::concat<Value>(newDimOperands, newSymOperands));
3162  rewriter.replaceOpWithNewOp<T>(affineOp, newMap, newOperands);
3163 
3164  return success();
3165  }
3166 };
3167 
3168 /// Canonicalize the result expression order of an affine map and return success
3169 /// if the order changed.
3170 ///
3171 /// The function flattens the map's affine expressions to coefficient arrays and
3172 /// sorts them in lexicographic order. A coefficient array contains a multiplier
3173 /// for every dimension/symbol and a constant term. The canonicalization fails
3174 /// if a result expression is not pure or if the flattening requires local
3175 /// variables that, unlike dimensions and symbols, have no global order.
3177  SmallVector<SmallVector<int64_t>> flattenedExprs;
3178  for (const AffineExpr &resultExpr : map.getResults()) {
3179  // Fail if the expression is not pure.
3180  if (!resultExpr.isPureAffine())
3181  return failure();
3182 
3183  SimpleAffineExprFlattener flattener(map.getNumDims(), map.getNumSymbols());
3184  flattener.walkPostOrder(resultExpr);
3185 
3186  // Fail if the flattened expression has local variables.
3187  if (flattener.operandExprStack.back().size() !=
3188  map.getNumDims() + map.getNumSymbols() + 1)
3189  return failure();
3190 
3191  flattenedExprs.emplace_back(flattener.operandExprStack.back().begin(),
3192  flattener.operandExprStack.back().end());
3193  }
3194 
3195  // Fail if sorting is not necessary.
3196  if (llvm::is_sorted(flattenedExprs))
3197  return failure();
3198 
3199  // Reorder the result expressions according to their flattened form.
3200  SmallVector<unsigned> resultPermutation =
3201  llvm::to_vector(llvm::seq<unsigned>(0, map.getNumResults()));
3202  llvm::sort(resultPermutation, [&](unsigned lhs, unsigned rhs) {
3203  return flattenedExprs[lhs] < flattenedExprs[rhs];
3204  });
3205  SmallVector<AffineExpr> newExprs;
3206  for (unsigned idx : resultPermutation)
3207  newExprs.push_back(map.getResult(idx));
3208 
3209  map = AffineMap::get(map.getNumDims(), map.getNumSymbols(), newExprs,
3210  map.getContext());
3211  return success();
3212 }
3213 
3214 /// Canonicalize the affine map result expression order of an affine min/max
3215 /// operation.
3216 ///
3217 /// The pattern calls `canonicalizeMapExprAndTermOrder` to order the result
3218 /// expressions and replaces the operation if the order changed.
3219 ///
3220 /// For example, the following operation:
3221 ///
3222 /// %0 = affine.min affine_map<(d0, d1) -> (d0 + d1, d1 + 16, 32)> (%i0, %i1)
3223 ///
3224 /// Turns into:
3225 ///
3226 /// %0 = affine.min affine_map<(d0, d1) -> (32, d1 + 16, d0 + d1)> (%i0, %i1)
3227 template <typename T>
3230 
3232  PatternRewriter &rewriter) const override {
3233  AffineMap map = affineOp.getAffineMap();
3235  return failure();
3236 
3237  rewriter.replaceOpWithNewOp<T>(affineOp, map, affineOp.getMapOperands());
3238  return success();
3239  }
3240 };
3241 
3242 template <typename T>
3245 
3247  PatternRewriter &rewriter) const override {
3248  if (affineOp.getMap().getNumResults() != 1)
3249  return failure();
3250  rewriter.replaceOpWithNewOp<AffineApplyOp>(affineOp, affineOp.getMap(),
3251  affineOp.getOperands());
3252  return success();
3253  }
3254 };
3255 
3256 //===----------------------------------------------------------------------===//
3257 // AffineMinOp
3258 //===----------------------------------------------------------------------===//
3259 //
3260 // %0 = affine.min (d0) -> (1000, d0 + 512) (%i0)
3261 //
3262 
3263 OpFoldResult AffineMinOp::fold(ArrayRef<Attribute> operands) {
3264  return foldMinMaxOp(*this, operands);
3265 }
3266 
3267 void AffineMinOp::getCanonicalizationPatterns(RewritePatternSet &patterns,
3268  MLIRContext *context) {
3271  MergeAffineMinMaxOp<AffineMinOp>, SimplifyAffineOp<AffineMinOp>,
3273  context);
3274 }
3275 
3277 
3278 ParseResult AffineMinOp::parse(OpAsmParser &parser, OperationState &result) {
3279  return parseAffineMinMaxOp<AffineMinOp>(parser, result);
3280 }
3281 
3282 void AffineMinOp::print(OpAsmPrinter &p) { printAffineMinMaxOp(p, *this); }
3283 
3284 //===----------------------------------------------------------------------===//
3285 // AffineMaxOp
3286 //===----------------------------------------------------------------------===//
3287 //
3288 // %0 = affine.max (d0) -> (1000, d0 + 512) (%i0)
3289 //
3290 
3291 OpFoldResult AffineMaxOp::fold(ArrayRef<Attribute> operands) {
3292  return foldMinMaxOp(*this, operands);
3293 }
3294 
3295 void AffineMaxOp::getCanonicalizationPatterns(RewritePatternSet &patterns,
3296  MLIRContext *context) {
3299  MergeAffineMinMaxOp<AffineMaxOp>, SimplifyAffineOp<AffineMaxOp>,
3301  context);
3302 }
3303 
3305 
3306 ParseResult AffineMaxOp::parse(OpAsmParser &parser, OperationState &result) {
3307  return parseAffineMinMaxOp<AffineMaxOp>(parser, result);
3308 }
3309 
3310 void AffineMaxOp::print(OpAsmPrinter &p) { printAffineMinMaxOp(p, *this); }
3311 
3312 //===----------------------------------------------------------------------===//
3313 // AffinePrefetchOp
3314 //===----------------------------------------------------------------------===//
3315 
3316 //
3317 // affine.prefetch %0[%i, %j + 5], read, locality<3>, data : memref<400x400xi32>
3318 //
3319 ParseResult AffinePrefetchOp::parse(OpAsmParser &parser,
3320  OperationState &result) {
3321  auto &builder = parser.getBuilder();
3322  auto indexTy = builder.getIndexType();
3323 
3324  MemRefType type;
3325  OpAsmParser::UnresolvedOperand memrefInfo;
3326  IntegerAttr hintInfo;
3327  auto i32Type = parser.getBuilder().getIntegerType(32);
3328  StringRef readOrWrite, cacheType;
3329 
3330  AffineMapAttr mapAttr;
3332  if (parser.parseOperand(memrefInfo) ||
3333  parser.parseAffineMapOfSSAIds(mapOperands, mapAttr,
3334  AffinePrefetchOp::getMapAttrStrName(),
3335  result.attributes) ||
3336  parser.parseComma() || parser.parseKeyword(&readOrWrite) ||
3337  parser.parseComma() || parser.parseKeyword("locality") ||
3338  parser.parseLess() ||
3339  parser.parseAttribute(hintInfo, i32Type,
3340  AffinePrefetchOp::getLocalityHintAttrStrName(),
3341  result.attributes) ||
3342  parser.parseGreater() || parser.parseComma() ||
3343  parser.parseKeyword(&cacheType) ||
3344  parser.parseOptionalAttrDict(result.attributes) ||
3345  parser.parseColonType(type) ||
3346  parser.resolveOperand(memrefInfo, type, result.operands) ||
3347  parser.resolveOperands(mapOperands, indexTy, result.operands))
3348  return failure();
3349 
3350  if (!readOrWrite.equals("read") && !readOrWrite.equals("write"))
3351  return parser.emitError(parser.getNameLoc(),
3352  "rw specifier has to be 'read' or 'write'");
3353  result.addAttribute(
3354  AffinePrefetchOp::getIsWriteAttrStrName(),
3355  parser.getBuilder().getBoolAttr(readOrWrite.equals("write")));
3356 
3357  if (!cacheType.equals("data") && !cacheType.equals("instr"))
3358  return parser.emitError(parser.getNameLoc(),
3359  "cache type has to be 'data' or 'instr'");
3360 
3361  result.addAttribute(
3362  AffinePrefetchOp::getIsDataCacheAttrStrName(),
3363  parser.getBuilder().getBoolAttr(cacheType.equals("data")));
3364 
3365  return success();
3366 }
3367 
3369  p << " " << getMemref() << '[';
3370  AffineMapAttr mapAttr =
3371  (*this)->getAttrOfType<AffineMapAttr>(getMapAttrStrName());
3372  if (mapAttr)
3373  p.printAffineMapOfSSAIds(mapAttr, getMapOperands());
3374  p << ']' << ", " << (getIsWrite() ? "write" : "read") << ", "
3375  << "locality<" << getLocalityHint() << ">, "
3376  << (getIsDataCache() ? "data" : "instr");
3378  (*this)->getAttrs(),
3379  /*elidedAttrs=*/{getMapAttrStrName(), getLocalityHintAttrStrName(),
3380  getIsDataCacheAttrStrName(), getIsWriteAttrStrName()});
3381  p << " : " << getMemRefType();
3382 }
3383 
3385  auto mapAttr = (*this)->getAttrOfType<AffineMapAttr>(getMapAttrStrName());
3386  if (mapAttr) {
3387  AffineMap map = mapAttr.getValue();
3388  if (map.getNumResults() != getMemRefType().getRank())
3389  return emitOpError("affine.prefetch affine map num results must equal"
3390  " memref rank");
3391  if (map.getNumInputs() + 1 != getNumOperands())
3392  return emitOpError("too few operands");
3393  } else {
3394  if (getNumOperands() != 1)
3395  return emitOpError("too few operands");
3396  }
3397 
3398  Region *scope = getAffineScope(*this);
3399  for (auto idx : getMapOperands()) {
3400  if (!isValidAffineIndexOperand(idx, scope))
3401  return emitOpError("index must be a dimension or symbol identifier");
3402  }
3403  return success();
3404 }
3405 
3406 void AffinePrefetchOp::getCanonicalizationPatterns(RewritePatternSet &results,
3407  MLIRContext *context) {
3408  // prefetch(memrefcast) -> prefetch
3409  results.add<SimplifyAffineOp<AffinePrefetchOp>>(context);
3410 }
3411 
3412 LogicalResult AffinePrefetchOp::fold(ArrayRef<Attribute> cstOperands,
3413  SmallVectorImpl<OpFoldResult> &results) {
3414  /// prefetch(memrefcast) -> prefetch
3415  return memref::foldMemRefCast(*this);
3416 }
3417 
3418 //===----------------------------------------------------------------------===//
3419 // AffineParallelOp
3420 //===----------------------------------------------------------------------===//
3421 
3422 void AffineParallelOp::build(OpBuilder &builder, OperationState &result,
3423  TypeRange resultTypes,
3424  ArrayRef<arith::AtomicRMWKind> reductions,
3425  ArrayRef<int64_t> ranges) {
3426  SmallVector<AffineMap> lbs(ranges.size(), builder.getConstantAffineMap(0));
3427  auto ubs = llvm::to_vector<4>(llvm::map_range(ranges, [&](int64_t value) {
3428  return builder.getConstantAffineMap(value);
3429  }));
3430  SmallVector<int64_t> steps(ranges.size(), 1);
3431  build(builder, result, resultTypes, reductions, lbs, /*lbArgs=*/{}, ubs,
3432  /*ubArgs=*/{}, steps);
3433 }
3434 
3435 void AffineParallelOp::build(OpBuilder &builder, OperationState &result,
3436  TypeRange resultTypes,
3437  ArrayRef<arith::AtomicRMWKind> reductions,
3438  ArrayRef<AffineMap> lbMaps, ValueRange lbArgs,
3439  ArrayRef<AffineMap> ubMaps, ValueRange ubArgs,
3440  ArrayRef<int64_t> steps) {
3441  assert(llvm::all_of(lbMaps,
3442  [lbMaps](AffineMap m) {
3443  return m.getNumDims() == lbMaps[0].getNumDims() &&
3444  m.getNumSymbols() == lbMaps[0].getNumSymbols();
3445  }) &&
3446  "expected all lower bounds maps to have the same number of dimensions "
3447  "and symbols");
3448  assert(llvm::all_of(ubMaps,
3449  [ubMaps](AffineMap m) {
3450  return m.getNumDims() == ubMaps[0].getNumDims() &&
3451  m.getNumSymbols() == ubMaps[0].getNumSymbols();
3452  }) &&
3453  "expected all upper bounds maps to have the same number of dimensions "
3454  "and symbols");
3455  assert((lbMaps.empty() || lbMaps[0].getNumInputs() == lbArgs.size()) &&
3456  "expected lower bound maps to have as many inputs as lower bound "
3457  "operands");
3458  assert((ubMaps.empty() || ubMaps[0].getNumInputs() == ubArgs.size()) &&
3459  "expected upper bound maps to have as many inputs as upper bound "
3460  "operands");
3461 
3462  result.addTypes(resultTypes);
3463 
3464  // Convert the reductions to integer attributes.
3465  SmallVector<Attribute, 4> reductionAttrs;
3466  for (arith::AtomicRMWKind reduction : reductions)
3467  reductionAttrs.push_back(
3468  builder.getI64IntegerAttr(static_cast<int64_t>(reduction)));
3469  result.addAttribute(getReductionsAttrStrName(),
3470  builder.getArrayAttr(reductionAttrs));
3471 
3472  // Concatenates maps defined in the same input space (same dimensions and
3473  // symbols), assumes there is at least one map.
3474  auto concatMapsSameInput = [&builder](ArrayRef<AffineMap> maps,
3475  SmallVectorImpl<int32_t> &groups) {
3476  if (maps.empty())
3477  return AffineMap::get(builder.getContext());
3479  groups.reserve(groups.size() + maps.size());
3480  exprs.reserve(maps.size());
3481  for (AffineMap m : maps) {
3482  llvm::append_range(exprs, m.getResults());
3483  groups.push_back(m.getNumResults());
3484  }
3485  return AffineMap::get(maps[0].getNumDims(), maps[0].getNumSymbols(), exprs,
3486  maps[0].getContext());
3487  };
3488 
3489  // Set up the bounds.
3490  SmallVector<int32_t> lbGroups, ubGroups;
3491  AffineMap lbMap = concatMapsSameInput(lbMaps, lbGroups);
3492  AffineMap ubMap = concatMapsSameInput(ubMaps, ubGroups);
3493  result.addAttribute(getLowerBoundsMapAttrStrName(),
3494  AffineMapAttr::get(lbMap));
3495  result.addAttribute(getLowerBoundsGroupsAttrStrName(),
3496  builder.getI32TensorAttr(lbGroups));
3497  result.addAttribute(getUpperBoundsMapAttrStrName(),
3498  AffineMapAttr::get(ubMap));
3499  result.addAttribute(getUpperBoundsGroupsAttrStrName(),
3500  builder.getI32TensorAttr(ubGroups));
3501  result.addAttribute(getStepsAttrStrName(), builder.getI64ArrayAttr(steps));
3502  result.addOperands(lbArgs);
3503  result.addOperands(ubArgs);
3504 
3505  // Create a region and a block for the body.
3506  auto *bodyRegion = result.addRegion();
3507  auto *body = new Block();
3508  // Add all the block arguments.
3509  for (unsigned i = 0, e = steps.size(); i < e; ++i)
3510  body->addArgument(IndexType::get(builder.getContext()), result.location);
3511  bodyRegion->push_back(body);
3512  if (resultTypes.empty())
3513  ensureTerminator(*bodyRegion, builder, result.location);
3514 }
3515 
3516 Region &AffineParallelOp::getLoopBody() { return getRegion(); }
3517 
3518 unsigned AffineParallelOp::getNumDims() { return getSteps().size(); }
3519 
3520 AffineParallelOp::operand_range AffineParallelOp::getLowerBoundsOperands() {
3521  return getOperands().take_front(getLowerBoundsMap().getNumInputs());
3522 }
3523 
3524 AffineParallelOp::operand_range AffineParallelOp::getUpperBoundsOperands() {
3525  return getOperands().drop_front(getLowerBoundsMap().getNumInputs());
3526 }
3527 
3528 AffineMap AffineParallelOp::getLowerBoundMap(unsigned pos) {
3529  auto values = getLowerBoundsGroups().getValues<int32_t>();
3530  unsigned start = 0;
3531  for (unsigned i = 0; i < pos; ++i)
3532  start += values[i];
3533  return getLowerBoundsMap().getSliceMap(start, values[pos]);
3534 }
3535 
3536 AffineMap AffineParallelOp::getUpperBoundMap(unsigned pos) {
3537  auto values = getUpperBoundsGroups().getValues<int32_t>();
3538  unsigned start = 0;
3539  for (unsigned i = 0; i < pos; ++i)
3540  start += values[i];
3541  return getUpperBoundsMap().getSliceMap(start, values[pos]);
3542 }
3543 
3544 AffineValueMap AffineParallelOp::getLowerBoundsValueMap() {
3545  return AffineValueMap(getLowerBoundsMap(), getLowerBoundsOperands());
3546 }
3547 
3548 AffineValueMap AffineParallelOp::getUpperBoundsValueMap() {
3549  return AffineValueMap(getUpperBoundsMap(), getUpperBoundsOperands());
3550 }
3551 
3552 Optional<SmallVector<int64_t, 8>> AffineParallelOp::getConstantRanges() {
3553  if (hasMinMaxBounds())
3554  return llvm::None;
3555 
3556  // Try to convert all the ranges to constant expressions.
3558  AffineValueMap rangesValueMap;
3559  AffineValueMap::difference(getUpperBoundsValueMap(), getLowerBoundsValueMap(),
3560  &rangesValueMap);
3561  out.reserve(rangesValueMap.getNumResults());
3562  for (unsigned i = 0, e = rangesValueMap.getNumResults(); i < e; ++i) {
3563  auto expr = rangesValueMap.getResult(i);
3564  auto cst = expr.dyn_cast<AffineConstantExpr>();
3565  if (!cst)
3566  return llvm::None;
3567  out.push_back(cst.getValue());
3568  }
3569  return out;
3570 }
3571 
3572 Block *AffineParallelOp::getBody() { return &getRegion().front(); }
3573 
3574 OpBuilder AffineParallelOp::getBodyBuilder() {
3575  return OpBuilder(getBody(), std::prev(getBody()->end()));
3576 }
3577 
3578 void AffineParallelOp::setLowerBounds(ValueRange lbOperands, AffineMap map) {
3579  assert(lbOperands.size() == map.getNumInputs() &&
3580  "operands to map must match number of inputs");
3581 
3582  auto ubOperands = getUpperBoundsOperands();
3583 
3584  SmallVector<Value, 4> newOperands(lbOperands);
3585  newOperands.append(ubOperands.begin(), ubOperands.end());
3586  (*this)->setOperands(newOperands);
3587 
3588  setLowerBoundsMapAttr(AffineMapAttr::get(map));
3589 }
3590 
3591 void AffineParallelOp::setUpperBounds(ValueRange ubOperands, AffineMap map) {
3592  assert(ubOperands.size() == map.getNumInputs() &&
3593  "operands to map must match number of inputs");
3594 
3595  SmallVector<Value, 4> newOperands(getLowerBoundsOperands());
3596  newOperands.append(ubOperands.begin(), ubOperands.end());
3597  (*this)->setOperands(newOperands);
3598 
3599  setUpperBoundsMapAttr(AffineMapAttr::get(map));
3600 }
3601 
3602 void AffineParallelOp::setSteps(ArrayRef<int64_t> newSteps) {
3603  setStepsAttr(getBodyBuilder().getI64ArrayAttr(newSteps));
3604 }
3605 
3607  auto numDims = getNumDims();
3608  if (getLowerBoundsGroups().getNumElements() != numDims ||
3609  getUpperBoundsGroups().getNumElements() != numDims ||
3610  getSteps().size() != numDims || getBody()->getNumArguments() != numDims) {
3611  return emitOpError() << "the number of region arguments ("
3612  << getBody()->getNumArguments()
3613  << ") and the number of map groups for lower ("
3614  << getLowerBoundsGroups().getNumElements()
3615  << ") and upper bound ("
3616  << getUpperBoundsGroups().getNumElements()
3617  << "), and the number of steps (" << getSteps().size()
3618  << ") must all match";
3619  }
3620 
3621  unsigned expectedNumLBResults = 0;
3622  for (APInt v : getLowerBoundsGroups())
3623  expectedNumLBResults += v.getZExtValue();
3624  if (expectedNumLBResults != getLowerBoundsMap().getNumResults())
3625  return emitOpError() << "expected lower bounds map to have "
3626  << expectedNumLBResults << " results";
3627  unsigned expectedNumUBResults = 0;
3628  for (APInt v : getUpperBoundsGroups())
3629  expectedNumUBResults += v.getZExtValue();
3630  if (expectedNumUBResults != getUpperBoundsMap().getNumResults())
3631  return emitOpError() << "expected upper bounds map to have "
3632  << expectedNumUBResults << " results";
3633 
3634  if (getReductions().size() != getNumResults())
3635  return emitOpError("a reduction must be specified for each output");
3636 
3637  // Verify reduction ops are all valid
3638  for (Attribute attr : getReductions()) {
3639  auto intAttr = attr.dyn_cast<IntegerAttr>();
3640  if (!intAttr || !arith::symbolizeAtomicRMWKind(intAttr.getInt()))
3641  return emitOpError("invalid reduction attribute");
3642  }
3643 
3644  // Verify that the bound operands are valid dimension/symbols.
3645  /// Lower bounds.
3646  if (failed(verifyDimAndSymbolIdentifiers(*this, getLowerBoundsOperands(),
3647  getLowerBoundsMap().getNumDims())))
3648  return failure();
3649  /// Upper bounds.
3650  if (failed(verifyDimAndSymbolIdentifiers(*this, getUpperBoundsOperands(),
3651  getUpperBoundsMap().getNumDims())))
3652  return failure();
3653  return success();
3654 }
3655 
3657  SmallVector<Value, 4> newOperands{operands};
3658  auto newMap = getAffineMap();
3659  composeAffineMapAndOperands(&newMap, &newOperands);
3660  if (newMap == getAffineMap() && newOperands == operands)
3661  return failure();
3662  reset(newMap, newOperands);
3663  return success();
3664 }
3665 
3666 /// Canonicalize the bounds of the given loop.
3667 static LogicalResult canonicalizeLoopBounds(AffineParallelOp op) {
3668  AffineValueMap lb = op.getLowerBoundsValueMap();
3669  bool lbCanonicalized = succeeded(lb.canonicalize());
3670 
3671  AffineValueMap ub = op.getUpperBoundsValueMap();
3672  bool ubCanonicalized = succeeded(ub.canonicalize());
3673 
3674  // Any canonicalization change always leads to updated map(s).
3675  if (!lbCanonicalized && !ubCanonicalized)
3676  return failure();
3677 
3678  if (lbCanonicalized)
3679  op.setLowerBounds(lb.getOperands(), lb.getAffineMap());
3680  if (ubCanonicalized)
3681  op.setUpperBounds(ub.getOperands(), ub.getAffineMap());
3682 
3683  return success();
3684 }
3685 
3686 LogicalResult AffineParallelOp::fold(ArrayRef<Attribute> operands,
3687  SmallVectorImpl<OpFoldResult> &results) {
3688  return canonicalizeLoopBounds(*this);
3689 }
3690 
3691 /// Prints a lower(upper) bound of an affine parallel loop with max(min)
3692 /// conditions in it. `mapAttr` is a flat list of affine expressions and `group`
3693 /// identifies which of the those expressions form max/min groups. `operands`
3694 /// are the SSA values of dimensions and symbols and `keyword` is either "min"
3695 /// or "max".
3696 static void printMinMaxBound(OpAsmPrinter &p, AffineMapAttr mapAttr,
3697  DenseIntElementsAttr group, ValueRange operands,
3698  StringRef keyword) {
3699  AffineMap map = mapAttr.getValue();
3700  unsigned numDims = map.getNumDims();
3701  ValueRange dimOperands = operands.take_front(numDims);
3702  ValueRange symOperands = operands.drop_front(numDims);
3703  unsigned start = 0;
3704  for (llvm::APInt groupSize : group) {
3705  if (start != 0)
3706  p << ", ";
3707 
3708  unsigned size = groupSize.getZExtValue();
3709  if (size == 1) {
3710  p.printAffineExprOfSSAIds(map.getResult(start), dimOperands, symOperands);
3711  ++start;
3712  } else {
3713  p << keyword << '(';
3714  AffineMap submap = map.getSliceMap(start, size);
3715  p.printAffineMapOfSSAIds(AffineMapAttr::get(submap), operands);
3716  p << ')';
3717  start += size;
3718  }
3719  }
3720 }
3721 
3723  p << " (" << getBody()->getArguments() << ") = (";
3724  printMinMaxBound(p, getLowerBoundsMapAttr(), getLowerBoundsGroupsAttr(),
3725  getLowerBoundsOperands(), "max");
3726  p << ") to (";
3727  printMinMaxBound(p, getUpperBoundsMapAttr(), getUpperBoundsGroupsAttr(),
3728  getUpperBoundsOperands(), "min");
3729  p << ')';
3730  SmallVector<int64_t, 8> steps = getSteps();
3731  bool elideSteps = llvm::all_of(steps, [](int64_t step) { return step == 1; });
3732  if (!elideSteps) {
3733  p << " step (";
3734  llvm::interleaveComma(steps, p);
3735  p << ')';
3736  }
3737  if (getNumResults()) {
3738  p << " reduce (";
3739  llvm::interleaveComma(getReductions(), p, [&](auto &attr) {
3740  arith::AtomicRMWKind sym = *arith::symbolizeAtomicRMWKind(
3741  attr.template cast<IntegerAttr>().getInt());
3742  p << "\"" << arith::stringifyAtomicRMWKind(sym) << "\"";
3743  });
3744  p << ") -> (" << getResultTypes() << ")";
3745  }
3746 
3747  p << ' ';
3748  p.printRegion(getRegion(), /*printEntryBlockArgs=*/false,
3749  /*printBlockTerminators=*/getNumResults());
3751  (*this)->getAttrs(),
3752  /*elidedAttrs=*/{AffineParallelOp::getReductionsAttrStrName(),
3753  AffineParallelOp::getLowerBoundsMapAttrStrName(),
3754  AffineParallelOp::getLowerBoundsGroupsAttrStrName(),
3755  AffineParallelOp::getUpperBoundsMapAttrStrName(),
3756  AffineParallelOp::getUpperBoundsGroupsAttrStrName(),
3757  AffineParallelOp::getStepsAttrStrName()});
3758 }
3759 
3760 /// Given a list of lists of parsed operands, populates `uniqueOperands` with
3761 /// unique operands. Also populates `replacements with affine expressions of
3762 /// `kind` that can be used to update affine maps previously accepting a
3763 /// `operands` to accept `uniqueOperands` instead.
3765  OpAsmParser &parser,
3767  SmallVectorImpl<Value> &uniqueOperands,
3768  SmallVectorImpl<AffineExpr> &replacements, AffineExprKind kind) {
3769  assert((kind == AffineExprKind::DimId || kind == AffineExprKind::SymbolId) &&
3770  "expected operands to be dim or symbol expression");
3771 
3772  Type indexType = parser.getBuilder().getIndexType();
3773  for (const auto &list : operands) {
3774  SmallVector<Value> valueOperands;
3775  if (parser.resolveOperands(list, indexType, valueOperands))
3776  return failure();
3777  for (Value operand : valueOperands) {
3778  unsigned pos = std::distance(uniqueOperands.begin(),
3779  llvm::find(uniqueOperands, operand));
3780  if (pos == uniqueOperands.size())
3781  uniqueOperands.push_back(operand);
3782  replacements.push_back(
3783  kind == AffineExprKind::DimId
3784  ? getAffineDimExpr(pos, parser.getContext())
3785  : getAffineSymbolExpr(pos, parser.getContext()));
3786  }
3787  }
3788  return success();
3789 }
3790 
3791 namespace {
3792 enum class MinMaxKind { Min, Max };
3793 } // namespace
3794 
3795 /// Parses an affine map that can contain a min/max for groups of its results,
3796 /// e.g., max(expr-1, expr-2), expr-3, max(expr-4, expr-5, expr-6). Populates
3797 /// `result` attributes with the map (flat list of expressions) and the grouping
3798 /// (list of integers that specify how many expressions to put into each
3799 /// min/max) attributes. Deduplicates repeated operands.
3800 ///
3801 /// parallel-bound ::= `(` parallel-group-list `)`
3802 /// parallel-group-list ::= parallel-group (`,` parallel-group-list)?
3803 /// parallel-group ::= simple-group | min-max-group
3804 /// simple-group ::= expr-of-ssa-ids
3805 /// min-max-group ::= ( `min` | `max` ) `(` expr-of-ssa-ids-list `)`
3806 /// expr-of-ssa-ids-list ::= expr-of-ssa-ids (`,` expr-of-ssa-id-list)?
3807 ///
3808 /// Examples:
3809 /// (%0, min(%1 + %2, %3), %4, min(%5 floordiv 32, %6))
3810 /// (%0, max(%1 - 2 * %2))
3812  OperationState &result,
3813  MinMaxKind kind) {
3814  // Using `const` not `constexpr` below to workaround a MSVC optimizer bug,
3815  // see: https://reviews.llvm.org/D134227#3821753
3816  const llvm::StringLiteral tmpAttrStrName = "__pseudo_bound_map";
3817 
3818  StringRef mapName = kind == MinMaxKind::Min
3819  ? AffineParallelOp::getUpperBoundsMapAttrStrName()
3820  : AffineParallelOp::getLowerBoundsMapAttrStrName();
3821  StringRef groupsName =
3822  kind == MinMaxKind::Min
3823  ? AffineParallelOp::getUpperBoundsGroupsAttrStrName()
3824  : AffineParallelOp::getLowerBoundsGroupsAttrStrName();
3825 
3826  if (failed(parser.parseLParen()))
3827  return failure();
3828 
3829  if (succeeded(parser.parseOptionalRParen())) {
3830  result.addAttribute(
3831  mapName, AffineMapAttr::get(parser.getBuilder().getEmptyAffineMap()));
3832  result.addAttribute(groupsName, parser.getBuilder().getI32TensorAttr({}));
3833  return success();
3834  }
3835 
3836  SmallVector<AffineExpr> flatExprs;
3839  SmallVector<int32_t> numMapsPerGroup;
3841  auto parseOperands = [&]() {
3842  if (succeeded(parser.parseOptionalKeyword(
3843  kind == MinMaxKind::Min ? "min" : "max"))) {
3844  mapOperands.clear();
3845  AffineMapAttr map;
3846  if (failed(parser.parseAffineMapOfSSAIds(mapOperands, map, tmpAttrStrName,
3847  result.attributes,
3849  return failure();
3850  result.attributes.erase(tmpAttrStrName);
3851  llvm::append_range(flatExprs, map.getValue().getResults());
3852  auto operandsRef = llvm::makeArrayRef(mapOperands);
3853  auto dimsRef = operandsRef.take_front(map.getValue().getNumDims());
3854  SmallVector<OpAsmParser::UnresolvedOperand> dims(dimsRef.begin(),
3855  dimsRef.end());
3856  auto symsRef = operandsRef.drop_front(map.getValue().getNumDims());
3857  SmallVector<OpAsmParser::UnresolvedOperand> syms(symsRef.begin(),
3858  symsRef.end());
3859  flatDimOperands.append(map.getValue().getNumResults(), dims);
3860  flatSymOperands.append(map.getValue().getNumResults(), syms);
3861  numMapsPerGroup.push_back(map.getValue().getNumResults());
3862  } else {
3863  if (failed(parser.parseAffineExprOfSSAIds(flatDimOperands.emplace_back(),
3864  flatSymOperands.emplace_back(),
3865  flatExprs.emplace_back())))
3866  return failure();
3867  numMapsPerGroup.push_back(1);
3868  }
3869  return success();
3870  };
3871  if (parser.parseCommaSeparatedList(parseOperands) || parser.parseRParen())
3872  return failure();
3873 
3874  unsigned totalNumDims = 0;
3875  unsigned totalNumSyms = 0;
3876  for (unsigned i = 0, e = flatExprs.size(); i < e; ++i) {
3877  unsigned numDims = flatDimOperands[i].size();
3878  unsigned numSyms = flatSymOperands[i].size();
3879  flatExprs[i] = flatExprs[i]
3880  .shiftDims(numDims, totalNumDims)
3881  .shiftSymbols(numSyms, totalNumSyms);
3882  totalNumDims += numDims;
3883  totalNumSyms += numSyms;
3884  }
3885 
3886  // Deduplicate map operands.
3887  SmallVector<Value> dimOperands, symOperands;
3888  SmallVector<AffineExpr> dimRplacements, symRepacements;
3889  if (deduplicateAndResolveOperands(parser, flatDimOperands, dimOperands,
3890  dimRplacements, AffineExprKind::DimId) ||
3891  deduplicateAndResolveOperands(parser, flatSymOperands, symOperands,
3892  symRepacements, AffineExprKind::SymbolId))
3893  return failure();
3894 
3895  result.operands.append(dimOperands.begin(), dimOperands.end());
3896  result.operands.append(symOperands.begin(), symOperands.end());
3897 
3898  Builder &builder = parser.getBuilder();
3899  auto flatMap = AffineMap::get(totalNumDims, totalNumSyms, flatExprs,
3900  parser.getContext());
3901  flatMap = flatMap.replaceDimsAndSymbols(
3902  dimRplacements, symRepacements, dimOperands.size(), symOperands.size());
3903 
3904  result.addAttribute(mapName, AffineMapAttr::get(flatMap));
3905  result.addAttribute(groupsName, builder.getI32TensorAttr(numMapsPerGroup));
3906  return success();
3907 }
3908 
3909 //
3910 // operation ::= `affine.parallel` `(` ssa-ids `)` `=` parallel-bound
3911 // `to` parallel-bound steps? region attr-dict?
3912 // steps ::= `steps` `(` integer-literals `)`
3913 //
3914 ParseResult AffineParallelOp::parse(OpAsmParser &parser,
3915  OperationState &result) {
3916  auto &builder = parser.getBuilder();
3917  auto indexType = builder.getIndexType();
3920  parser.parseEqual() ||
3921  parseAffineMapWithMinMax(parser, result, MinMaxKind::Max) ||
3922  parser.parseKeyword("to") ||
3923  parseAffineMapWithMinMax(parser, result, MinMaxKind::Min))
3924  return failure();
3925 
3926  AffineMapAttr stepsMapAttr;
3927  NamedAttrList stepsAttrs;
3929  if (failed(parser.parseOptionalKeyword("step"))) {
3930  SmallVector<int64_t, 4> steps(ivs.size(), 1);
3931  result.addAttribute(AffineParallelOp::getStepsAttrStrName(),
3932  builder.getI64ArrayAttr(steps));
3933  } else {
3934  if (parser.parseAffineMapOfSSAIds(stepsMapOperands, stepsMapAttr,
3935  AffineParallelOp::getStepsAttrStrName(),
3936  stepsAttrs,
3938  return failure();
3939 
3940  // Convert steps from an AffineMap into an I64ArrayAttr.
3942  auto stepsMap = stepsMapAttr.getValue();
3943  for (const auto &result : stepsMap.getResults()) {
3944  auto constExpr = result.dyn_cast<AffineConstantExpr>();
3945  if (!constExpr)
3946  return parser.emitError(parser.getNameLoc(),
3947  "steps must be constant integers");
3948  steps.push_back(constExpr.getValue());
3949  }
3950  result.addAttribute(AffineParallelOp::getStepsAttrStrName(),
3951  builder.getI64ArrayAttr(steps));
3952  }
3953 
3954  // Parse optional clause of the form: `reduce ("addf", "maxf")`, where the
3955  // quoted strings are a member of the enum AtomicRMWKind.
3956  SmallVector<Attribute, 4> reductions;
3957  if (succeeded(parser.parseOptionalKeyword("reduce"))) {
3958  if (parser.parseLParen())
3959  return failure();
3960  auto parseAttributes = [&]() -> ParseResult {
3961  // Parse a single quoted string via the attribute parsing, and then
3962  // verify it is a member of the enum and convert to it's integer
3963  // representation.
3964  StringAttr attrVal;
3965  NamedAttrList attrStorage;
3966  auto loc = parser.getCurrentLocation();
3967  if (parser.parseAttribute(attrVal, builder.getNoneType(), "reduce",
3968  attrStorage))
3969  return failure();
3971  arith::symbolizeAtomicRMWKind(attrVal.getValue());
3972  if (!reduction)
3973  return parser.emitError(loc, "invalid reduction value: ") << attrVal;
3974  reductions.push_back(
3975  builder.getI64IntegerAttr(static_cast<int64_t>(reduction.value())));
3976  // While we keep getting commas, keep parsing.
3977  return success();
3978  };
3979  if (parser.parseCommaSeparatedList(parseAttributes) || parser.parseRParen())
3980  return failure();
3981  }
3982  result.addAttribute(AffineParallelOp::getReductionsAttrStrName(),
3983  builder.getArrayAttr(reductions));
3984 
3985  // Parse return types of reductions (if any)
3986  if (parser.parseOptionalArrowTypeList(result.types))
3987  return failure();
3988 
3989  // Now parse the body.
3990  Region *body = result.addRegion();
3991  for (auto &iv : ivs)
3992  iv.type = indexType;
3993  if (parser.parseRegion(*body, ivs) ||
3994  parser.parseOptionalAttrDict(result.attributes))
3995  return failure();
3996 
3997  // Add a terminator if none was parsed.
3998  AffineParallelOp::ensureTerminator(*body, builder, result.location);
3999  return success();
4000 }
4001 
4002 //===----------------------------------------------------------------------===//
4003 // AffineYieldOp
4004 //===----------------------------------------------------------------------===//
4005 
4007  auto *parentOp = (*this)->getParentOp();
4008  auto results = parentOp->getResults();
4009  auto operands = getOperands();
4010 
4011  if (!isa<AffineParallelOp, AffineIfOp, AffineForOp>(parentOp))
4012  return emitOpError() << "only terminates affine.if/for/parallel regions";
4013  if (parentOp->getNumResults() != getNumOperands())
4014  return emitOpError() << "parent of yield must have same number of "
4015  "results as the yield operands";
4016  for (auto it : llvm::zip(results, operands)) {
4017  if (std::get<0>(it).getType() != std::get<1>(it).getType())
4018  return emitOpError() << "types mismatch between yield op and its parent";
4019  }
4020 
4021  return success();
4022 }
4023 
4024 //===----------------------------------------------------------------------===//
4025 // AffineVectorLoadOp
4026 //===----------------------------------------------------------------------===//
4027 
4028 void AffineVectorLoadOp::build(OpBuilder &builder, OperationState &result,
4029  VectorType resultType, AffineMap map,
4030  ValueRange operands) {
4031  assert(operands.size() == 1 + map.getNumInputs() && "inconsistent operands");
4032  result.addOperands(operands);
4033  if (map)
4034  result.addAttribute(getMapAttrStrName(), AffineMapAttr::get(map));
4035  result.types.push_back(resultType);
4036 }
4037 
4038 void AffineVectorLoadOp::build(OpBuilder &builder, OperationState &result,
4039  VectorType resultType, Value memref,
4040  AffineMap map, ValueRange mapOperands) {
4041  assert(map.getNumInputs() == mapOperands.size() && "inconsistent index info");
4042  result.addOperands(memref);
4043  result.addOperands(mapOperands);
4044  result.addAttribute(getMapAttrStrName(), AffineMapAttr::get(map));
4045  result.types.push_back(resultType);
4046 }
4047 
4048 void AffineVectorLoadOp::build(OpBuilder &builder, OperationState &result,
4049  VectorType resultType, Value memref,
4050  ValueRange indices) {
4051  auto memrefType = memref.getType().cast<MemRefType>();
4052  int64_t rank = memrefType.getRank();
4053  // Create identity map for memrefs with at least one dimension or () -> ()
4054  // for zero-dimensional memrefs.
4055  auto map =
4056  rank ? builder.getMultiDimIdentityMap(rank) : builder.getEmptyAffineMap();
4057  build(builder, result, resultType, memref, map, indices);
4058 }
4059 
4060 void AffineVectorLoadOp::getCanonicalizationPatterns(RewritePatternSet &results,
4061  MLIRContext *context) {
4062  results.add<SimplifyAffineOp<AffineVectorLoadOp>>(context);
4063 }
4064 
4065 ParseResult AffineVectorLoadOp::parse(OpAsmParser &parser,
4066  OperationState &result) {
4067  auto &builder = parser.getBuilder();
4068  auto indexTy = builder.getIndexType();
4069 
4070  MemRefType memrefType;
4071  VectorType resultType;
4072  OpAsmParser::UnresolvedOperand memrefInfo;
4073  AffineMapAttr mapAttr;
4075  return failure(
4076  parser.parseOperand(memrefInfo) ||
4077  parser.parseAffineMapOfSSAIds(mapOperands, mapAttr,
4078  AffineVectorLoadOp::getMapAttrStrName(),
4079  result.attributes) ||
4080  parser.parseOptionalAttrDict(result.attributes) ||
4081  parser.parseColonType(memrefType) || parser.parseComma() ||
4082  parser.parseType(resultType) ||
4083  parser.resolveOperand(memrefInfo, memrefType, result.operands) ||
4084  parser.resolveOperands(mapOperands, indexTy, result.operands) ||
4085  parser.addTypeToList(resultType, result.types));
4086 }
4087 
4089  p << " " << getMemRef() << '[';
4090  if (AffineMapAttr mapAttr =
4091  (*this)->getAttrOfType<AffineMapAttr>(getMapAttrStrName()))
4092  p.printAffineMapOfSSAIds(mapAttr, getMapOperands());
4093  p << ']';
4094  p.printOptionalAttrDict((*this)->getAttrs(),
4095  /*elidedAttrs=*/{getMapAttrStrName()});
4096  p << " : " << getMemRefType() << ", " << getType();
4097 }
4098 
4099 /// Verify common invariants of affine.vector_load and affine.vector_store.
4100 static LogicalResult verifyVectorMemoryOp(Operation *op, MemRefType memrefType,
4101  VectorType vectorType) {
4102  // Check that memref and vector element types match.
4103  if (memrefType.getElementType() != vectorType.getElementType())
4104  return op->emitOpError(
4105  "requires memref and vector types of the same elemental type");
4106  return success();
4107 }
4108 
4110  MemRefType memrefType = getMemRefType();
4112  getOperation(),
4113  (*this)->getAttrOfType<AffineMapAttr>(getMapAttrStrName()),
4114  getMapOperands(), memrefType,
4115  /*numIndexOperands=*/getNumOperands() - 1)))
4116  return failure();
4117 
4118  if (failed(verifyVectorMemoryOp(getOperation(), memrefType, getVectorType())))
4119  return failure();
4120 
4121  return success();
4122 }
4123 
4124 //===----------------------------------------------------------------------===//
4125 // AffineVectorStoreOp
4126 //===----------------------------------------------------------------------===//
4127 
4128 void AffineVectorStoreOp::build(OpBuilder &builder, OperationState &result,
4129  Value valueToStore, Value memref, AffineMap map,
4130  ValueRange mapOperands) {
4131  assert(map.getNumInputs() == mapOperands.size() && "inconsistent index info");
4132  result.addOperands(valueToStore);
4133  result.addOperands(memref);
4134  result.addOperands(mapOperands);
4135  result.addAttribute(getMapAttrStrName(), AffineMapAttr::get(map));
4136 }
4137 
4138 // Use identity map.
4139 void AffineVectorStoreOp::build(OpBuilder &builder, OperationState &result,
4140  Value valueToStore, Value memref,
4141  ValueRange indices) {
4142  auto memrefType = memref.getType().cast<MemRefType>();
4143  int64_t rank = memrefType.getRank();
4144  // Create identity map for memrefs with at least one dimension or () -> ()
4145  // for zero-dimensional memrefs.
4146  auto map =
4147  rank ? builder.getMultiDimIdentityMap(rank) : builder.getEmptyAffineMap();
4148  build(builder, result, valueToStore, memref, map, indices);
4149 }
4150 void AffineVectorStoreOp::getCanonicalizationPatterns(
4151  RewritePatternSet &results, MLIRContext *context) {
4152  results.add<SimplifyAffineOp<AffineVectorStoreOp>>(context);
4153 }
4154 
4155 ParseResult AffineVectorStoreOp::parse(OpAsmParser &parser,
4156  OperationState &result) {
4157  auto indexTy = parser.getBuilder().getIndexType();
4158 
4159  MemRefType memrefType;
4160  VectorType resultType;
4161  OpAsmParser::UnresolvedOperand storeValueInfo;
4162  OpAsmParser::UnresolvedOperand memrefInfo;
4163  AffineMapAttr mapAttr;
4165  return failure(
4166  parser.parseOperand(storeValueInfo) || parser.parseComma() ||
4167  parser.parseOperand(memrefInfo) ||
4168  parser.parseAffineMapOfSSAIds(mapOperands, mapAttr,
4169  AffineVectorStoreOp::getMapAttrStrName(),
4170  result.attributes) ||
4171  parser.parseOptionalAttrDict(result.attributes) ||
4172  parser.parseColonType(memrefType) || parser.parseComma() ||
4173  parser.parseType(resultType) ||
4174  parser.resolveOperand(storeValueInfo, resultType, result.operands) ||
4175  parser.resolveOperand(memrefInfo, memrefType, result.operands) ||
4176  parser.resolveOperands(mapOperands, indexTy, result.operands));
4177 }
4178 
4180  p << " " << getValueToStore();
4181  p << ", " << getMemRef() << '[';
4182  if (AffineMapAttr mapAttr =
4183  (*this)->getAttrOfType<AffineMapAttr>(getMapAttrStrName()))
4184  p.printAffineMapOfSSAIds(mapAttr, getMapOperands());
4185  p << ']';
4186  p.printOptionalAttrDict((*this)->getAttrs(),
4187  /*elidedAttrs=*/{getMapAttrStrName()});
4188  p << " : " << getMemRefType() << ", " << getValueToStore().getType();
4189 }
4190 
4192  MemRefType memrefType = getMemRefType();
4194  *this, (*this)->getAttrOfType<AffineMapAttr>(getMapAttrStrName()),
4195  getMapOperands(), memrefType,
4196  /*numIndexOperands=*/getNumOperands() - 2)))
4197  return failure();
4198 
4199  if (failed(verifyVectorMemoryOp(*this, memrefType, getVectorType())))
4200  return failure();
4201 
4202  return success();
4203 }
4204 
4205 //===----------------------------------------------------------------------===//
4206 // DelinearizeIndexOp
4207 //===----------------------------------------------------------------------===//
4208 
4209 void AffineDelinearizeIndexOp::build(OpBuilder &builder, OperationState &result,
4210  Value linearIndex,
4211  ArrayRef<OpFoldResult> basis) {
4212  result.addTypes(SmallVector<Type>(basis.size(), builder.getIndexType()));
4213  result.addOperands(linearIndex);
4214  SmallVector<Value> basisValues =
4215  llvm::to_vector(llvm::map_range(basis, [&](OpFoldResult ofr) -> Value {
4216  Optional<int64_t> staticDim = getConstantIntValue(ofr);
4217  if (staticDim.has_value())
4218  return builder.create<arith::ConstantIndexOp>(result.location,
4219  *staticDim);
4220  return ofr.dyn_cast<Value>();
4221  }));
4222  result.addOperands(basisValues);
4223 }
4224 
4226  if (getBasis().empty())
4227  return emitOpError("basis should not be empty");
4228  if (getNumResults() != getBasis().size())
4229  return emitOpError("should return an index for each basis element");
4230  return success();
4231 }
4232 
4233 //===----------------------------------------------------------------------===//
4234 // TableGen'd op method definitions
4235 //===----------------------------------------------------------------------===//
4236 
4237 #define GET_OP_CLASSES
4238 #include "mlir/Dialect/Affine/IR/AffineOps.cpp.inc"
static AffineForOp buildAffineLoopFromConstants(OpBuilder &builder, Location loc, int64_t lb, int64_t ub, int64_t step, AffineForOp::BodyBuilderFn bodyBuilderFn)
Creates an affine loop from the bounds known to be constants.
Definition: AffineOps.cpp:2365
static void materializeConstants(OpBuilder &b, Location loc, ArrayRef< OpFoldResult > values, SmallVectorImpl< Operation * > &constants, SmallVectorImpl< Value > &actualValues)
Given a list of OpFoldResult, build the necessary operations to populate actualValues with values pro...
Definition: AffineOps.cpp:883
static bool hasTrivialZeroTripCount(AffineForOp op)
Returns true if the affine.for has zero iterations in trivial cases.
Definition: AffineOps.cpp:2127
static void composeMultiResultAffineMap(AffineMap &map, SmallVectorImpl< Value > &operands)
Composes the given affine map with the given list of operands, pulling in the maps from any affine....
Definition: AffineOps.cpp:983
static void printAffineMinMaxOp(OpAsmPrinter &p, T op)
Definition: AffineOps.cpp:2980
static void printMinMaxBound(OpAsmPrinter &p, AffineMapAttr mapAttr, DenseIntElementsAttr group, ValueRange operands, StringRef keyword)
Prints a lower(upper) bound of an affine parallel loop with max(min) conditions in it.
Definition: AffineOps.cpp:3696
static void LLVM_ATTRIBUTE_UNUSED simplifyMapWithOperands(AffineMap &map, ArrayRef< Value > operands)
Simplify the map while exploiting information on the values in operands.
Definition: AffineOps.cpp:740
static OpFoldResult foldMinMaxOp(T op, ArrayRef< Attribute > operands)
Fold an affine min or max operation with the given operands.
Definition: AffineOps.cpp:3016
static LogicalResult canonicalizeLoopBounds(AffineForOp forOp)
Canonicalize the bounds of the given loop.
Definition: AffineOps.cpp:1977
static Value createFoldedComposedAffineApply(OpBuilder &b, Location loc, AffineMap map, ValueRange operandsRef)
Fully compose map with operands and canonicalize the result.
Definition: AffineOps.cpp:1086
static bool isValidAffineIndexOperand(Value value, Region *region)
Definition: AffineOps.cpp:430
static void canonicalizeMapOrSetAndOperands(MapOrSet *mapOrSet, SmallVectorImpl< Value > *operands)
Definition: AffineOps.cpp:1156
static void composeAffineMapAndOperands(AffineMap *map, SmallVectorImpl< Value > *operands)
Iterate over operands and fold away all those produced by an AffineApplyOp iteratively.
Definition: AffineOps.cpp:805
static ParseResult parseBound(bool isLower, OperationState &result, OpAsmParser &p)
Parse a for operation loop bounds.
Definition: AffineOps.cpp:1686
static void composeSetAndOperands(IntegerSet &set, SmallVectorImpl< Value > &operands)
Compose any affine.apply ops feeding into operands of the integer set set by composing the maps of su...
Definition: AffineOps.cpp:2689
static LogicalResult replaceDimOrSym(AffineMap *map, unsigned dimOrSymbolPosition, SmallVectorImpl< Value > &dims, SmallVectorImpl< Value > &syms)
Replace all occurrences of AffineExpr at position pos in map by the defining AffineApplyOp expression...
Definition: AffineOps.cpp:762
static void canonicalizePromotedSymbols(MapOrSet *mapOrSet, SmallVectorImpl< Value > *operands)
Definition: AffineOps.cpp:1113
static LogicalResult verifyVectorMemoryOp(Operation *op, MemRefType memrefType, VectorType vectorType)
Verify common invariants of affine.vector_load and affine.vector_store.
Definition: AffineOps.cpp:4100
static ParseResult parseAffineMinMaxOp(OpAsmParser &parser, OperationState &result)
Definition: AffineOps.cpp:2993
static bool isMemRefSizeValidSymbol(AnyMemRefDefOp memrefDefOp, unsigned index, Region *region)
Returns true if the 'index' dimension of the memref defined by memrefDefOp is a statically shaped one...
Definition: AffineOps.cpp:313
static bool isNonNegativeBoundedBy(AffineExpr e, ArrayRef< Value > operands, int64_t k)
Check if e is known to be: 0 <= e < k.
Definition: AffineOps.cpp:622
static ParseResult parseAffineMapWithMinMax(OpAsmParser &parser, OperationState &result, MinMaxKind kind)
Parses an affine map that can contain a min/max for groups of its results, e.g., max(expr-1,...
Definition: AffineOps.cpp:3811
static AffineForOp buildAffineLoopFromValues(OpBuilder &builder, Location loc, Value lb, Value ub, int64_t step, AffineForOp::BodyBuilderFn bodyBuilderFn)
Creates an affine loop from the bounds that may or may not be constants.
Definition: AffineOps.cpp:2374
static std::enable_if_t< OpTy::template hasTrait< OpTrait::OneResult >), OpFoldResult > createOrFold(OpBuilder &b, Location loc, ValueRange operands, Args &&... leadingArguments)
Create an operation of the type provided as template argument and attempt to fold it immediately.
Definition: AffineOps.cpp:920
static void printDimAndSymbolList(Operation::operand_iterator begin, Operation::operand_iterator end, unsigned numDims, OpAsmPrinter &printer)
Prints dimension and symbol list.
Definition: AffineOps.cpp:435
static int64_t getLargestKnownDivisor(AffineExpr e, ArrayRef< Value > operands)
Returns the largest known divisor of e.
Definition: AffineOps.cpp:584
static void buildAffineLoopNestImpl(OpBuilder &builder, Location loc, BoundListTy lbs, BoundListTy ubs, ArrayRef< int64_t > steps, function_ref< void(OpBuilder &, Location, ValueRange)> bodyBuilderFn, LoopCreatorTy &&loopCreatorFn)
Builds an affine loop nest, using "loopCreatorFn" to create individual loop operations.
Definition: AffineOps.cpp:2324
static LogicalResult foldLoopBounds(AffineForOp forOp)
Fold the constant bounds of a loop.
Definition: AffineOps.cpp:1931
static LogicalResult verifyDimAndSymbolIdentifiers(OpTy &op, Operation::operand_range operands, unsigned numDims)
Utility function to verify that a set of operands are valid dimension and symbol identifiers.
Definition: AffineOps.cpp:468
static OpFoldResult makeComposedFoldedMinMax(OpBuilder &b, Location loc, AffineMap map, ArrayRef< OpFoldResult > operands)
Definition: AffineOps.cpp:1055
static bool isDimOpValidSymbol(ShapedDimOpInterface dimOp, Region *region)
Returns true if the result of the dim op is a valid symbol for region.
Definition: AffineOps.cpp:326
static void simplifyExprAndOperands(AffineExpr &expr, ArrayRef< Value > operands)
Simplify expr while exploiting information from the values in operands.
Definition: AffineOps.cpp:674
static bool isQTimesDPlusR(AffineExpr e, ArrayRef< Value > operands, int64_t &div, AffineExpr &quotientTimesDiv, AffineExpr &rem)
Check if expression e is of the form d*e_1 + e_2 where 0 <= e_2 < d.
Definition: AffineOps.cpp:650
static ParseResult deduplicateAndResolveOperands(OpAsmParser &parser, ArrayRef< SmallVector< OpAsmParser::UnresolvedOperand >> operands, SmallVectorImpl< Value > &uniqueOperands, SmallVectorImpl< AffineExpr > &replacements, AffineExprKind kind)
Given a list of lists of parsed operands, populates uniqueOperands with unique operands.
Definition: AffineOps.cpp:3764
static LogicalResult verifyAffineMinMaxOp(T op)
Definition: AffineOps.cpp:2970
static void printBound(AffineMapAttr boundMap, Operation::operand_range boundOperands, const char *prefix, OpAsmPrinter &p)
Definition: AffineOps.cpp:1848
static LogicalResult verifyMemoryOpIndexing(Operation *op, AffineMapAttr mapAttr, Operation::operand_range mapOperands, MemRefType memrefType, unsigned numIndexOperands)
Verify common indexing invariants of affine.load, affine.store, affine.vector_load and affine....
Definition: AffineOps.cpp:2796
static bool remainsLegalAfterInline(Value value, Region *src, Region *dest, const BlockAndValueMapping &mapping, function_ref< bool(Value, Region *)> legalityCheck)
Checks if value known to be a legal affine dimension or symbol in src region remains legal if the ope...
Definition: AffineOps.cpp:48
static LogicalResult canonicalizeMapExprAndTermOrder(AffineMap &map)
Canonicalize the result expression order of an affine map and return success if the order changed.
Definition: AffineOps.cpp:3176
static Operation * materializeConstant(Dialect *dialect, OpBuilder &builder, Attribute value, Type type, Location loc)
A utility function used to materialize a constant for a given attribute and type.
Definition: FoldUtils.cpp:50
static bool isLegalToInline(InlinerInterface &interface, Region *src, Region *insertRegion, bool shouldCloneInlinedRegion, BlockAndValueMapping &valueMapping)
Utility to check that all of the operations within 'src' can be inlined.
static constexpr const bool value
@ None
static Operation::operand_range getLowerBoundOperands(AffineForOp forOp)
Definition: SCFToGPU.cpp:74
static Operation::operand_range getUpperBoundOperands(AffineForOp forOp)
Definition: SCFToGPU.cpp:79
static void print(spirv::VerCapExtAttr triple, DialectAsmPrinter &printer)
static VectorType getVectorType(Type scalarTy, const VectorizationStrategy *strategy)
Returns the vector type resulting from applying the provided vectorization strategy on the scalar typ...
static int64_t getNumElements(ShapedType type)
Definition: TensorOps.cpp:1252
Affine binary operation expression.
Definition: AffineExpr.h:207
AffineBound represents a lower or upper bound in the for operation.
Definition: AffineOps.h:486
An integer constant appearing in affine expression.
Definition: AffineExpr.h:232
int64_t getValue() const
Definition: AffineExpr.cpp:505
A dimensional identifier appearing in an affine expression.
Definition: AffineExpr.h:216
AffineDmaStartOp starts a non-blocking DMA operation that transfers data from a source memref to a de...
Definition: AffineOps.h:92
static StringRef getDstMapAttrStrName()
Definition: AffineOps.h:237
static StringRef getTagMapAttrStrName()
Definition: AffineOps.h:238
Value getNumElements()
Returns the number of elements being transferred by this DMA operation.
Definition: AffineOps.h:197
unsigned getSrcMemRefOperandIndex()
Returns the operand index of the source memref.
Definition: AffineOps.h:104
unsigned getTagMemRefOperandIndex()
Returns the operand index of the tag memref.
Definition: AffineOps.h:168
AffineMapAttr getTagMapAttr()
Definition: AffineOps.h:185
MemRefType getSrcMemRefType()
Definition: AffineOps.h:108
bool isStrided()
Returns true if this DMA operation is strided, returns false otherwise.
Definition: AffineOps.h:249
MemRefType getTagMemRefType()
Definition: AffineOps.h:174
void print(OpAsmPrinter &p)
Definition: AffineOps.cpp:1362
Value getStride()
Returns the stride value for this DMA operation.
Definition: AffineOps.h:255
Value getTagMemRef()
Returns the Tag MemRef for this DMA operation.
Definition: AffineOps.h:173
AffineMap getTagMap()
Returns the affine map used to access the tag memref.
Definition: AffineOps.h:184
Value getDstMemRef()
Returns the destination MemRefType for this DMA operation.
Definition: AffineOps.h:139
LogicalResult verifyInvariantsImpl()
Definition: AffineOps.cpp:1460
operand_range getDstIndices()
Returns the destination memref indices for this DMA operation.
Definition: AffineOps.h:161
Value getSrcMemRef()
Returns the source MemRefType for this DMA operation.
Definition: AffineOps.h:107
operand_range getTagIndices()
Returns the tag memref indices for this DMA operation.
Definition: AffineOps.h:190
static StringRef getSrcMapAttrStrName()
Definition: AffineOps.h:236
unsigned getDstMemRefOperandIndex()
Returns the operand index of the destination memref.
Definition: AffineOps.h:134
static ParseResult parse(OpAsmParser &parser, OperationState &result)
Definition: AffineOps.cpp:1384
AffineMap getSrcMap()
Returns the affine map used to access the source memref.
Definition: AffineOps.h:116
AffineMap getDstMap()
Returns the affine map used to access the destination memref.
Definition: AffineOps.h:155
AffineMapAttr getDstMapAttr()
Definition: AffineOps.h:156
Value getNumElementsPerStride()
Returns the number of elements to transfer per stride for this DMA op.
Definition: AffineOps.h:262
LogicalResult fold(ArrayRef< Attribute > cstOperands, SmallVectorImpl< OpFoldResult > &results)
Definition: AffineOps.cpp:1498
MemRefType getDstMemRefType()
Definition: AffineOps.h:140
AffineMapAttr getSrcMapAttr()
Definition: AffineOps.h:117
operand_range getSrcIndices()
Returns the source memref affine map indices for this DMA operation.
Definition: AffineOps.h:122
AffineDmaWaitOp blocks until the completion of a DMA operation associated with the tag element 'tag[i...
Definition: AffineOps.h:285
LogicalResult verifyInvariantsImpl()
Definition: AffineOps.cpp:1563
AffineMapAttr getTagMapAttr()
Definition: AffineOps.h:303
Value getNumElements()
Returns the number of elements transferred by the associated DMA op.
Definition: AffineOps.h:327
operand_range getTagIndices()
Returns the tag memref index for this DMA operation.
Definition: AffineOps.h:308
static ParseResult parse(OpAsmParser &parser, OperationState &result)
Definition: AffineOps.cpp:1532
LogicalResult fold(ArrayRef< Attribute > cstOperands, SmallVectorImpl< OpFoldResult > &results)
Definition: AffineOps.cpp:1576
void print(OpAsmPrinter &p)
Definition: AffineOps.cpp:1518
Value getTagMemRef()
Returns the Tag MemRef associated with the DMA operation being waited on.
Definition: AffineOps.h:296
static void build(OpBuilder &builder, OperationState &result, Value tagMemRef, AffineMap tagMap, ValueRange tagIndices, Value numElements)
Definition: AffineOps.cpp:1509
static StringRef getTagMapAttrStrName()
Definition: AffineOps.h:329
RetTy walkPostOrder(AffineExpr expr)
Base type for affine expression.
Definition: AffineExpr.h:68
AffineExpr floorDiv(uint64_t v) const
Definition: AffineExpr.cpp:764
int64_t getLargestKnownDivisor() const
Returns the greatest known integral divisor of this affine expression.
Definition: AffineExpr.cpp:219
MLIRContext * getContext() const
Definition: AffineExpr.cpp:24
U dyn_cast() const
Definition: AffineExpr.h:281
A multi-dimensional affine map Affine map's are immutable like Type's, and they are uniqued.
Definition: AffineMap.h:42
AffineMap getSliceMap(unsigned start, unsigned length) const
Returns the map consisting of length expressions starting from start.
Definition: AffineMap.cpp:538
MLIRContext * getContext() const
Definition: AffineMap.cpp:265
bool isFunctionOfDim(unsigned position) const
Return true if any affine expression involves AffineDimExpr position.
Definition: AffineMap.h:175
static AffineMap get(MLIRContext *context)
Returns a zero result affine map with no dimensions or symbols: () -> ().
AffineMap shiftDims(unsigned shift, unsigned offset=0) const
Replace dims[offset ...
Definition: AffineMap.h:221
unsigned getNumSymbols() const
Definition: AffineMap.cpp:310
unsigned getNumDims() const
Definition: AffineMap.cpp:306
ArrayRef< AffineExpr > getResults() const
Definition: AffineMap.cpp:319
bool isFunctionOfSymbol(unsigned position) const
Return true if any affine expression involves AffineSymbolExpr position.
Definition: AffineMap.h:182
LogicalResult constantFold(ArrayRef< Attribute > operandConstants, SmallVectorImpl< Attribute > &results) const
Folds the results of the application of an affine map on the provided operands to a constant if possi...
Definition: AffineMap.cpp:347
unsigned getNumResults() const
Definition: AffineMap.cpp:314
AffineMap replaceDimsAndSymbols(ArrayRef< AffineExpr > dimReplacements, ArrayRef< AffineExpr > symReplacements, unsigned numResultDims, unsigned numResultSyms) const
This method substitutes any uses of dimensions and symbols (e.g.
Definition: AffineMap.cpp:408
unsigned getNumInputs() const
Definition: AffineMap.cpp:315
AffineMap shiftSymbols(unsigned shift, unsigned offset=0) const
Replace symbols[offset ...
Definition: AffineMap.h:234
static SmallVector< AffineMap, 4 > inferFromExprList(ArrayRef< ArrayRef< AffineExpr >> exprsList)
Returns a vector of AffineMaps; each with as many results as exprs.size(), as many dims as the larges...
Definition: AffineMap.cpp:236
AffineExpr getResult(unsigned idx) const
Definition: AffineMap.cpp:323
AffineMap replace(AffineExpr expr, AffineExpr replacement, unsigned numResultDims, unsigned numResultSyms) const
Sparse replace method.
Definition: AffineMap.cpp:423
static AffineMap getConstantMap(int64_t val, MLIRContext *context)
Returns a single constant result affine map.
Definition: AffineMap.cpp:96
AffineMap getSubMap(ArrayRef< unsigned > resultPos) const
Returns the map consisting of the resultPos subset.
Definition: AffineMap.cpp:530
A symbolic identifier appearing in an affine expression.
Definition: AffineExpr.h:224
An AffineValueMap is an affine map plus its ML value operands and results for analysis purposes.
unsigned getNumResults() const
LogicalResult canonicalize()
Attempts to canonicalize the map and operands.
Definition: AffineOps.cpp:3656
ArrayRef< Value > getOperands() const
AffineMap getAffineMap() const
AffineExpr getResult(unsigned i)
void reset(AffineMap map, ValueRange operands, ValueRange results={})
static void difference(const AffineValueMap &a, const AffineValueMap &b, AffineValueMap *res)
Return the value map that is the difference of value maps 'a' and 'b', represented as an affine map a...
@ Paren
Parens surrounding zero or more operands.
@ OptionalSquare
Square brackets supporting zero or more ops, or nothing.
virtual ParseResult parseColonTypeList(SmallVectorImpl< Type > &result)=0
Parse a colon followed by a type list, which must have at least one type.
virtual ParseResult parseCommaSeparatedList(Delimiter delimiter, function_ref< ParseResult()> parseElementFn, StringRef contextMessage=StringRef())=0
Parse a list of comma-separated items with an optional delimiter.
virtual Builder & getBuilder() const =0
Return a builder which provides useful access to MLIRContext, global objects like types and attribute...
virtual ParseResult parseOptionalAttrDict(NamedAttrList &result)=0
Parse a named dictionary into 'result' if it is present.
virtual ParseResult parseOptionalKeyword(StringRef keyword)=0
Parse the given keyword if present.
MLIRContext * getContext() const
Definition: AsmPrinter.cpp:67
virtual ParseResult parseRParen()=0
Parse a ) token.
virtual InFlightDiagnostic emitError(SMLoc loc, const Twine &message={})=0
Emit a diagnostic at the specified location and return failure.
ParseResult addTypeToList(Type type, SmallVectorImpl< Type > &result)
Add the specified type to the end of the specified type list and return success.
virtual ParseResult parseOptionalRParen()=0
Parse a ) token if present.
virtual ParseResult parseLess()=0
Parse a '<' token.
virtual ParseResult parseEqual()=0
Parse a = token.
virtual ParseResult parseColonType(Type &result)=0
Parse a colon followed by a type.
virtual SMLoc getCurrentLocation()=0
Get the location of the next token and store it into the argument.
virtual SMLoc getNameLoc() const =0
Return the location of the original name token.
virtual ParseResult parseGreater()=0
Parse a '>' token.
virtual ParseResult parseLParen()=0
Parse a ( token.
virtual ParseResult parseType(Type &result)=0
Parse a type.
virtual ParseResult parseComma()=0
Parse a , token.
virtual ParseResult parseOptionalArrowTypeList(SmallVectorImpl< Type > &result)=0
Parse an optional arrow followed by a type list.
virtual ParseResult parseArrowTypeList(SmallVectorImpl< Type > &result)=0
Parse an arrow followed by a type list.
ParseResult parseKeyword(StringRef keyword)
Parse a given keyword.
virtual ParseResult parseAttribute(Attribute &result, Type type={})=0
Parse an arbitrary attribute of a given type and return it in result.
void printOptionalArrowTypeList(TypeRange &&types)
Print an optional arrow followed by a type list.
Attributes are known-constant values of operations.
Definition: Attributes.h:25
U dyn_cast() const
Definition: Attributes.h:127
U cast() const
Definition: Attributes.h:137
Block * lookup(Block *from) const
Lookup a mapped value within the map.
This class represents an argument of a Block.
Definition: Value.h:296
Block * getOwner() const
Returns the block that owns this argument.
Definition: Value.h:305
Block represents an ordered list of Operations.
Definition: Block.h:30
Region * getParent() const
Provide a 'getParent' method for ilist_node_with_parent methods.
Definition: Block.cpp:26
Operation * getTerminator()
Get the terminator operation of this block.
Definition: Block.cpp:232
BlockArgument addArgument(Type type, Location loc)
Add one value to the argument list.
Definition: Block.cpp:141
BlockArgListType getArguments()
Definition: Block.h:76
Operation & front()
Definition: Block.h:142
This class is a general helper class for creating context-global objects like types,...
Definition: Builders.h:49
IntegerAttr getIndexAttr(int64_t value)
Definition: Builders.cpp:109
IntegerAttr getIntegerAttr(Type type, int64_t value)
Definition: Builders.cpp:212
AffineMap getDimIdentityMap()
Definition: Builders.cpp:346
AffineMap getMultiDimIdentityMap(unsigned rank)
Definition: Builders.cpp:350
DenseIntElementsAttr getI32TensorAttr(ArrayRef< int32_t > values)
Tensor-typed DenseIntElementsAttr getters.
Definition: Builders.cpp:169
IntegerAttr getI64IntegerAttr(int64_t value)
Definition: Builders.cpp:113
IntegerType getIntegerType(unsigned width)
Definition: Builders.cpp:72
NoneType getNoneType()
Definition: Builders.cpp:89
BoolAttr getBoolAttr(bool value)
Definition: Builders.cpp:101
AffineMap getEmptyAffineMap()
Returns a zero result affine map with no dimensions or symbols: () -> ().
Definition: Builders.cpp:339
AffineMap getConstantAffineMap(int64_t val)
Returns a single constant result affine map with 0 dimensions and 0 symbols.
Definition: Builders.cpp:341
MLIRContext * getContext() const
Definition: Builders.h:54
AffineMap getSymbolIdentityMap()
Definition: Builders.cpp:359
ArrayAttr getArrayAttr(ArrayRef< Attribute > value)
Definition: Builders.cpp:247
ArrayAttr getI64ArrayAttr(ArrayRef< int64_t > values)
Definition: Builders.cpp:262
IndexType getIndexType()
Definition: Builders.cpp:56
An attribute that represents a reference to a dense vector or tensor object.
std::enable_if_t<!std::is_base_of< Attribute, T >::value||std::is_same< Attribute, T >::value, T > getSplatValue() const
Return the splat value for this attribute.
An attribute that represents a reference to a dense integer vector or tensor object.
This is the interface that must be implemented by the dialects of operations to be inlined.
Definition: InliningUtils.h:41
DialectInlinerInterface(Dialect *dialect)
Definition: InliningUtils.h:43
An integer set representing a conjunction of one or more affine equalities and inequalities.
Definition: IntegerSet.h:44
unsigned getNumDims() const
Definition: IntegerSet.cpp:15
static IntegerSet get(unsigned dimCount, unsigned symbolCount, ArrayRef< AffineExpr > constraints, ArrayRef< bool > eqFlags)
MLIRContext * getContext() const
Definition: IntegerSet.cpp:57
unsigned getNumInputs() const
Definition: IntegerSet.cpp:17
ArrayRef< AffineExpr > getConstraints() const
Definition: IntegerSet.cpp:41
ArrayRef< bool > getEqFlags() const
Returns the equality bits, which specify whether each of the constraints is an equality or inequality...
Definition: IntegerSet.cpp:51
unsigned getNumSymbols() const
Definition: IntegerSet.cpp:16
This class defines the main interface for locations in MLIR and acts as a non-nullable wrapper around...
Definition: Location.h:64
MLIRContext is the top-level object for a collection of MLIR operations.
Definition: MLIRContext.h:56
Dialect * getLoadedDialect(StringRef name)
Get a registered IR dialect with the given namespace.
NamedAttrList is array of NamedAttributes that tracks whether it is sorted and does some basic work t...
void pop_back()
Pop last element from list.
Attribute erase(StringAttr name)
Erase the attribute with the given name from the list.
The OpAsmParser has methods for interacting with the asm parser: parsing things from it,...
virtual ParseResult parseRegion(Region &region, ArrayRef< Argument > arguments={}, bool enableNameShadowing=false)=0
Parses a region.
virtual ParseResult parseArgument(Argument &result, bool allowType=false, bool allowAttrs=false)=0
Parse a single argument with the following syntax:
ParseResult parseTrailingOperandList(SmallVectorImpl< UnresolvedOperand > &result, Delimiter delimiter=Delimiter::None)
Parse zero or more trailing SSA comma-separated trailing operand references with a specified surround...
virtual ParseResult parseArgumentList(SmallVectorImpl< Argument > &result, Delimiter delimiter=Delimiter::None, bool allowType=false, bool allowAttrs=false)=0
Parse zero or more arguments with a specified surrounding delimiter.
virtual ParseResult parseAffineMapOfSSAIds(SmallVectorImpl< UnresolvedOperand > &operands, Attribute &map, StringRef attrName, NamedAttrList &attrs, Delimiter delimiter=Delimiter::Square)=0
Parses an affine map attribute where dims and symbols are SSA operands.
ParseResult parseAssignmentList(SmallVectorImpl< Argument > &lhs, SmallVectorImpl< UnresolvedOperand > &rhs)
Parse a list of assignments of the form (x1 = y1, x2 = y2, ...)
virtual ParseResult resolveOperand(const UnresolvedOperand &operand, Type type, SmallVectorImpl< Value > &result)=0
Resolve an operand to an SSA value, emitting an error on failure.
ParseResult resolveOperands(Operands &&operands, Type type, SmallVectorImpl< Value > &result)
Resolve a list of operands to SSA values, emitting an error on failure, or appending the results to t...
virtual ParseResult parseOperand(UnresolvedOperand &result, bool allowResultNumber=true)=0
Parse a single SSA value operand name along with a result number if allowResultNumber is true.
virtual ParseResult parseAffineExprOfSSAIds(SmallVectorImpl< UnresolvedOperand > &dimOperands, SmallVectorImpl< UnresolvedOperand > &symbOperands, AffineExpr &expr)=0
Parses an affine expression where dims and symbols are SSA operands.
virtual ParseResult parseOperandList(SmallVectorImpl< UnresolvedOperand > &result, Delimiter delimiter=Delimiter::None, bool allowResultNumber=true, int requiredOperandCount=-1)=0
Parse zero or more SSA comma-separated operand references with a specified surrounding delimiter,...
This is a pure-virtual base class that exposes the asmprinter hooks necessary to implement a custom p...
virtual void printOptionalAttrDict(ArrayRef< NamedAttribute > attrs, ArrayRef< StringRef > elidedAttrs={})=0
If the specified operation has attributes, print out an attribute dictionary with their values.
virtual void printAffineExprOfSSAIds(AffineExpr expr, ValueRange dimOperands, ValueRange symOperands)=0
Prints an affine expression of SSA ids with SSA id names used instead of dims and symbols.
virtual void printAffineMapOfSSAIds(AffineMapAttr mapAttr, ValueRange operands)=0
Prints an affine map of SSA ids, where SSA id names are used in place of dims/symbols.
virtual void printRegion(Region &blocks, bool printEntryBlockArgs=true, bool printBlockTerminators=true, bool printEmptyBlock=false)=0
Prints a region.
virtual void printRegionArgument(BlockArgument arg, ArrayRef< NamedAttribute > argAttrs={}, bool omitType=false)=0
Print a block argument in the usual format of: ssaName : type {attr1=42} loc("here") where location p...
virtual void printOperand(Value value)=0
Print implementations for various things an operation contains.
RAII guard to reset the insertion point of the builder when destroyed.
Definition: Builders.h:300
This class helps build Operations.
Definition: Builders.h:198
void setListener(Listener *newListener)
Sets the listener of this builder to the one provided.
Definition: Builders.h:268
void setInsertionPointToStart(Block *block)
Sets the insertion point to the start of the specified block.
Definition: Builders.h:383
void setInsertionPoint(Block *block, Block::iterator insertPoint)
Set the insertion point to the specified location.
Definition: Builders.h:350
Listener * getListener() const
Returns the current listener of this builder, or nullptr if this builder doesn't have a listener.
Definition: Builders.h:272
void createOrFold(SmallVectorImpl< Value > &results, Location location, Args &&...args)
Create an operation of specific op type at the current insertion point, and immediately try to fold i...
Definition: Builders.h:472
Operation * create(const OperationState &state)
Creates an operation given the fields represented as an OperationState.
Definition: Builders.cpp:422
This class represents a single result from folding an operation.
Definition: OpDefinition.h:233
InFlightDiagnostic emitOpError(const Twine &message={})
Emit an error with the op name prefixed, like "'dim' op " which is convenient for verifiers.
Definition: Operation.cpp:639
A trait of region holding operations that defines a new scope for polyhedral optimization purposes.
This class provides the API for ops that are known to be isolated from above.
A trait used to provide symbol table functionalities to a region operation.
Definition: SymbolTable.h:341
This class implements the operand iterators for the Operation class.
Definition: ValueRange.h:41
Operation is a basic unit of execution within MLIR.
Definition: Operation.h:31
bool hasTrait()
Returns true if the operation was registered with a particular trait, e.g.
Definition: Operation.h:528
Operation * getParentOp()
Returns the closest surrounding operation that contains this operation or nullptr if this is a top-le...
Definition: Operation.h:165
operand_range getOperands()
Returns an iterator on the underlying Value's.
Definition: Operation.h:295
Region * getParentRegion()
Returns the region to which the instruction belongs.
Definition: Operation.h:161
operand_range::iterator operand_iterator
Definition: Operation.h:289
InFlightDiagnostic emitOpError(const Twine &message={})
Emit an error with the op name prefixed, like "'dim' op " which is convenient for verifiers.
Definition: Operation.cpp:512
void erase()
Remove this operation from its parent block and delete it.
Definition: Operation.cpp:418
This class represents success/failure for parsing-like operations that find it important to chain tog...
A special type of RewriterBase that coordinates the application of a rewrite pattern on the current I...
Definition: PatternMatch.h:605
This class represents a successor of a region.
This class contains a list of basic blocks and a link to the parent operation it is attached to.
Definition: Region.h:26
void push_back(Block *block)
Definition: Region.h:61
Operation * getParentOp()
Return the parent operation this region is attached to.
Definition: Region.h:200
bool empty()
Definition: Region.h:60
Block & front()
Definition: Region.h:65
BlockArgument addArgument(Type type, Location loc)
Add one value to the argument list.
Definition: Region.h:98
RewritePatternSet & add(ConstructorArg &&arg, ConstructorArgs &&...args)
Add an instance of each of the pattern types 'Ts' to the pattern list with the given arguments.
virtual void eraseBlock(Block *block)
This method erases all operations in a block.
void mergeBlockBefore(Block *source, Operation *op, ValueRange argValues=llvm::None)
virtual void replaceOp(Operation *op, ValueRange newValues)
This method replaces the results of the operation with the specified list of values.
virtual void eraseOp(Operation *op)
This method erases an operation that is known to have no uses.
virtual void finalizeRootUpdate(Operation *op)
This method is used to signal the end of a root update on the given operation.
Definition: PatternMatch.h:489
virtual void startRootUpdate(Operation *op)
This method is used to notify the rewriter that an in-place operation modification is about to happen...
Definition: PatternMatch.h:484
OpTy replaceOpWithNewOp(Operation *op, Args &&...args)
Replaces the result op with a new op that is created without verification.
Definition: PatternMatch.h:451
std::vector< SmallVector< int64_t, 8 > > operandExprStack
An attribute that represents a reference to a splat vector or tensor constant, meaning all of the ele...
static Operation * lookupSymbolIn(Operation *op, StringAttr symbol)
Returns the operation registered with the given symbol name with the regions of 'symbolTableOp'.
This class provides an abstraction over the various different ranges of value types.
Definition: TypeRange.h:36
Instances of the Type class are uniqued, have an immutable identifier and an optional mutable compone...
Definition: Types.h:74
U cast() const
Definition: Types.h:280
bool isa() const
Definition: Types.h:260
This class provides an abstraction over the different types of ranges over Values.
Definition: ValueRange.h:349
type_range getType() const
This class represents an instance of an SSA value in the MLIR system, representing a computable value...
Definition: Value.h:85
Type getType() const
Return the type of this value.
Definition: Value.h:114
U dyn_cast() const
Definition: Value.h:95
Location getLoc() const
Return the location of this value.
Definition: Value.cpp:26
Operation * getDefiningOp() const
If this value is the result of an operation, return the operation that defines it.
Definition: Value.cpp:20
constexpr auto RecursivelySpeculatable
Speculatability
This enum is returned from the getSpeculatability method in the ConditionallySpeculatable op interfac...
constexpr auto NotSpeculatable
BaseMemRefType getMemRefType(Value value, const BufferizationOptions &options, MemRefLayoutAttrInterface layout={}, Attribute memorySpace=nullptr)
Return a MemRefType to which the type of the given value can be bufferized.
constexpr void enumerate(std::tuple< Tys... > &tuple, CallbackT &&callback)
Definition: Matchers.h:230
LogicalResult foldMemRefCast(Operation *op, Value inner=nullptr)
This is a common utility used for patterns of the form "someop(memref.cast) -> someop".
Definition: MemRefOps.cpp:89
LLVM_ATTRIBUTE_ALWAYS_INLINE MPInt gcd(const MPInt &a, const MPInt &b)
Definition: MPInt.h:399
Include the generated interface declarations.
AffineMap simplifyAffineMap(AffineMap map)
Simplifies an affine map by simplifying its underlying AffineExpr results.
Definition: AffineMap.cpp:650
bool matchPattern(Value value, const Pattern &pattern)
Entry point for matching a pattern over a Value.
Definition: Matchers.h:329
void canonicalizeSetAndOperands(IntegerSet *set, SmallVectorImpl< Value > *operands)
Canonicalizes an integer set the same way canonicalizeMapAndOperands does for affine maps.
Definition: AffineOps.cpp:1237
LogicalResult failure(bool isFailure=true)
Utility function to generate a LogicalResult.
Definition: LogicalResult.h:62
bool isForInductionVar(Value val)
Returns true if the provided value is the induction variable of a AffineForOp.
Definition: AffineOps.cpp:2295
void extractForInductionVars(ArrayRef< AffineForOp > forInsts, SmallVectorImpl< Value > *ivs)
Extracts the induction variables from a list of AffineForOps and places them in the output argument i...
Definition: AffineOps.cpp:2314
AffineMap removeDuplicateExprs(AffineMap map)
Returns a map with the same dimension and symbol count as map, but whose results are the unique affin...
Definition: AffineMap.cpp:660
OpFoldResult makeComposedFoldedAffineApply(OpBuilder &b, Location loc, AffineMap map, ArrayRef< OpFoldResult > operands)
Constructs an AffineApplyOp that applies map to operands after composing the map with the maps of any...
Definition: AffineOps.cpp:1012
OpFoldResult makeComposedFoldedAffineMax(OpBuilder &b, Location loc, AffineMap map, ArrayRef< OpFoldResult > operands)
Constructs an AffineMinOp that computes a maximum across the results of applying map to operands,...
Definition: AffineOps.cpp:1079
OpFoldResult makeComposedFoldedAffineMin(OpBuilder &b, Location loc, AffineMap map, ArrayRef< OpFoldResult > operands)
Constructs an AffineMinOp that computes a minimum across the results of applying map to operands,...
Definition: AffineOps.cpp:1073
bool succeeded(LogicalResult result)
Utility function that returns true if the provided LogicalResult corresponds to a success value.
Definition: LogicalResult.h:68
AffineApplyOp makeComposedAffineApply(OpBuilder &b, Location loc, AffineMap map, ValueRange operands)
Returns a composed AffineApplyOp by composing map and operands with other AffineApplyOps supplying th...
Definition: AffineOps.cpp:964
Optional< int64_t > getConstantIntValue(OpFoldResult ofr)
If ofr is a constant integer or an IntegerAttr, return the integer.
LogicalResult success(bool isSuccess=true)
Utility function to generate a LogicalResult.
Definition: LogicalResult.h:56
AffineForOp replaceForOpWithNewYields(OpBuilder &b, AffineForOp loop, ValueRange newIterOperands, ValueRange newYieldedValues, ValueRange newIterArgs, bool replaceLoopResults=true)
Replace loop with a new loop where newIterOperands are appended with new initialization values and ne...
Definition: AffineOps.cpp:2403
void buildAffineLoopNest(OpBuilder &builder, Location loc, ArrayRef< int64_t > lbs, ArrayRef< int64_t > ubs, ArrayRef< int64_t > steps, function_ref< void(OpBuilder &, Location, ValueRange)> bodyBuilderFn=nullptr)
Builds a perfect nest of affine.for loops, i.e., each loop except the innermost one contains only ano...
Definition: AffineOps.cpp:2387
AffineExprKind
Definition: AffineExpr.h:40
@ Mod
RHS of mod is always a constant or a symbolic expression with a positive value.
@ DimId
Dimensional identifier.
@ FloorDiv
RHS of floordiv is always a constant or a symbolic expression.
@ SymbolId
Symbolic identifier.
AffineExpr getAffineBinaryOpExpr(AffineExprKind kind, AffineExpr lhs, AffineExpr rhs)
Definition: AffineExpr.cpp:46
bool isValidSymbol(Value value)
Returns true if the given value can be used as a symbol in the region of the closest surrounding op t...
Definition: AffineOps.cpp:356
AffineForOp getForInductionVarOwner(Value val)
Returns the loop parent of an induction variable.
Definition: AffineOps.cpp:2301
SmallVector< OpFoldResult > makeComposedFoldedMultiResultAffineApply(OpBuilder &b, Location loc, AffineMap map, ArrayRef< OpFoldResult > operands)
Variant of makeComposedFoldedAffineApply suitable for multi-result maps.
Definition: AffineOps.cpp:1037
bool isValidDim(Value value)
Returns true if the given Value can be used as a dimension id in the region of the closest surroundin...
Definition: AffineOps.cpp:262
bool isTopLevelValue(Value value)
TODO: These should be renamed if they are on the mlir namespace.
Definition: AffineOps.cpp:232
AffineExpr getAffineConstantExpr(int64_t constant, MLIRContext *context)
Definition: AffineExpr.cpp:513
void fullyComposeAffineMapAndOperands(AffineMap *map, SmallVectorImpl< Value > *operands)
Given an affine map map and its input operands, this method composes into map, maps of AffineApplyOps...
Definition: AffineOps.cpp:867
Value makeComposedAffineMin(OpBuilder &b, Location loc, AffineMap map, ValueRange operands)
Returns an AffineMinOp obtained by composing map and operands with AffineApplyOps supplying those ope...
Definition: AffineOps.cpp:1047
detail::constant_op_matcher m_Constant()
Matches a constant foldable operation.
Definition: Matchers.h:255
SmallVector< Value, 4 > applyMapToValues(OpBuilder &b, Location loc, AffineMap map, ValueRange values)
Returns the values obtained by applying map to the list of values.
Definition: AffineOps.cpp:1095
void canonicalizeMapAndOperands(AffineMap *map, SmallVectorImpl< Value > *operands)
Modifies both map and operands in-place so as to:
Definition: AffineOps.cpp:1232
ParseResult parseDimAndSymbolList(OpAsmParser &parser, SmallVectorImpl< Value > &operands, unsigned &numDims)
Parses dimension and symbol list.
Definition: AffineOps.cpp:445
AffineExpr getAffineDimExpr(unsigned position, MLIRContext *context)
These free functions allow clients of the API to not use classes in detail.
Definition: AffineExpr.cpp:488
LogicalResult verify(Operation *op, bool verifyRecursively=true)
Perform (potentially expensive) checks of invariants, used to detect compiler bugs,...
Definition: Verifier.cpp:372
bool failed(LogicalResult result)
Utility function that returns true if the provided LogicalResult corresponds to a failure value.
Definition: LogicalResult.h:72
AffineExpr getAffineSymbolExpr(unsigned position, MLIRContext *context)
Definition: AffineExpr.cpp:498
Region * getAffineScope(Operation *op)
Returns the closest region enclosing op that is held by an operation with trait AffineScope; nullptr ...
Definition: AffineOps.cpp:247
Canonicalize the affine map result expression order of an affine min/max operation.
Definition: AffineOps.cpp:3228
LogicalResult matchAndRewrite(T affineOp, PatternRewriter &rewriter) const override
Definition: AffineOps.cpp:3231
LogicalResult matchAndRewrite(T affineOp, PatternRewriter &rewriter) const override
Definition: AffineOps.cpp:3246
Remove duplicated expressions in affine min/max ops.
Definition: AffineOps.cpp:3046
LogicalResult matchAndRewrite(T affineOp, PatternRewriter &rewriter) const override
Definition: AffineOps.cpp:3049
Merge an affine min/max op to its consumers if its consumer is also an affine min/max op.
Definition: AffineOps.cpp:3089
LogicalResult matchAndRewrite(T affineOp, PatternRewriter &rewriter) const override
Definition: AffineOps.cpp:3092
This class represents an efficient way to signal success or failure.
Definition: LogicalResult.h:26
This is the representation of an operand reference.
This class represents a listener that may be used to hook into various actions within an OpBuilder.
Definition: Builders.h:255
virtual void notifyOperationInserted(Operation *op)
Notification handler for when an operation is inserted into the builder.
Definition: Builders.h:260
OpRewritePattern is a wrapper around RewritePattern that allows for matching and rewriting against an...
Definition: PatternMatch.h:356
This represents an operation in an abstracted form, suitable for use with the builder APIs.
SmallVector< Value, 4 > operands
void addOperands(ValueRange newOperands)
void addAttribute(StringRef name, Attribute attr)
Add an attribute with the specified name.
void addTypes(ArrayRef< Type > newTypes)
SmallVector< std::unique_ptr< Region >, 1 > regions
Regions that the op will hold.
NamedAttrList attributes
SmallVector< Type, 4 > types
Types of the results of this operation.
Region * addRegion()
Create a region that should be attached to the operation.
LogicalResult matchAndRewrite(Operation *op, PatternRewriter &rewriter) const final
Attempt to match against code rooted at the specified operation, which is the same operation code as ...
Definition: PatternMatch.h:327