MLIR  14.0.0git
AffineOps.cpp
Go to the documentation of this file.
1 //===- AffineOps.cpp - MLIR Affine Operations -----------------------------===//
2 //
3 // Part of the LLVM Project, under the Apache License v2.0 with LLVM Exceptions.
4 // See https://llvm.org/LICENSE.txt for license information.
5 // SPDX-License-Identifier: Apache-2.0 WITH LLVM-exception
6 //
7 //===----------------------------------------------------------------------===//
8 
16 #include "mlir/IR/BuiltinOps.h"
17 #include "mlir/IR/IntegerSet.h"
18 #include "mlir/IR/Matchers.h"
20 #include "mlir/IR/PatternMatch.h"
22 #include "llvm/ADT/SetVector.h"
23 #include "llvm/ADT/SmallBitVector.h"
24 #include "llvm/ADT/TypeSwitch.h"
25 #include "llvm/Support/Debug.h"
26 
27 using namespace mlir;
28 
29 #define DEBUG_TYPE "affine-analysis"
30 
31 #include "mlir/Dialect/Affine/IR/AffineOpsDialect.cpp.inc"
32 
33 /// A utility function to check if a value is defined at the top level of
34 /// `region` or is an argument of `region`. A value of index type defined at the
35 /// top level of a `AffineScope` region is always a valid symbol for all
36 /// uses in that region.
37 static bool isTopLevelValue(Value value, Region *region) {
38  if (auto arg = value.dyn_cast<BlockArgument>())
39  return arg.getParentRegion() == region;
40  return value.getDefiningOp()->getParentRegion() == region;
41 }
42 
43 /// Checks if `value` known to be a legal affine dimension or symbol in `src`
44 /// region remains legal if the operation that uses it is inlined into `dest`
45 /// with the given value mapping. `legalityCheck` is either `isValidDim` or
46 /// `isValidSymbol`, depending on the value being required to remain a valid
47 /// dimension or symbol.
48 static bool
50  const BlockAndValueMapping &mapping,
51  function_ref<bool(Value, Region *)> legalityCheck) {
52  // If the value is a valid dimension for any other reason than being
53  // a top-level value, it will remain valid: constants get inlined
54  // with the function, transitive affine applies also get inlined and
55  // will be checked themselves, etc.
56  if (!isTopLevelValue(value, src))
57  return true;
58 
59  // If it's a top-level value because it's a block operand, i.e. a
60  // function argument, check whether the value replacing it after
61  // inlining is a valid dimension in the new region.
62  if (value.isa<BlockArgument>())
63  return legalityCheck(mapping.lookup(value), dest);
64 
65  // If it's a top-level value because it's defined in the region,
66  // it can only be inlined if the defining op is a constant or a
67  // `dim`, which can appear anywhere and be valid, since the defining
68  // op won't be top-level anymore after inlining.
69  Attribute operandCst;
70  return matchPattern(value.getDefiningOp(), m_Constant(&operandCst)) ||
71  value.getDefiningOp<memref::DimOp>() ||
72  value.getDefiningOp<tensor::DimOp>();
73 }
74 
75 /// Checks if all values known to be legal affine dimensions or symbols in `src`
76 /// remain so if their respective users are inlined into `dest`.
77 static bool
79  const BlockAndValueMapping &mapping,
80  function_ref<bool(Value, Region *)> legalityCheck) {
81  return llvm::all_of(values, [&](Value v) {
82  return remainsLegalAfterInline(v, src, dest, mapping, legalityCheck);
83  });
84 }
85 
86 /// Checks if an affine read or write operation remains legal after inlining
87 /// from `src` to `dest`.
88 template <typename OpTy>
89 static bool remainsLegalAfterInline(OpTy op, Region *src, Region *dest,
90  const BlockAndValueMapping &mapping) {
91  static_assert(llvm::is_one_of<OpTy, AffineReadOpInterface,
92  AffineWriteOpInterface>::value,
93  "only ops with affine read/write interface are supported");
94 
95  AffineMap map = op.getAffineMap();
96  ValueRange dimOperands = op.getMapOperands().take_front(map.getNumDims());
97  ValueRange symbolOperands =
98  op.getMapOperands().take_back(map.getNumSymbols());
100  dimOperands, src, dest, mapping,
101  static_cast<bool (*)(Value, Region *)>(isValidDim)))
102  return false;
104  symbolOperands, src, dest, mapping,
105  static_cast<bool (*)(Value, Region *)>(isValidSymbol)))
106  return false;
107  return true;
108 }
109 
110 /// Checks if an affine apply operation remains legal after inlining from `src`
111 /// to `dest`.
112 // Use "unused attribute" marker to silence clang-tidy warning stemming from
113 // the inability to see through "llvm::TypeSwitch".
114 template <>
115 bool LLVM_ATTRIBUTE_UNUSED
116 remainsLegalAfterInline(AffineApplyOp op, Region *src, Region *dest,
117  const BlockAndValueMapping &mapping) {
118  // If it's a valid dimension, we need to check that it remains so.
119  if (isValidDim(op.getResult(), src))
121  op.getMapOperands(), src, dest, mapping,
122  static_cast<bool (*)(Value, Region *)>(isValidDim));
123 
124  // Otherwise it must be a valid symbol, check that it remains so.
126  op.getMapOperands(), src, dest, mapping,
127  static_cast<bool (*)(Value, Region *)>(isValidSymbol));
128 }
129 
130 //===----------------------------------------------------------------------===//
131 // AffineDialect Interfaces
132 //===----------------------------------------------------------------------===//
133 
134 namespace {
135 /// This class defines the interface for handling inlining with affine
136 /// operations.
137 struct AffineInlinerInterface : public DialectInlinerInterface {
139 
140  //===--------------------------------------------------------------------===//
141  // Analysis Hooks
142  //===--------------------------------------------------------------------===//
143 
144  /// Returns true if the given region 'src' can be inlined into the region
145  /// 'dest' that is attached to an operation registered to the current dialect.
146  /// 'wouldBeCloned' is set if the region is cloned into its new location
147  /// rather than moved, indicating there may be other users.
148  bool isLegalToInline(Region *dest, Region *src, bool wouldBeCloned,
149  BlockAndValueMapping &valueMapping) const final {
150  // We can inline into affine loops and conditionals if this doesn't break
151  // affine value categorization rules.
152  Operation *destOp = dest->getParentOp();
153  if (!isa<AffineParallelOp, AffineForOp, AffineIfOp>(destOp))
154  return false;
155 
156  // Multi-block regions cannot be inlined into affine constructs, all of
157  // which require single-block regions.
158  if (!llvm::hasSingleElement(*src))
159  return false;
160 
161  // Side-effecting operations that the affine dialect cannot understand
162  // should not be inlined.
163  Block &srcBlock = src->front();
164  for (Operation &op : srcBlock) {
165  // Ops with no side effects are fine,
166  if (auto iface = dyn_cast<MemoryEffectOpInterface>(op)) {
167  if (iface.hasNoEffect())
168  continue;
169  }
170 
171  // Assuming the inlined region is valid, we only need to check if the
172  // inlining would change it.
173  bool remainsValid =
175  .Case<AffineApplyOp, AffineReadOpInterface,
176  AffineWriteOpInterface>([&](auto op) {
177  return remainsLegalAfterInline(op, src, dest, valueMapping);
178  })
179  .Default([](Operation *) {
180  // Conservatively disallow inlining ops we cannot reason about.
181  return false;
182  });
183 
184  if (!remainsValid)
185  return false;
186  }
187 
188  return true;
189  }
190 
191  /// Returns true if the given operation 'op', that is registered to this
192  /// dialect, can be inlined into the given region, false otherwise.
193  bool isLegalToInline(Operation *op, Region *region, bool wouldBeCloned,
194  BlockAndValueMapping &valueMapping) const final {
195  // Always allow inlining affine operations into a region that is marked as
196  // affine scope, or into affine loops and conditionals. There are some edge
197  // cases when inlining *into* affine structures, but that is handled in the
198  // other 'isLegalToInline' hook above.
199  Operation *parentOp = region->getParentOp();
200  return parentOp->hasTrait<OpTrait::AffineScope>() ||
201  isa<AffineForOp, AffineParallelOp, AffineIfOp>(parentOp);
202  }
203 
204  /// Affine regions should be analyzed recursively.
205  bool shouldAnalyzeRecursively(Operation *op) const final { return true; }
206 };
207 } // namespace
208 
209 //===----------------------------------------------------------------------===//
210 // AffineDialect
211 //===----------------------------------------------------------------------===//
212 
213 void AffineDialect::initialize() {
214  addOperations<AffineDmaStartOp, AffineDmaWaitOp,
215 #define GET_OP_LIST
216 #include "mlir/Dialect/Affine/IR/AffineOps.cpp.inc"
217  >();
218  addInterfaces<AffineInlinerInterface>();
219 }
220 
221 /// Materialize a single constant operation from a given attribute value with
222 /// the desired resultant type.
224  Attribute value, Type type,
225  Location loc) {
226  return builder.create<arith::ConstantOp>(loc, type, value);
227 }
228 
229 /// A utility function to check if a value is defined at the top level of an
230 /// op with trait `AffineScope`. If the value is defined in an unlinked region,
231 /// conservatively assume it is not top-level. A value of index type defined at
232 /// the top level is always a valid symbol.
234  if (auto arg = value.dyn_cast<BlockArgument>()) {
235  // The block owning the argument may be unlinked, e.g. when the surrounding
236  // region has not yet been attached to an Op, at which point the parent Op
237  // is null.
238  Operation *parentOp = arg.getOwner()->getParentOp();
239  return parentOp && parentOp->hasTrait<OpTrait::AffineScope>();
240  }
241  // The defining Op may live in an unlinked block so its parent Op may be null.
242  Operation *parentOp = value.getDefiningOp()->getParentOp();
243  return parentOp && parentOp->hasTrait<OpTrait::AffineScope>();
244 }
245 
246 /// Returns the closest region enclosing `op` that is held by an operation with
247 /// trait `AffineScope`; `nullptr` if there is no such region.
248 // TODO: getAffineScope should be publicly exposed for affine passes/utilities.
250  auto *curOp = op;
251  while (auto *parentOp = curOp->getParentOp()) {
252  if (parentOp->hasTrait<OpTrait::AffineScope>())
253  return curOp->getParentRegion();
254  curOp = parentOp;
255  }
256  return nullptr;
257 }
258 
259 // A Value can be used as a dimension id iff it meets one of the following
260 // conditions:
261 // *) It is valid as a symbol.
262 // *) It is an induction variable.
263 // *) It is the result of affine apply operation with dimension id arguments.
264 bool mlir::isValidDim(Value value) {
265  // The value must be an index type.
266  if (!value.getType().isIndex())
267  return false;
268 
269  if (auto *defOp = value.getDefiningOp())
270  return isValidDim(value, getAffineScope(defOp));
271 
272  // This value has to be a block argument for an op that has the
273  // `AffineScope` trait or for an affine.for or affine.parallel.
274  auto *parentOp = value.cast<BlockArgument>().getOwner()->getParentOp();
275  return parentOp && (parentOp->hasTrait<OpTrait::AffineScope>() ||
276  isa<AffineForOp, AffineParallelOp>(parentOp));
277 }
278 
279 // Value can be used as a dimension id iff it meets one of the following
280 // conditions:
281 // *) It is valid as a symbol.
282 // *) It is an induction variable.
283 // *) It is the result of an affine apply operation with dimension id operands.
284 bool mlir::isValidDim(Value value, Region *region) {
285  // The value must be an index type.
286  if (!value.getType().isIndex())
287  return false;
288 
289  // All valid symbols are okay.
290  if (isValidSymbol(value, region))
291  return true;
292 
293  auto *op = value.getDefiningOp();
294  if (!op) {
295  // This value has to be a block argument for an affine.for or an
296  // affine.parallel.
297  auto *parentOp = value.cast<BlockArgument>().getOwner()->getParentOp();
298  return isa<AffineForOp, AffineParallelOp>(parentOp);
299  }
300 
301  // Affine apply operation is ok if all of its operands are ok.
302  if (auto applyOp = dyn_cast<AffineApplyOp>(op))
303  return applyOp.isValidDim(region);
304  // The dim op is okay if its operand memref/tensor is defined at the top
305  // level.
306  if (auto dimOp = dyn_cast<memref::DimOp>(op))
307  return isTopLevelValue(dimOp.source());
308  if (auto dimOp = dyn_cast<tensor::DimOp>(op))
309  return isTopLevelValue(dimOp.source());
310  return false;
311 }
312 
313 /// Returns true if the 'index' dimension of the `memref` defined by
314 /// `memrefDefOp` is a statically shaped one or defined using a valid symbol
315 /// for `region`.
316 template <typename AnyMemRefDefOp>
317 static bool isMemRefSizeValidSymbol(AnyMemRefDefOp memrefDefOp, unsigned index,
318  Region *region) {
319  auto memRefType = memrefDefOp.getType();
320  // Statically shaped.
321  if (!memRefType.isDynamicDim(index))
322  return true;
323  // Get the position of the dimension among dynamic dimensions;
324  unsigned dynamicDimPos = memRefType.getDynamicDimIndex(index);
325  return isValidSymbol(*(memrefDefOp.getDynamicSizes().begin() + dynamicDimPos),
326  region);
327 }
328 
329 /// Returns true if the result of the dim op is a valid symbol for `region`.
330 template <typename OpTy>
331 static bool isDimOpValidSymbol(OpTy dimOp, Region *region) {
332  // The dim op is okay if its source is defined at the top level.
333  if (isTopLevelValue(dimOp.source()))
334  return true;
335 
336  // Conservatively handle remaining BlockArguments as non-valid symbols.
337  // E.g. scf.for iterArgs.
338  if (dimOp.source().template isa<BlockArgument>())
339  return false;
340 
341  // The dim op is also okay if its operand memref is a view/subview whose
342  // corresponding size is a valid symbol.
343  Optional<int64_t> index = dimOp.getConstantIndex();
344  assert(index.hasValue() &&
345  "expect only `dim` operations with a constant index");
346  int64_t i = index.getValue();
347  return TypeSwitch<Operation *, bool>(dimOp.source().getDefiningOp())
348  .Case<memref::ViewOp, memref::SubViewOp, memref::AllocOp>(
349  [&](auto op) { return isMemRefSizeValidSymbol(op, i, region); })
350  .Default([](Operation *) { return false; });
351 }
352 
353 // A value can be used as a symbol (at all its use sites) iff it meets one of
354 // the following conditions:
355 // *) It is a constant.
356 // *) Its defining op or block arg appearance is immediately enclosed by an op
357 // with `AffineScope` trait.
358 // *) It is the result of an affine.apply operation with symbol operands.
359 // *) It is a result of the dim op on a memref whose corresponding size is a
360 // valid symbol.
362  if (!value)
363  return false;
364 
365  // The value must be an index type.
366  if (!value.getType().isIndex())
367  return false;
368 
369  // Check that the value is a top level value.
370  if (isTopLevelValue(value))
371  return true;
372 
373  if (auto *defOp = value.getDefiningOp())
374  return isValidSymbol(value, getAffineScope(defOp));
375 
376  return false;
377 }
378 
379 /// A value can be used as a symbol for `region` iff it meets one of the
380 /// following conditions:
381 /// *) It is a constant.
382 /// *) It is the result of an affine apply operation with symbol arguments.
383 /// *) It is a result of the dim op on a memref whose corresponding size is
384 /// a valid symbol.
385 /// *) It is defined at the top level of 'region' or is its argument.
386 /// *) It dominates `region`'s parent op.
387 /// If `region` is null, conservatively assume the symbol definition scope does
388 /// not exist and only accept the values that would be symbols regardless of
389 /// the surrounding region structure, i.e. the first three cases above.
390 bool mlir::isValidSymbol(Value value, Region *region) {
391  // The value must be an index type.
392  if (!value.getType().isIndex())
393  return false;
394 
395  // A top-level value is a valid symbol.
396  if (region && ::isTopLevelValue(value, region))
397  return true;
398 
399  auto *defOp = value.getDefiningOp();
400  if (!defOp) {
401  // A block argument that is not a top-level value is a valid symbol if it
402  // dominates region's parent op.
403  Operation *regionOp = region ? region->getParentOp() : nullptr;
404  if (regionOp && !regionOp->hasTrait<OpTrait::IsIsolatedFromAbove>())
405  if (auto *parentOpRegion = region->getParentOp()->getParentRegion())
406  return isValidSymbol(value, parentOpRegion);
407  return false;
408  }
409 
410  // Constant operation is ok.
411  Attribute operandCst;
412  if (matchPattern(defOp, m_Constant(&operandCst)))
413  return true;
414 
415  // Affine apply operation is ok if all of its operands are ok.
416  if (auto applyOp = dyn_cast<AffineApplyOp>(defOp))
417  return applyOp.isValidSymbol(region);
418 
419  // Dim op results could be valid symbols at any level.
420  if (auto dimOp = dyn_cast<memref::DimOp>(defOp))
421  return isDimOpValidSymbol(dimOp, region);
422  if (auto dimOp = dyn_cast<tensor::DimOp>(defOp))
423  return isDimOpValidSymbol(dimOp, region);
424 
425  // Check for values dominating `region`'s parent op.
426  Operation *regionOp = region ? region->getParentOp() : nullptr;
427  if (regionOp && !regionOp->hasTrait<OpTrait::IsIsolatedFromAbove>())
428  if (auto *parentRegion = region->getParentOp()->getParentRegion())
429  return isValidSymbol(value, parentRegion);
430 
431  return false;
432 }
433 
434 // Returns true if 'value' is a valid index to an affine operation (e.g.
435 // affine.load, affine.store, affine.dma_start, affine.dma_wait) where
436 // `region` provides the polyhedral symbol scope. Returns false otherwise.
437 static bool isValidAffineIndexOperand(Value value, Region *region) {
438  return isValidDim(value, region) || isValidSymbol(value, region);
439 }
440 
441 /// Prints dimension and symbol list.
444  unsigned numDims, OpAsmPrinter &printer) {
445  OperandRange operands(begin, end);
446  printer << '(' << operands.take_front(numDims) << ')';
447  if (operands.size() > numDims)
448  printer << '[' << operands.drop_front(numDims) << ']';
449 }
450 
451 /// Parses dimension and symbol list and returns true if parsing failed.
453  SmallVectorImpl<Value> &operands,
454  unsigned &numDims) {
456  if (parser.parseOperandList(opInfos, OpAsmParser::Delimiter::Paren))
457  return failure();
458  // Store number of dimensions for validation by caller.
459  numDims = opInfos.size();
460 
461  // Parse the optional symbol operands.
462  auto indexTy = parser.getBuilder().getIndexType();
463  return failure(parser.parseOperandList(
465  parser.resolveOperands(opInfos, indexTy, operands));
466 }
467 
468 /// Utility function to verify that a set of operands are valid dimension and
469 /// symbol identifiers. The operands should be laid out such that the dimension
470 /// operands are before the symbol operands. This function returns failure if
471 /// there was an invalid operand. An operation is provided to emit any necessary
472 /// errors.
473 template <typename OpTy>
474 static LogicalResult
476  unsigned numDims) {
477  unsigned opIt = 0;
478  for (auto operand : operands) {
479  if (opIt++ < numDims) {
480  if (!isValidDim(operand, getAffineScope(op)))
481  return op.emitOpError("operand cannot be used as a dimension id");
482  } else if (!isValidSymbol(operand, getAffineScope(op))) {
483  return op.emitOpError("operand cannot be used as a symbol");
484  }
485  }
486  return success();
487 }
488 
489 //===----------------------------------------------------------------------===//
490 // AffineApplyOp
491 //===----------------------------------------------------------------------===//
492 
493 AffineValueMap AffineApplyOp::getAffineValueMap() {
494  return AffineValueMap(getAffineMap(), getOperands(), getResult());
495 }
496 
498  OperationState &result) {
499  auto &builder = parser.getBuilder();
500  auto indexTy = builder.getIndexType();
501 
502  AffineMapAttr mapAttr;
503  unsigned numDims;
504  if (parser.parseAttribute(mapAttr, "map", result.attributes) ||
505  parseDimAndSymbolList(parser, result.operands, numDims) ||
506  parser.parseOptionalAttrDict(result.attributes))
507  return failure();
508  auto map = mapAttr.getValue();
509 
510  if (map.getNumDims() != numDims ||
511  numDims + map.getNumSymbols() != result.operands.size()) {
512  return parser.emitError(parser.getNameLoc(),
513  "dimension or symbol index mismatch");
514  }
515 
516  result.types.append(map.getNumResults(), indexTy);
517  return success();
518 }
519 
520 static void print(OpAsmPrinter &p, AffineApplyOp op) {
521  p << " " << op.mapAttr();
522  printDimAndSymbolList(op.operand_begin(), op.operand_end(),
523  op.getAffineMap().getNumDims(), p);
524  p.printOptionalAttrDict(op->getAttrs(), /*elidedAttrs=*/{"map"});
525 }
526 
527 static LogicalResult verify(AffineApplyOp op) {
528  // Check input and output dimensions match.
529  auto map = op.map();
530 
531  // Verify that operand count matches affine map dimension and symbol count.
532  if (op.getNumOperands() != map.getNumDims() + map.getNumSymbols())
533  return op.emitOpError(
534  "operand count and affine map dimension and symbol count must match");
535 
536  // Verify that the map only produces one result.
537  if (map.getNumResults() != 1)
538  return op.emitOpError("mapping must produce one value");
539 
540  return success();
541 }
542 
543 // The result of the affine apply operation can be used as a dimension id if all
544 // its operands are valid dimension ids.
546  return llvm::all_of(getOperands(),
547  [](Value op) { return mlir::isValidDim(op); });
548 }
549 
550 // The result of the affine apply operation can be used as a dimension id if all
551 // its operands are valid dimension ids with the parent operation of `region`
552 // defining the polyhedral scope for symbols.
553 bool AffineApplyOp::isValidDim(Region *region) {
554  return llvm::all_of(getOperands(),
555  [&](Value op) { return ::isValidDim(op, region); });
556 }
557 
558 // The result of the affine apply operation can be used as a symbol if all its
559 // operands are symbols.
561  return llvm::all_of(getOperands(),
562  [](Value op) { return mlir::isValidSymbol(op); });
563 }
564 
565 // The result of the affine apply operation can be used as a symbol in `region`
566 // if all its operands are symbols in `region`.
567 bool AffineApplyOp::isValidSymbol(Region *region) {
568  return llvm::all_of(getOperands(), [&](Value operand) {
569  return mlir::isValidSymbol(operand, region);
570  });
571 }
572 
573 OpFoldResult AffineApplyOp::fold(ArrayRef<Attribute> operands) {
574  auto map = getAffineMap();
575 
576  // Fold dims and symbols to existing values.
577  auto expr = map.getResult(0);
578  if (auto dim = expr.dyn_cast<AffineDimExpr>())
579  return getOperand(dim.getPosition());
580  if (auto sym = expr.dyn_cast<AffineSymbolExpr>())
581  return getOperand(map.getNumDims() + sym.getPosition());
582 
583  // Otherwise, default to folding the map.
585  if (failed(map.constantFold(operands, result)))
586  return {};
587  return result[0];
588 }
589 
590 /// Replace all occurrences of AffineExpr at position `pos` in `map` by the
591 /// defining AffineApplyOp expression and operands.
592 /// When `dimOrSymbolPosition < dims.size()`, AffineDimExpr@[pos] is replaced.
593 /// When `dimOrSymbolPosition >= dims.size()`,
594 /// AffineSymbolExpr@[pos - dims.size()] is replaced.
595 /// Mutate `map`,`dims` and `syms` in place as follows:
596 /// 1. `dims` and `syms` are only appended to.
597 /// 2. `map` dim and symbols are gradually shifted to higer positions.
598 /// 3. Old `dim` and `sym` entries are replaced by nullptr
599 /// This avoids the need for any bookkeeping.
601  unsigned dimOrSymbolPosition,
603  SmallVectorImpl<Value> &syms) {
604  bool isDimReplacement = (dimOrSymbolPosition < dims.size());
605  unsigned pos = isDimReplacement ? dimOrSymbolPosition
606  : dimOrSymbolPosition - dims.size();
607  Value &v = isDimReplacement ? dims[pos] : syms[pos];
608  if (!v)
609  return failure();
610 
611  auto affineApply = v.getDefiningOp<AffineApplyOp>();
612  if (!affineApply)
613  return failure();
614 
615  // At this point we will perform a replacement of `v`, set the entry in `dim`
616  // or `sym` to nullptr immediately.
617  v = nullptr;
618 
619  // Compute the map, dims and symbols coming from the AffineApplyOp.
620  AffineMap composeMap = affineApply.getAffineMap();
621  assert(composeMap.getNumResults() == 1 && "affine.apply with >1 results");
622  AffineExpr composeExpr =
623  composeMap.shiftDims(dims.size()).shiftSymbols(syms.size()).getResult(0);
624  ValueRange composeDims =
625  affineApply.getMapOperands().take_front(composeMap.getNumDims());
626  ValueRange composeSyms =
627  affineApply.getMapOperands().take_back(composeMap.getNumSymbols());
628 
629  // Append the dims and symbols where relevant and perform the replacement.
630  MLIRContext *ctx = map->getContext();
631  AffineExpr toReplace = isDimReplacement ? getAffineDimExpr(pos, ctx)
632  : getAffineSymbolExpr(pos, ctx);
633  dims.append(composeDims.begin(), composeDims.end());
634  syms.append(composeSyms.begin(), composeSyms.end());
635  *map = map->replace(toReplace, composeExpr, dims.size(), syms.size());
636 
637  return success();
638 }
639 
640 /// Iterate over `operands` and fold away all those produced by an AffineApplyOp
641 /// iteratively. Perform canonicalization of map and operands as well as
642 /// AffineMap simplification. `map` and `operands` are mutated in place.
644  SmallVectorImpl<Value> *operands) {
645  if (map->getNumResults() == 0) {
646  canonicalizeMapAndOperands(map, operands);
647  *map = simplifyAffineMap(*map);
648  return;
649  }
650 
651  MLIRContext *ctx = map->getContext();
652  SmallVector<Value, 4> dims(operands->begin(),
653  operands->begin() + map->getNumDims());
654  SmallVector<Value, 4> syms(operands->begin() + map->getNumDims(),
655  operands->end());
656 
657  // Iterate over dims and symbols coming from AffineApplyOp and replace until
658  // exhaustion. This iteratively mutates `map`, `dims` and `syms`. Both `dims`
659  // and `syms` can only increase by construction.
660  // The implementation uses a `while` loop to support the case of symbols
661  // that may be constructed from dims ;this may be overkill.
662  while (true) {
663  bool changed = false;
664  for (unsigned pos = 0; pos != dims.size() + syms.size(); ++pos)
665  if ((changed |= succeeded(replaceDimOrSym(map, pos, dims, syms))))
666  break;
667  if (!changed)
668  break;
669  }
670 
671  // Clear operands so we can fill them anew.
672  operands->clear();
673 
674  // At this point we may have introduced null operands, prune them out before
675  // canonicalizing map and operands.
676  unsigned nDims = 0, nSyms = 0;
677  SmallVector<AffineExpr, 4> dimReplacements, symReplacements;
678  dimReplacements.reserve(dims.size());
679  symReplacements.reserve(syms.size());
680  for (auto *container : {&dims, &syms}) {
681  bool isDim = (container == &dims);
682  auto &repls = isDim ? dimReplacements : symReplacements;
683  for (const auto &en : llvm::enumerate(*container)) {
684  Value v = en.value();
685  if (!v) {
686  assert(isDim ? !map->isFunctionOfDim(en.index())
687  : !map->isFunctionOfSymbol(en.index()) &&
688  "map is function of unexpected expr@pos");
689  repls.push_back(getAffineConstantExpr(0, ctx));
690  continue;
691  }
692  repls.push_back(isDim ? getAffineDimExpr(nDims++, ctx)
693  : getAffineSymbolExpr(nSyms++, ctx));
694  operands->push_back(v);
695  }
696  }
697  *map = map->replaceDimsAndSymbols(dimReplacements, symReplacements, nDims,
698  nSyms);
699 
700  // Canonicalize and simplify before returning.
701  canonicalizeMapAndOperands(map, operands);
702  *map = simplifyAffineMap(*map);
703 }
704 
706  SmallVectorImpl<Value> *operands) {
707  while (llvm::any_of(*operands, [](Value v) {
708  return isa_and_nonnull<AffineApplyOp>(v.getDefiningOp());
709  })) {
710  composeAffineMapAndOperands(map, operands);
711  }
712 }
713 
715  AffineMap map,
716  ValueRange operands) {
717  AffineMap normalizedMap = map;
718  SmallVector<Value, 8> normalizedOperands(operands.begin(), operands.end());
719  composeAffineMapAndOperands(&normalizedMap, &normalizedOperands);
720  assert(normalizedMap);
721  return b.create<AffineApplyOp>(loc, normalizedMap, normalizedOperands);
722 }
723 
725  AffineExpr e, ValueRange values) {
728  values);
729 }
730 
731 /// Fully compose map with operands and canonicalize the result.
732 /// Return the `createOrFold`'ed AffineApply op.
734  AffineMap map,
735  ValueRange operandsRef) {
736  SmallVector<Value, 4> operands(operandsRef.begin(), operandsRef.end());
737  fullyComposeAffineMapAndOperands(&map, &operands);
738  canonicalizeMapAndOperands(&map, &operands);
739  return b.createOrFold<AffineApplyOp>(loc, map, operands);
740 }
741 
743  AffineMap map, ValueRange values) {
745  res.reserve(map.getNumResults());
746  unsigned numDims = map.getNumDims(), numSym = map.getNumSymbols();
747  // For each `expr` in `map`, applies the `expr` to the values extracted from
748  // ranges. If the resulting application can be folded into a Value, the
749  // folding occurs eagerly.
750  for (auto expr : map.getResults()) {
751  AffineMap map = AffineMap::get(numDims, numSym, expr);
752  res.push_back(createFoldedComposedAffineApply(b, loc, map, values));
753  }
754  return res;
755 }
756 
757 // A symbol may appear as a dim in affine.apply operations. This function
758 // canonicalizes dims that are valid symbols into actual symbols.
759 template <class MapOrSet>
760 static void canonicalizePromotedSymbols(MapOrSet *mapOrSet,
761  SmallVectorImpl<Value> *operands) {
762  if (!mapOrSet || operands->empty())
763  return;
764 
765  assert(mapOrSet->getNumInputs() == operands->size() &&
766  "map/set inputs must match number of operands");
767 
768  auto *context = mapOrSet->getContext();
769  SmallVector<Value, 8> resultOperands;
770  resultOperands.reserve(operands->size());
771  SmallVector<Value, 8> remappedSymbols;
772  remappedSymbols.reserve(operands->size());
773  unsigned nextDim = 0;
774  unsigned nextSym = 0;
775  unsigned oldNumSyms = mapOrSet->getNumSymbols();
776  SmallVector<AffineExpr, 8> dimRemapping(mapOrSet->getNumDims());
777  for (unsigned i = 0, e = mapOrSet->getNumInputs(); i != e; ++i) {
778  if (i < mapOrSet->getNumDims()) {
779  if (isValidSymbol((*operands)[i])) {
780  // This is a valid symbol that appears as a dim, canonicalize it.
781  dimRemapping[i] = getAffineSymbolExpr(oldNumSyms + nextSym++, context);
782  remappedSymbols.push_back((*operands)[i]);
783  } else {
784  dimRemapping[i] = getAffineDimExpr(nextDim++, context);
785  resultOperands.push_back((*operands)[i]);
786  }
787  } else {
788  resultOperands.push_back((*operands)[i]);
789  }
790  }
791 
792  resultOperands.append(remappedSymbols.begin(), remappedSymbols.end());
793  *operands = resultOperands;
794  *mapOrSet = mapOrSet->replaceDimsAndSymbols(dimRemapping, {}, nextDim,
795  oldNumSyms + nextSym);
796 
797  assert(mapOrSet->getNumInputs() == operands->size() &&
798  "map/set inputs must match number of operands");
799 }
800 
801 // Works for either an affine map or an integer set.
802 template <class MapOrSet>
803 static void canonicalizeMapOrSetAndOperands(MapOrSet *mapOrSet,
804  SmallVectorImpl<Value> *operands) {
806  "Argument must be either of AffineMap or IntegerSet type");
807 
808  if (!mapOrSet || operands->empty())
809  return;
810 
811  assert(mapOrSet->getNumInputs() == operands->size() &&
812  "map/set inputs must match number of operands");
813 
814  canonicalizePromotedSymbols<MapOrSet>(mapOrSet, operands);
815 
816  // Check to see what dims are used.
817  llvm::SmallBitVector usedDims(mapOrSet->getNumDims());
818  llvm::SmallBitVector usedSyms(mapOrSet->getNumSymbols());
819  mapOrSet->walkExprs([&](AffineExpr expr) {
820  if (auto dimExpr = expr.dyn_cast<AffineDimExpr>())
821  usedDims[dimExpr.getPosition()] = true;
822  else if (auto symExpr = expr.dyn_cast<AffineSymbolExpr>())
823  usedSyms[symExpr.getPosition()] = true;
824  });
825 
826  auto *context = mapOrSet->getContext();
827 
828  SmallVector<Value, 8> resultOperands;
829  resultOperands.reserve(operands->size());
830 
831  llvm::SmallDenseMap<Value, AffineExpr, 8> seenDims;
832  SmallVector<AffineExpr, 8> dimRemapping(mapOrSet->getNumDims());
833  unsigned nextDim = 0;
834  for (unsigned i = 0, e = mapOrSet->getNumDims(); i != e; ++i) {
835  if (usedDims[i]) {
836  // Remap dim positions for duplicate operands.
837  auto it = seenDims.find((*operands)[i]);
838  if (it == seenDims.end()) {
839  dimRemapping[i] = getAffineDimExpr(nextDim++, context);
840  resultOperands.push_back((*operands)[i]);
841  seenDims.insert(std::make_pair((*operands)[i], dimRemapping[i]));
842  } else {
843  dimRemapping[i] = it->second;
844  }
845  }
846  }
847  llvm::SmallDenseMap<Value, AffineExpr, 8> seenSymbols;
848  SmallVector<AffineExpr, 8> symRemapping(mapOrSet->getNumSymbols());
849  unsigned nextSym = 0;
850  for (unsigned i = 0, e = mapOrSet->getNumSymbols(); i != e; ++i) {
851  if (!usedSyms[i])
852  continue;
853  // Handle constant operands (only needed for symbolic operands since
854  // constant operands in dimensional positions would have already been
855  // promoted to symbolic positions above).
856  IntegerAttr operandCst;
857  if (matchPattern((*operands)[i + mapOrSet->getNumDims()],
858  m_Constant(&operandCst))) {
859  symRemapping[i] =
860  getAffineConstantExpr(operandCst.getValue().getSExtValue(), context);
861  continue;
862  }
863  // Remap symbol positions for duplicate operands.
864  auto it = seenSymbols.find((*operands)[i + mapOrSet->getNumDims()]);
865  if (it == seenSymbols.end()) {
866  symRemapping[i] = getAffineSymbolExpr(nextSym++, context);
867  resultOperands.push_back((*operands)[i + mapOrSet->getNumDims()]);
868  seenSymbols.insert(std::make_pair((*operands)[i + mapOrSet->getNumDims()],
869  symRemapping[i]));
870  } else {
871  symRemapping[i] = it->second;
872  }
873  }
874  *mapOrSet = mapOrSet->replaceDimsAndSymbols(dimRemapping, symRemapping,
875  nextDim, nextSym);
876  *operands = resultOperands;
877 }
878 
880  SmallVectorImpl<Value> *operands) {
881  canonicalizeMapOrSetAndOperands<AffineMap>(map, operands);
882 }
883 
885  SmallVectorImpl<Value> *operands) {
886  canonicalizeMapOrSetAndOperands<IntegerSet>(set, operands);
887 }
888 
889 namespace {
890 /// Simplify AffineApply, AffineLoad, and AffineStore operations by composing
891 /// maps that supply results into them.
892 ///
893 template <typename AffineOpTy>
894 struct SimplifyAffineOp : public OpRewritePattern<AffineOpTy> {
896 
897  /// Replace the affine op with another instance of it with the supplied
898  /// map and mapOperands.
899  void replaceAffineOp(PatternRewriter &rewriter, AffineOpTy affineOp,
900  AffineMap map, ArrayRef<Value> mapOperands) const;
901 
902  LogicalResult matchAndRewrite(AffineOpTy affineOp,
903  PatternRewriter &rewriter) const override {
904  static_assert(
905  llvm::is_one_of<AffineOpTy, AffineLoadOp, AffinePrefetchOp,
906  AffineStoreOp, AffineApplyOp, AffineMinOp, AffineMaxOp,
907  AffineVectorStoreOp, AffineVectorLoadOp>::value,
908  "affine load/store/vectorstore/vectorload/apply/prefetch/min/max op "
909  "expected");
910  auto map = affineOp.getAffineMap();
911  AffineMap oldMap = map;
912  auto oldOperands = affineOp.getMapOperands();
913  SmallVector<Value, 8> resultOperands(oldOperands);
914  composeAffineMapAndOperands(&map, &resultOperands);
915  canonicalizeMapAndOperands(&map, &resultOperands);
916  if (map == oldMap && std::equal(oldOperands.begin(), oldOperands.end(),
917  resultOperands.begin()))
918  return failure();
919 
920  replaceAffineOp(rewriter, affineOp, map, resultOperands);
921  return success();
922  }
923 };
924 
925 // Specialize the template to account for the different build signatures for
926 // affine load, store, and apply ops.
927 template <>
928 void SimplifyAffineOp<AffineLoadOp>::replaceAffineOp(
929  PatternRewriter &rewriter, AffineLoadOp load, AffineMap map,
930  ArrayRef<Value> mapOperands) const {
931  rewriter.replaceOpWithNewOp<AffineLoadOp>(load, load.getMemRef(), map,
932  mapOperands);
933 }
934 template <>
935 void SimplifyAffineOp<AffinePrefetchOp>::replaceAffineOp(
936  PatternRewriter &rewriter, AffinePrefetchOp prefetch, AffineMap map,
937  ArrayRef<Value> mapOperands) const {
938  rewriter.replaceOpWithNewOp<AffinePrefetchOp>(
939  prefetch, prefetch.memref(), map, mapOperands, prefetch.localityHint(),
940  prefetch.isWrite(), prefetch.isDataCache());
941 }
942 template <>
943 void SimplifyAffineOp<AffineStoreOp>::replaceAffineOp(
944  PatternRewriter &rewriter, AffineStoreOp store, AffineMap map,
945  ArrayRef<Value> mapOperands) const {
946  rewriter.replaceOpWithNewOp<AffineStoreOp>(
947  store, store.getValueToStore(), store.getMemRef(), map, mapOperands);
948 }
949 template <>
950 void SimplifyAffineOp<AffineVectorLoadOp>::replaceAffineOp(
951  PatternRewriter &rewriter, AffineVectorLoadOp vectorload, AffineMap map,
952  ArrayRef<Value> mapOperands) const {
953  rewriter.replaceOpWithNewOp<AffineVectorLoadOp>(
954  vectorload, vectorload.getVectorType(), vectorload.getMemRef(), map,
955  mapOperands);
956 }
957 template <>
958 void SimplifyAffineOp<AffineVectorStoreOp>::replaceAffineOp(
959  PatternRewriter &rewriter, AffineVectorStoreOp vectorstore, AffineMap map,
960  ArrayRef<Value> mapOperands) const {
961  rewriter.replaceOpWithNewOp<AffineVectorStoreOp>(
962  vectorstore, vectorstore.getValueToStore(), vectorstore.getMemRef(), map,
963  mapOperands);
964 }
965 
966 // Generic version for ops that don't have extra operands.
967 template <typename AffineOpTy>
968 void SimplifyAffineOp<AffineOpTy>::replaceAffineOp(
969  PatternRewriter &rewriter, AffineOpTy op, AffineMap map,
970  ArrayRef<Value> mapOperands) const {
971  rewriter.replaceOpWithNewOp<AffineOpTy>(op, map, mapOperands);
972 }
973 } // namespace
974 
975 void AffineApplyOp::getCanonicalizationPatterns(RewritePatternSet &results,
976  MLIRContext *context) {
977  results.add<SimplifyAffineOp<AffineApplyOp>>(context);
978 }
979 
980 //===----------------------------------------------------------------------===//
981 // Common canonicalization pattern support logic
982 //===----------------------------------------------------------------------===//
983 
984 /// This is a common class used for patterns of the form
985 /// "someop(memrefcast) -> someop". It folds the source of any memref.cast
986 /// into the root operation directly.
987 static LogicalResult foldMemRefCast(Operation *op, Value ignore = nullptr) {
988  bool folded = false;
989  for (OpOperand &operand : op->getOpOperands()) {
990  auto cast = operand.get().getDefiningOp<memref::CastOp>();
991  if (cast && operand.get() != ignore &&
992  !cast.getOperand().getType().isa<UnrankedMemRefType>()) {
993  operand.set(cast.getOperand());
994  folded = true;
995  }
996  }
997  return success(folded);
998 }
999 
1000 //===----------------------------------------------------------------------===//
1001 // AffineDmaStartOp
1002 //===----------------------------------------------------------------------===//
1003 
1004 // TODO: Check that map operands are loop IVs or symbols.
1006  Value srcMemRef, AffineMap srcMap,
1007  ValueRange srcIndices, Value destMemRef,
1008  AffineMap dstMap, ValueRange destIndices,
1009  Value tagMemRef, AffineMap tagMap,
1010  ValueRange tagIndices, Value numElements,
1011  Value stride, Value elementsPerStride) {
1012  result.addOperands(srcMemRef);
1013  result.addAttribute(getSrcMapAttrName(), AffineMapAttr::get(srcMap));
1014  result.addOperands(srcIndices);
1015  result.addOperands(destMemRef);
1016  result.addAttribute(getDstMapAttrName(), AffineMapAttr::get(dstMap));
1017  result.addOperands(destIndices);
1018  result.addOperands(tagMemRef);
1019  result.addAttribute(getTagMapAttrName(), AffineMapAttr::get(tagMap));
1020  result.addOperands(tagIndices);
1021  result.addOperands(numElements);
1022  if (stride) {
1023  result.addOperands({stride, elementsPerStride});
1024  }
1025 }
1026 
1028  p << " " << getSrcMemRef() << '[';
1029  p.printAffineMapOfSSAIds(getSrcMapAttr(), getSrcIndices());
1030  p << "], " << getDstMemRef() << '[';
1031  p.printAffineMapOfSSAIds(getDstMapAttr(), getDstIndices());
1032  p << "], " << getTagMemRef() << '[';
1033  p.printAffineMapOfSSAIds(getTagMapAttr(), getTagIndices());
1034  p << "], " << getNumElements();
1035  if (isStrided()) {
1036  p << ", " << getStride();
1037  p << ", " << getNumElementsPerStride();
1038  }
1039  p << " : " << getSrcMemRefType() << ", " << getDstMemRefType() << ", "
1040  << getTagMemRefType();
1041 }
1042 
1043 // Parse AffineDmaStartOp.
1044 // Ex:
1045 // affine.dma_start %src[%i, %j], %dst[%k, %l], %tag[%index], %size,
1046 // %stride, %num_elt_per_stride
1047 // : memref<3076 x f32, 0>, memref<1024 x f32, 2>, memref<1 x i32>
1048 //
1050  OperationState &result) {
1051  OpAsmParser::OperandType srcMemRefInfo;
1052  AffineMapAttr srcMapAttr;
1054  OpAsmParser::OperandType dstMemRefInfo;
1055  AffineMapAttr dstMapAttr;
1057  OpAsmParser::OperandType tagMemRefInfo;
1058  AffineMapAttr tagMapAttr;
1060  OpAsmParser::OperandType numElementsInfo;
1062 
1063  SmallVector<Type, 3> types;
1064  auto indexType = parser.getBuilder().getIndexType();
1065 
1066  // Parse and resolve the following list of operands:
1067  // *) dst memref followed by its affine maps operands (in square brackets).
1068  // *) src memref followed by its affine map operands (in square brackets).
1069  // *) tag memref followed by its affine map operands (in square brackets).
1070  // *) number of elements transferred by DMA operation.
1071  if (parser.parseOperand(srcMemRefInfo) ||
1072  parser.parseAffineMapOfSSAIds(srcMapOperands, srcMapAttr,
1073  getSrcMapAttrName(), result.attributes) ||
1074  parser.parseComma() || parser.parseOperand(dstMemRefInfo) ||
1075  parser.parseAffineMapOfSSAIds(dstMapOperands, dstMapAttr,
1076  getDstMapAttrName(), result.attributes) ||
1077  parser.parseComma() || parser.parseOperand(tagMemRefInfo) ||
1078  parser.parseAffineMapOfSSAIds(tagMapOperands, tagMapAttr,
1079  getTagMapAttrName(), result.attributes) ||
1080  parser.parseComma() || parser.parseOperand(numElementsInfo))
1081  return failure();
1082 
1083  // Parse optional stride and elements per stride.
1084  if (parser.parseTrailingOperandList(strideInfo)) {
1085  return failure();
1086  }
1087  if (!strideInfo.empty() && strideInfo.size() != 2) {
1088  return parser.emitError(parser.getNameLoc(),
1089  "expected two stride related operands");
1090  }
1091  bool isStrided = strideInfo.size() == 2;
1092 
1093  if (parser.parseColonTypeList(types))
1094  return failure();
1095 
1096  if (types.size() != 3)
1097  return parser.emitError(parser.getNameLoc(), "expected three types");
1098 
1099  if (parser.resolveOperand(srcMemRefInfo, types[0], result.operands) ||
1100  parser.resolveOperands(srcMapOperands, indexType, result.operands) ||
1101  parser.resolveOperand(dstMemRefInfo, types[1], result.operands) ||
1102  parser.resolveOperands(dstMapOperands, indexType, result.operands) ||
1103  parser.resolveOperand(tagMemRefInfo, types[2], result.operands) ||
1104  parser.resolveOperands(tagMapOperands, indexType, result.operands) ||
1105  parser.resolveOperand(numElementsInfo, indexType, result.operands))
1106  return failure();
1107 
1108  if (isStrided) {
1109  if (parser.resolveOperands(strideInfo, indexType, result.operands))
1110  return failure();
1111  }
1112 
1113  // Check that src/dst/tag operand counts match their map.numInputs.
1114  if (srcMapOperands.size() != srcMapAttr.getValue().getNumInputs() ||
1115  dstMapOperands.size() != dstMapAttr.getValue().getNumInputs() ||
1116  tagMapOperands.size() != tagMapAttr.getValue().getNumInputs())
1117  return parser.emitError(parser.getNameLoc(),
1118  "memref operand count not equal to map.numInputs");
1119  return success();
1120 }
1121 
1123  if (!getOperand(getSrcMemRefOperandIndex()).getType().isa<MemRefType>())
1124  return emitOpError("expected DMA source to be of memref type");
1125  if (!getOperand(getDstMemRefOperandIndex()).getType().isa<MemRefType>())
1126  return emitOpError("expected DMA destination to be of memref type");
1127  if (!getOperand(getTagMemRefOperandIndex()).getType().isa<MemRefType>())
1128  return emitOpError("expected DMA tag to be of memref type");
1129 
1130  unsigned numInputsAllMaps = getSrcMap().getNumInputs() +
1131  getDstMap().getNumInputs() +
1132  getTagMap().getNumInputs();
1133  if (getNumOperands() != numInputsAllMaps + 3 + 1 &&
1134  getNumOperands() != numInputsAllMaps + 3 + 1 + 2) {
1135  return emitOpError("incorrect number of operands");
1136  }
1137 
1138  Region *scope = getAffineScope(*this);
1139  for (auto idx : getSrcIndices()) {
1140  if (!idx.getType().isIndex())
1141  return emitOpError("src index to dma_start must have 'index' type");
1142  if (!isValidAffineIndexOperand(idx, scope))
1143  return emitOpError("src index must be a dimension or symbol identifier");
1144  }
1145  for (auto idx : getDstIndices()) {
1146  if (!idx.getType().isIndex())
1147  return emitOpError("dst index to dma_start must have 'index' type");
1148  if (!isValidAffineIndexOperand(idx, scope))
1149  return emitOpError("dst index must be a dimension or symbol identifier");
1150  }
1151  for (auto idx : getTagIndices()) {
1152  if (!idx.getType().isIndex())
1153  return emitOpError("tag index to dma_start must have 'index' type");
1154  if (!isValidAffineIndexOperand(idx, scope))
1155  return emitOpError("tag index must be a dimension or symbol identifier");
1156  }
1157  return success();
1158 }
1159 
1161  SmallVectorImpl<OpFoldResult> &results) {
1162  /// dma_start(memrefcast) -> dma_start
1163  return foldMemRefCast(*this);
1164 }
1165 
1166 //===----------------------------------------------------------------------===//
1167 // AffineDmaWaitOp
1168 //===----------------------------------------------------------------------===//
1169 
1170 // TODO: Check that map operands are loop IVs or symbols.
1172  Value tagMemRef, AffineMap tagMap,
1173  ValueRange tagIndices, Value numElements) {
1174  result.addOperands(tagMemRef);
1175  result.addAttribute(getTagMapAttrName(), AffineMapAttr::get(tagMap));
1176  result.addOperands(tagIndices);
1177  result.addOperands(numElements);
1178 }
1179 
1181  p << " " << getTagMemRef() << '[';
1182  SmallVector<Value, 2> operands(getTagIndices());
1183  p.printAffineMapOfSSAIds(getTagMapAttr(), operands);
1184  p << "], ";
1186  p << " : " << getTagMemRef().getType();
1187 }
1188 
1189 // Parse AffineDmaWaitOp.
1190 // Eg:
1191 // affine.dma_wait %tag[%index], %num_elements
1192 // : memref<1 x i32, (d0) -> (d0), 4>
1193 //
1195  OperationState &result) {
1196  OpAsmParser::OperandType tagMemRefInfo;
1197  AffineMapAttr tagMapAttr;
1199  Type type;
1200  auto indexType = parser.getBuilder().getIndexType();
1201  OpAsmParser::OperandType numElementsInfo;
1202 
1203  // Parse tag memref, its map operands, and dma size.
1204  if (parser.parseOperand(tagMemRefInfo) ||
1205  parser.parseAffineMapOfSSAIds(tagMapOperands, tagMapAttr,
1206  getTagMapAttrName(), result.attributes) ||
1207  parser.parseComma() || parser.parseOperand(numElementsInfo) ||
1208  parser.parseColonType(type) ||
1209  parser.resolveOperand(tagMemRefInfo, type, result.operands) ||
1210  parser.resolveOperands(tagMapOperands, indexType, result.operands) ||
1211  parser.resolveOperand(numElementsInfo, indexType, result.operands))
1212  return failure();
1213 
1214  if (!type.isa<MemRefType>())
1215  return parser.emitError(parser.getNameLoc(),
1216  "expected tag to be of memref type");
1217 
1218  if (tagMapOperands.size() != tagMapAttr.getValue().getNumInputs())
1219  return parser.emitError(parser.getNameLoc(),
1220  "tag memref operand count != to map.numInputs");
1221  return success();
1222 }
1223 
1225  if (!getOperand(0).getType().isa<MemRefType>())
1226  return emitOpError("expected DMA tag to be of memref type");
1227  Region *scope = getAffineScope(*this);
1228  for (auto idx : getTagIndices()) {
1229  if (!idx.getType().isIndex())
1230  return emitOpError("index to dma_wait must have 'index' type");
1231  if (!isValidAffineIndexOperand(idx, scope))
1232  return emitOpError("index must be a dimension or symbol identifier");
1233  }
1234  return success();
1235 }
1236 
1238  SmallVectorImpl<OpFoldResult> &results) {
1239  /// dma_wait(memrefcast) -> dma_wait
1240  return foldMemRefCast(*this);
1241 }
1242 
1243 //===----------------------------------------------------------------------===//
1244 // AffineForOp
1245 //===----------------------------------------------------------------------===//
1246 
1247 /// 'bodyBuilder' is used to build the body of affine.for. If iterArgs and
1248 /// bodyBuilder are empty/null, we include default terminator op.
1249 void AffineForOp::build(OpBuilder &builder, OperationState &result,
1250  ValueRange lbOperands, AffineMap lbMap,
1251  ValueRange ubOperands, AffineMap ubMap, int64_t step,
1252  ValueRange iterArgs, BodyBuilderFn bodyBuilder) {
1253  assert(((!lbMap && lbOperands.empty()) ||
1254  lbOperands.size() == lbMap.getNumInputs()) &&
1255  "lower bound operand count does not match the affine map");
1256  assert(((!ubMap && ubOperands.empty()) ||
1257  ubOperands.size() == ubMap.getNumInputs()) &&
1258  "upper bound operand count does not match the affine map");
1259  assert(step > 0 && "step has to be a positive integer constant");
1260 
1261  for (Value val : iterArgs)
1262  result.addTypes(val.getType());
1263 
1264  // Add an attribute for the step.
1265  result.addAttribute(getStepAttrName(),
1266  builder.getIntegerAttr(builder.getIndexType(), step));
1267 
1268  // Add the lower bound.
1269  result.addAttribute(getLowerBoundAttrName(), AffineMapAttr::get(lbMap));
1270  result.addOperands(lbOperands);
1271 
1272  // Add the upper bound.
1273  result.addAttribute(getUpperBoundAttrName(), AffineMapAttr::get(ubMap));
1274  result.addOperands(ubOperands);
1275 
1276  result.addOperands(iterArgs);
1277  // Create a region and a block for the body. The argument of the region is
1278  // the loop induction variable.
1279  Region *bodyRegion = result.addRegion();
1280  bodyRegion->push_back(new Block);
1281  Block &bodyBlock = bodyRegion->front();
1282  Value inductionVar =
1283  bodyBlock.addArgument(builder.getIndexType(), result.location);
1284  for (Value val : iterArgs)
1285  bodyBlock.addArgument(val.getType(), val.getLoc());
1286 
1287  // Create the default terminator if the builder is not provided and if the
1288  // iteration arguments are not provided. Otherwise, leave this to the caller
1289  // because we don't know which values to return from the loop.
1290  if (iterArgs.empty() && !bodyBuilder) {
1291  ensureTerminator(*bodyRegion, builder, result.location);
1292  } else if (bodyBuilder) {
1293  OpBuilder::InsertionGuard guard(builder);
1294  builder.setInsertionPointToStart(&bodyBlock);
1295  bodyBuilder(builder, result.location, inductionVar,
1296  bodyBlock.getArguments().drop_front());
1297  }
1298 }
1299 
1300 void AffineForOp::build(OpBuilder &builder, OperationState &result, int64_t lb,
1301  int64_t ub, int64_t step, ValueRange iterArgs,
1302  BodyBuilderFn bodyBuilder) {
1303  auto lbMap = AffineMap::getConstantMap(lb, builder.getContext());
1304  auto ubMap = AffineMap::getConstantMap(ub, builder.getContext());
1305  return build(builder, result, {}, lbMap, {}, ubMap, step, iterArgs,
1306  bodyBuilder);
1307 }
1308 
1309 static LogicalResult verify(AffineForOp op) {
1310  // Check that the body defines as single block argument for the induction
1311  // variable.
1312  auto *body = op.getBody();
1313  if (body->getNumArguments() == 0 || !body->getArgument(0).getType().isIndex())
1314  return op.emitOpError(
1315  "expected body to have a single index argument for the "
1316  "induction variable");
1317 
1318  // Verify that the bound operands are valid dimension/symbols.
1319  /// Lower bound.
1320  if (op.getLowerBoundMap().getNumInputs() > 0)
1321  if (failed(
1322  verifyDimAndSymbolIdentifiers(op, op.getLowerBoundOperands(),
1323  op.getLowerBoundMap().getNumDims())))
1324  return failure();
1325  /// Upper bound.
1326  if (op.getUpperBoundMap().getNumInputs() > 0)
1327  if (failed(
1328  verifyDimAndSymbolIdentifiers(op, op.getUpperBoundOperands(),
1329  op.getUpperBoundMap().getNumDims())))
1330  return failure();
1331 
1332  unsigned opNumResults = op.getNumResults();
1333  if (opNumResults == 0)
1334  return success();
1335 
1336  // If ForOp defines values, check that the number and types of the defined
1337  // values match ForOp initial iter operands and backedge basic block
1338  // arguments.
1339  if (op.getNumIterOperands() != opNumResults)
1340  return op.emitOpError(
1341  "mismatch between the number of loop-carried values and results");
1342  if (op.getNumRegionIterArgs() != opNumResults)
1343  return op.emitOpError(
1344  "mismatch between the number of basic block args and results");
1345 
1346  return success();
1347 }
1348 
1349 /// Parse a for operation loop bounds.
1350 static ParseResult parseBound(bool isLower, OperationState &result,
1351  OpAsmParser &p) {
1352  // 'min' / 'max' prefixes are generally syntactic sugar, but are required if
1353  // the map has multiple results.
1354  bool failedToParsedMinMax =
1355  failed(p.parseOptionalKeyword(isLower ? "max" : "min"));
1356 
1357  auto &builder = p.getBuilder();
1358  auto boundAttrName = isLower ? AffineForOp::getLowerBoundAttrName()
1359  : AffineForOp::getUpperBoundAttrName();
1360 
1361  // Parse ssa-id as identity map.
1363  if (p.parseOperandList(boundOpInfos))
1364  return failure();
1365 
1366  if (!boundOpInfos.empty()) {
1367  // Check that only one operand was parsed.
1368  if (boundOpInfos.size() > 1)
1369  return p.emitError(p.getNameLoc(),
1370  "expected only one loop bound operand");
1371 
1372  // TODO: improve error message when SSA value is not of index type.
1373  // Currently it is 'use of value ... expects different type than prior uses'
1374  if (p.resolveOperand(boundOpInfos.front(), builder.getIndexType(),
1375  result.operands))
1376  return failure();
1377 
1378  // Create an identity map using symbol id. This representation is optimized
1379  // for storage. Analysis passes may expand it into a multi-dimensional map
1380  // if desired.
1381  AffineMap map = builder.getSymbolIdentityMap();
1382  result.addAttribute(boundAttrName, AffineMapAttr::get(map));
1383  return success();
1384  }
1385 
1386  // Get the attribute location.
1387  llvm::SMLoc attrLoc = p.getCurrentLocation();
1388 
1389  Attribute boundAttr;
1390  if (p.parseAttribute(boundAttr, builder.getIndexType(), boundAttrName,
1391  result.attributes))
1392  return failure();
1393 
1394  // Parse full form - affine map followed by dim and symbol list.
1395  if (auto affineMapAttr = boundAttr.dyn_cast<AffineMapAttr>()) {
1396  unsigned currentNumOperands = result.operands.size();
1397  unsigned numDims;
1398  if (parseDimAndSymbolList(p, result.operands, numDims))
1399  return failure();
1400 
1401  auto map = affineMapAttr.getValue();
1402  if (map.getNumDims() != numDims)
1403  return p.emitError(
1404  p.getNameLoc(),
1405  "dim operand count and affine map dim count must match");
1406 
1407  unsigned numDimAndSymbolOperands =
1408  result.operands.size() - currentNumOperands;
1409  if (numDims + map.getNumSymbols() != numDimAndSymbolOperands)
1410  return p.emitError(
1411  p.getNameLoc(),
1412  "symbol operand count and affine map symbol count must match");
1413 
1414  // If the map has multiple results, make sure that we parsed the min/max
1415  // prefix.
1416  if (map.getNumResults() > 1 && failedToParsedMinMax) {
1417  if (isLower) {
1418  return p.emitError(attrLoc, "lower loop bound affine map with "
1419  "multiple results requires 'max' prefix");
1420  }
1421  return p.emitError(attrLoc, "upper loop bound affine map with multiple "
1422  "results requires 'min' prefix");
1423  }
1424  return success();
1425  }
1426 
1427  // Parse custom assembly form.
1428  if (auto integerAttr = boundAttr.dyn_cast<IntegerAttr>()) {
1429  result.attributes.pop_back();
1430  result.addAttribute(
1431  boundAttrName,
1432  AffineMapAttr::get(builder.getConstantAffineMap(integerAttr.getInt())));
1433  return success();
1434  }
1435 
1436  return p.emitError(
1437  p.getNameLoc(),
1438  "expected valid affine map representation for loop bounds");
1439 }
1440 
1442  OperationState &result) {
1443  auto &builder = parser.getBuilder();
1444  OpAsmParser::OperandType inductionVariable;
1445  // Parse the induction variable followed by '='.
1446  if (parser.parseRegionArgument(inductionVariable) || parser.parseEqual())
1447  return failure();
1448 
1449  // Parse loop bounds.
1450  if (parseBound(/*isLower=*/true, result, parser) ||
1451  parser.parseKeyword("to", " between bounds") ||
1452  parseBound(/*isLower=*/false, result, parser))
1453  return failure();
1454 
1455  // Parse the optional loop step, we default to 1 if one is not present.
1456  if (parser.parseOptionalKeyword("step")) {
1457  result.addAttribute(
1458  AffineForOp::getStepAttrName(),
1459  builder.getIntegerAttr(builder.getIndexType(), /*value=*/1));
1460  } else {
1461  llvm::SMLoc stepLoc = parser.getCurrentLocation();
1462  IntegerAttr stepAttr;
1463  if (parser.parseAttribute(stepAttr, builder.getIndexType(),
1464  AffineForOp::getStepAttrName().data(),
1465  result.attributes))
1466  return failure();
1467 
1468  if (stepAttr.getValue().getSExtValue() < 0)
1469  return parser.emitError(
1470  stepLoc,
1471  "expected step to be representable as a positive signed integer");
1472  }
1473 
1474  // Parse the optional initial iteration arguments.
1475  SmallVector<OpAsmParser::OperandType, 4> regionArgs, operands;
1476  SmallVector<Type, 4> argTypes;
1477  regionArgs.push_back(inductionVariable);
1478 
1479  if (succeeded(parser.parseOptionalKeyword("iter_args"))) {
1480  // Parse assignment list and results type list.
1481  if (parser.parseAssignmentList(regionArgs, operands) ||
1482  parser.parseArrowTypeList(result.types))
1483  return failure();
1484  // Resolve input operands.
1485  for (auto operandType : llvm::zip(operands, result.types))
1486  if (parser.resolveOperand(std::get<0>(operandType),
1487  std::get<1>(operandType), result.operands))
1488  return failure();
1489  }
1490  // Induction variable.
1491  Type indexType = builder.getIndexType();
1492  argTypes.push_back(indexType);
1493  // Loop carried variables.
1494  argTypes.append(result.types.begin(), result.types.end());
1495  // Parse the body region.
1496  Region *body = result.addRegion();
1497  if (regionArgs.size() != argTypes.size())
1498  return parser.emitError(
1499  parser.getNameLoc(),
1500  "mismatch between the number of loop-carried values and results");
1501  if (parser.parseRegion(*body, regionArgs, argTypes))
1502  return failure();
1503 
1504  AffineForOp::ensureTerminator(*body, builder, result.location);
1505 
1506  // Parse the optional attribute list.
1507  return parser.parseOptionalAttrDict(result.attributes);
1508 }
1509 
1510 static void printBound(AffineMapAttr boundMap,
1511  Operation::operand_range boundOperands,
1512  const char *prefix, OpAsmPrinter &p) {
1513  AffineMap map = boundMap.getValue();
1514 
1515  // Check if this bound should be printed using custom assembly form.
1516  // The decision to restrict printing custom assembly form to trivial cases
1517  // comes from the will to roundtrip MLIR binary -> text -> binary in a
1518  // lossless way.
1519  // Therefore, custom assembly form parsing and printing is only supported for
1520  // zero-operand constant maps and single symbol operand identity maps.
1521  if (map.getNumResults() == 1) {
1522  AffineExpr expr = map.getResult(0);
1523 
1524  // Print constant bound.
1525  if (map.getNumDims() == 0 && map.getNumSymbols() == 0) {
1526  if (auto constExpr = expr.dyn_cast<AffineConstantExpr>()) {
1527  p << constExpr.getValue();
1528  return;
1529  }
1530  }
1531 
1532  // Print bound that consists of a single SSA symbol if the map is over a
1533  // single symbol.
1534  if (map.getNumDims() == 0 && map.getNumSymbols() == 1) {
1535  if (auto symExpr = expr.dyn_cast<AffineSymbolExpr>()) {
1536  p.printOperand(*boundOperands.begin());
1537  return;
1538  }
1539  }
1540  } else {
1541  // Map has multiple results. Print 'min' or 'max' prefix.
1542  p << prefix << ' ';
1543  }
1544 
1545  // Print the map and its operands.
1546  p << boundMap;
1547  printDimAndSymbolList(boundOperands.begin(), boundOperands.end(),
1548  map.getNumDims(), p);
1549 }
1550 
1551 unsigned AffineForOp::getNumIterOperands() {
1552  AffineMap lbMap = getLowerBoundMapAttr().getValue();
1553  AffineMap ubMap = getUpperBoundMapAttr().getValue();
1554 
1555  return getNumOperands() - lbMap.getNumInputs() - ubMap.getNumInputs();
1556 }
1557 
1558 static void print(OpAsmPrinter &p, AffineForOp op) {
1559  p << ' ';
1560  p.printOperand(op.getBody()->getArgument(0));
1561  p << " = ";
1562  printBound(op.getLowerBoundMapAttr(), op.getLowerBoundOperands(), "max", p);
1563  p << " to ";
1564  printBound(op.getUpperBoundMapAttr(), op.getUpperBoundOperands(), "min", p);
1565 
1566  if (op.getStep() != 1)
1567  p << " step " << op.getStep();
1568 
1569  bool printBlockTerminators = false;
1570  if (op.getNumIterOperands() > 0) {
1571  p << " iter_args(";
1572  auto regionArgs = op.getRegionIterArgs();
1573  auto operands = op.getIterOperands();
1574 
1575  llvm::interleaveComma(llvm::zip(regionArgs, operands), p, [&](auto it) {
1576  p << std::get<0>(it) << " = " << std::get<1>(it);
1577  });
1578  p << ") -> (" << op.getResultTypes() << ")";
1579  printBlockTerminators = true;
1580  }
1581 
1582  p << ' ';
1583  p.printRegion(op.region(),
1584  /*printEntryBlockArgs=*/false, printBlockTerminators);
1585  p.printOptionalAttrDict(op->getAttrs(),
1586  /*elidedAttrs=*/{op.getLowerBoundAttrName(),
1587  op.getUpperBoundAttrName(),
1588  op.getStepAttrName()});
1589 }
1590 
1591 /// Fold the constant bounds of a loop.
1592 static LogicalResult foldLoopBounds(AffineForOp forOp) {
1593  auto foldLowerOrUpperBound = [&forOp](bool lower) {
1594  // Check to see if each of the operands is the result of a constant. If
1595  // so, get the value. If not, ignore it.
1596  SmallVector<Attribute, 8> operandConstants;
1597  auto boundOperands =
1598  lower ? forOp.getLowerBoundOperands() : forOp.getUpperBoundOperands();
1599  for (auto operand : boundOperands) {
1600  Attribute operandCst;
1601  matchPattern(operand, m_Constant(&operandCst));
1602  operandConstants.push_back(operandCst);
1603  }
1604 
1605  AffineMap boundMap =
1606  lower ? forOp.getLowerBoundMap() : forOp.getUpperBoundMap();
1607  assert(boundMap.getNumResults() >= 1 &&
1608  "bound maps should have at least one result");
1609  SmallVector<Attribute, 4> foldedResults;
1610  if (failed(boundMap.constantFold(operandConstants, foldedResults)))
1611  return failure();
1612 
1613  // Compute the max or min as applicable over the results.
1614  assert(!foldedResults.empty() && "bounds should have at least one result");
1615  auto maxOrMin = foldedResults[0].cast<IntegerAttr>().getValue();
1616  for (unsigned i = 1, e = foldedResults.size(); i < e; i++) {
1617  auto foldedResult = foldedResults[i].cast<IntegerAttr>().getValue();
1618  maxOrMin = lower ? llvm::APIntOps::smax(maxOrMin, foldedResult)
1619  : llvm::APIntOps::smin(maxOrMin, foldedResult);
1620  }
1621  lower ? forOp.setConstantLowerBound(maxOrMin.getSExtValue())
1622  : forOp.setConstantUpperBound(maxOrMin.getSExtValue());
1623  return success();
1624  };
1625 
1626  // Try to fold the lower bound.
1627  bool folded = false;
1628  if (!forOp.hasConstantLowerBound())
1629  folded |= succeeded(foldLowerOrUpperBound(/*lower=*/true));
1630 
1631  // Try to fold the upper bound.
1632  if (!forOp.hasConstantUpperBound())
1633  folded |= succeeded(foldLowerOrUpperBound(/*lower=*/false));
1634  return success(folded);
1635 }
1636 
1637 /// Canonicalize the bounds of the given loop.
1638 static LogicalResult canonicalizeLoopBounds(AffineForOp forOp) {
1639  SmallVector<Value, 4> lbOperands(forOp.getLowerBoundOperands());
1640  SmallVector<Value, 4> ubOperands(forOp.getUpperBoundOperands());
1641 
1642  auto lbMap = forOp.getLowerBoundMap();
1643  auto ubMap = forOp.getUpperBoundMap();
1644  auto prevLbMap = lbMap;
1645  auto prevUbMap = ubMap;
1646 
1647  composeAffineMapAndOperands(&lbMap, &lbOperands);
1648  canonicalizeMapAndOperands(&lbMap, &lbOperands);
1649  lbMap = removeDuplicateExprs(lbMap);
1650 
1651  composeAffineMapAndOperands(&ubMap, &ubOperands);
1652  canonicalizeMapAndOperands(&ubMap, &ubOperands);
1653  ubMap = removeDuplicateExprs(ubMap);
1654 
1655  // Any canonicalization change always leads to updated map(s).
1656  if (lbMap == prevLbMap && ubMap == prevUbMap)
1657  return failure();
1658 
1659  if (lbMap != prevLbMap)
1660  forOp.setLowerBound(lbOperands, lbMap);
1661  if (ubMap != prevUbMap)
1662  forOp.setUpperBound(ubOperands, ubMap);
1663  return success();
1664 }
1665 
1666 namespace {
1667 /// This is a pattern to fold trivially empty loop bodies.
1668 /// TODO: This should be moved into the folding hook.
1669 struct AffineForEmptyLoopFolder : public OpRewritePattern<AffineForOp> {
1671 
1672  LogicalResult matchAndRewrite(AffineForOp forOp,
1673  PatternRewriter &rewriter) const override {
1674  // Check that the body only contains a yield.
1675  if (!llvm::hasSingleElement(*forOp.getBody()))
1676  return failure();
1677  // The initial values of the iteration arguments would be the op's results.
1678  rewriter.replaceOp(forOp, forOp.getIterOperands());
1679  return success();
1680  }
1681 };
1682 } // namespace
1683 
1684 void AffineForOp::getCanonicalizationPatterns(RewritePatternSet &results,
1685  MLIRContext *context) {
1686  results.add<AffineForEmptyLoopFolder>(context);
1687 }
1688 
1689 /// Returns true if the affine.for has zero iterations in trivial cases.
1690 static bool hasTrivialZeroTripCount(AffineForOp op) {
1691  if (!op.hasConstantBounds())
1692  return false;
1693  int64_t lb = op.getConstantLowerBound();
1694  int64_t ub = op.getConstantUpperBound();
1695  return ub - lb <= 0;
1696 }
1697 
1698 LogicalResult AffineForOp::fold(ArrayRef<Attribute> operands,
1699  SmallVectorImpl<OpFoldResult> &results) {
1700  bool folded = succeeded(foldLoopBounds(*this));
1701  folded |= succeeded(canonicalizeLoopBounds(*this));
1702  if (hasTrivialZeroTripCount(*this)) {
1703  // The initial values of the loop-carried variables (iter_args) are the
1704  // results of the op.
1705  results.assign(getIterOperands().begin(), getIterOperands().end());
1706  folded = true;
1707  }
1708  return success(folded);
1709 }
1710 
1711 AffineBound AffineForOp::getLowerBound() {
1712  auto lbMap = getLowerBoundMap();
1713  return AffineBound(AffineForOp(*this), 0, lbMap.getNumInputs(), lbMap);
1714 }
1715 
1716 AffineBound AffineForOp::getUpperBound() {
1717  auto lbMap = getLowerBoundMap();
1718  auto ubMap = getUpperBoundMap();
1719  return AffineBound(AffineForOp(*this), lbMap.getNumInputs(),
1720  lbMap.getNumInputs() + ubMap.getNumInputs(), ubMap);
1721 }
1722 
1723 void AffineForOp::setLowerBound(ValueRange lbOperands, AffineMap map) {
1724  assert(lbOperands.size() == map.getNumInputs());
1725  assert(map.getNumResults() >= 1 && "bound map has at least one result");
1726 
1727  SmallVector<Value, 4> newOperands(lbOperands.begin(), lbOperands.end());
1728 
1729  auto ubOperands = getUpperBoundOperands();
1730  newOperands.append(ubOperands.begin(), ubOperands.end());
1731  auto iterOperands = getIterOperands();
1732  newOperands.append(iterOperands.begin(), iterOperands.end());
1733  (*this)->setOperands(newOperands);
1734 
1735  (*this)->setAttr(getLowerBoundAttrName(), AffineMapAttr::get(map));
1736 }
1737 
1738 void AffineForOp::setUpperBound(ValueRange ubOperands, AffineMap map) {
1739  assert(ubOperands.size() == map.getNumInputs());
1740  assert(map.getNumResults() >= 1 && "bound map has at least one result");
1741 
1743  newOperands.append(ubOperands.begin(), ubOperands.end());
1744  auto iterOperands = getIterOperands();
1745  newOperands.append(iterOperands.begin(), iterOperands.end());
1746  (*this)->setOperands(newOperands);
1747 
1748  (*this)->setAttr(getUpperBoundAttrName(), AffineMapAttr::get(map));
1749 }
1750 
1751 void AffineForOp::setLowerBoundMap(AffineMap map) {
1752  auto lbMap = getLowerBoundMap();
1753  assert(lbMap.getNumDims() == map.getNumDims() &&
1754  lbMap.getNumSymbols() == map.getNumSymbols());
1755  assert(map.getNumResults() >= 1 && "bound map has at least one result");
1756  (void)lbMap;
1757  (*this)->setAttr(getLowerBoundAttrName(), AffineMapAttr::get(map));
1758 }
1759 
1760 void AffineForOp::setUpperBoundMap(AffineMap map) {
1761  auto ubMap = getUpperBoundMap();
1762  assert(ubMap.getNumDims() == map.getNumDims() &&
1763  ubMap.getNumSymbols() == map.getNumSymbols());
1764  assert(map.getNumResults() >= 1 && "bound map has at least one result");
1765  (void)ubMap;
1766  (*this)->setAttr(getUpperBoundAttrName(), AffineMapAttr::get(map));
1767 }
1768 
1769 bool AffineForOp::hasConstantLowerBound() {
1770  return getLowerBoundMap().isSingleConstant();
1771 }
1772 
1773 bool AffineForOp::hasConstantUpperBound() {
1774  return getUpperBoundMap().isSingleConstant();
1775 }
1776 
1777 int64_t AffineForOp::getConstantLowerBound() {
1778  return getLowerBoundMap().getSingleConstantResult();
1779 }
1780 
1781 int64_t AffineForOp::getConstantUpperBound() {
1782  return getUpperBoundMap().getSingleConstantResult();
1783 }
1784 
1785 void AffineForOp::setConstantLowerBound(int64_t value) {
1786  setLowerBound({}, AffineMap::getConstantMap(value, getContext()));
1787 }
1788 
1789 void AffineForOp::setConstantUpperBound(int64_t value) {
1790  setUpperBound({}, AffineMap::getConstantMap(value, getContext()));
1791 }
1792 
1793 AffineForOp::operand_range AffineForOp::getLowerBoundOperands() {
1794  return {operand_begin(), operand_begin() + getLowerBoundMap().getNumInputs()};
1795 }
1796 
1797 AffineForOp::operand_range AffineForOp::getUpperBoundOperands() {
1798  return {operand_begin() + getLowerBoundMap().getNumInputs(),
1799  operand_begin() + getLowerBoundMap().getNumInputs() +
1800  getUpperBoundMap().getNumInputs()};
1801 }
1802 
1803 AffineForOp::operand_range AffineForOp::getControlOperands() {
1804  return {operand_begin(), operand_begin() + getLowerBoundMap().getNumInputs() +
1805  getUpperBoundMap().getNumInputs()};
1806 }
1807 
1808 bool AffineForOp::matchingBoundOperandList() {
1809  auto lbMap = getLowerBoundMap();
1810  auto ubMap = getUpperBoundMap();
1811  if (lbMap.getNumDims() != ubMap.getNumDims() ||
1812  lbMap.getNumSymbols() != ubMap.getNumSymbols())
1813  return false;
1814 
1815  unsigned numOperands = lbMap.getNumInputs();
1816  for (unsigned i = 0, e = lbMap.getNumInputs(); i < e; i++) {
1817  // Compare Value 's.
1818  if (getOperand(i) != getOperand(numOperands + i))
1819  return false;
1820  }
1821  return true;
1822 }
1823 
1824 Region &AffineForOp::getLoopBody() { return region(); }
1825 
1826 bool AffineForOp::isDefinedOutsideOfLoop(Value value) {
1827  return !region().isAncestor(value.getParentRegion());
1828 }
1829 
1830 LogicalResult AffineForOp::moveOutOfLoop(ArrayRef<Operation *> ops) {
1831  for (auto *op : ops)
1832  op->moveBefore(*this);
1833  return success();
1834 }
1835 
1836 /// Returns true if the provided value is the induction variable of a
1837 /// AffineForOp.
1839  return getForInductionVarOwner(val) != AffineForOp();
1840 }
1841 
1842 /// Returns the loop parent of an induction variable. If the provided value is
1843 /// not an induction variable, then return nullptr.
1845  auto ivArg = val.dyn_cast<BlockArgument>();
1846  if (!ivArg || !ivArg.getOwner())
1847  return AffineForOp();
1848  auto *containingInst = ivArg.getOwner()->getParent()->getParentOp();
1849  if (auto forOp = dyn_cast<AffineForOp>(containingInst))
1850  // Check to make sure `val` is the induction variable, not an iter_arg.
1851  return forOp.getInductionVar() == val ? forOp : AffineForOp();
1852  return AffineForOp();
1853 }
1854 
1855 /// Extracts the induction variables from a list of AffineForOps and returns
1856 /// them.
1858  SmallVectorImpl<Value> *ivs) {
1859  ivs->reserve(forInsts.size());
1860  for (auto forInst : forInsts)
1861  ivs->push_back(forInst.getInductionVar());
1862 }
1863 
1864 /// Builds an affine loop nest, using "loopCreatorFn" to create individual loop
1865 /// operations.
1866 template <typename BoundListTy, typename LoopCreatorTy>
1868  OpBuilder &builder, Location loc, BoundListTy lbs, BoundListTy ubs,
1869  ArrayRef<int64_t> steps,
1870  function_ref<void(OpBuilder &, Location, ValueRange)> bodyBuilderFn,
1871  LoopCreatorTy &&loopCreatorFn) {
1872  assert(lbs.size() == ubs.size() && "Mismatch in number of arguments");
1873  assert(lbs.size() == steps.size() && "Mismatch in number of arguments");
1874 
1875  // If there are no loops to be constructed, construct the body anyway.
1876  OpBuilder::InsertionGuard guard(builder);
1877  if (lbs.empty()) {
1878  if (bodyBuilderFn)
1879  bodyBuilderFn(builder, loc, ValueRange());
1880  return;
1881  }
1882 
1883  // Create the loops iteratively and store the induction variables.
1885  ivs.reserve(lbs.size());
1886  for (unsigned i = 0, e = lbs.size(); i < e; ++i) {
1887  // Callback for creating the loop body, always creates the terminator.
1888  auto loopBody = [&](OpBuilder &nestedBuilder, Location nestedLoc, Value iv,
1889  ValueRange iterArgs) {
1890  ivs.push_back(iv);
1891  // In the innermost loop, call the body builder.
1892  if (i == e - 1 && bodyBuilderFn) {
1893  OpBuilder::InsertionGuard nestedGuard(nestedBuilder);
1894  bodyBuilderFn(nestedBuilder, nestedLoc, ivs);
1895  }
1896  nestedBuilder.create<AffineYieldOp>(nestedLoc);
1897  };
1898 
1899  // Delegate actual loop creation to the callback in order to dispatch
1900  // between constant- and variable-bound loops.
1901  auto loop = loopCreatorFn(builder, loc, lbs[i], ubs[i], steps[i], loopBody);
1902  builder.setInsertionPointToStart(loop.getBody());
1903  }
1904 }
1905 
1906 /// Creates an affine loop from the bounds known to be constants.
1907 static AffineForOp
1909  int64_t ub, int64_t step,
1910  AffineForOp::BodyBuilderFn bodyBuilderFn) {
1911  return builder.create<AffineForOp>(loc, lb, ub, step, /*iterArgs=*/llvm::None,
1912  bodyBuilderFn);
1913 }
1914 
1915 /// Creates an affine loop from the bounds that may or may not be constants.
1916 static AffineForOp
1918  int64_t step,
1919  AffineForOp::BodyBuilderFn bodyBuilderFn) {
1920  auto lbConst = lb.getDefiningOp<arith::ConstantIndexOp>();
1921  auto ubConst = ub.getDefiningOp<arith::ConstantIndexOp>();
1922  if (lbConst && ubConst)
1923  return buildAffineLoopFromConstants(builder, loc, lbConst.value(),
1924  ubConst.value(), step, bodyBuilderFn);
1925  return builder.create<AffineForOp>(loc, lb, builder.getDimIdentityMap(), ub,
1926  builder.getDimIdentityMap(), step,
1927  /*iterArgs=*/llvm::None, bodyBuilderFn);
1928 }
1929 
1931  OpBuilder &builder, Location loc, ArrayRef<int64_t> lbs,
1933  function_ref<void(OpBuilder &, Location, ValueRange)> bodyBuilderFn) {
1934  buildAffineLoopNestImpl(builder, loc, lbs, ubs, steps, bodyBuilderFn,
1936 }
1937 
1939  OpBuilder &builder, Location loc, ValueRange lbs, ValueRange ubs,
1940  ArrayRef<int64_t> steps,
1941  function_ref<void(OpBuilder &, Location, ValueRange)> bodyBuilderFn) {
1942  buildAffineLoopNestImpl(builder, loc, lbs, ubs, steps, bodyBuilderFn,
1944 }
1945 
1946 AffineForOp mlir::replaceForOpWithNewYields(OpBuilder &b, AffineForOp loop,
1947  ValueRange newIterOperands,
1948  ValueRange newYieldedValues,
1949  ValueRange newIterArgs,
1950  bool replaceLoopResults) {
1951  assert(newIterOperands.size() == newYieldedValues.size() &&
1952  "newIterOperands must be of the same size as newYieldedValues");
1953  // Create a new loop before the existing one, with the extra operands.
1955  b.setInsertionPoint(loop);
1956  auto operands = llvm::to_vector<4>(loop.getIterOperands());
1957  operands.append(newIterOperands.begin(), newIterOperands.end());
1958  SmallVector<Value, 4> lbOperands(loop.getLowerBoundOperands());
1959  SmallVector<Value, 4> ubOperands(loop.getUpperBoundOperands());
1960  SmallVector<Value, 4> steps(loop.getStep());
1961  auto lbMap = loop.getLowerBoundMap();
1962  auto ubMap = loop.getUpperBoundMap();
1963  AffineForOp newLoop =
1964  b.create<AffineForOp>(loop.getLoc(), lbOperands, lbMap, ubOperands, ubMap,
1965  loop.getStep(), operands);
1966  // Take the body of the original parent loop.
1967  newLoop.getLoopBody().takeBody(loop.getLoopBody());
1968  for (Value val : newIterArgs)
1969  newLoop.getLoopBody().addArgument(val.getType(), val.getLoc());
1970 
1971  // Update yield operation with new values to be added.
1972  if (!newYieldedValues.empty()) {
1973  auto yield = cast<AffineYieldOp>(newLoop.getBody()->getTerminator());
1974  b.setInsertionPoint(yield);
1975  auto yieldOperands = llvm::to_vector<4>(yield.getOperands());
1976  yieldOperands.append(newYieldedValues.begin(), newYieldedValues.end());
1977  b.create<AffineYieldOp>(yield.getLoc(), yieldOperands);
1978  yield.erase();
1979  }
1980  if (replaceLoopResults) {
1981  for (auto it : llvm::zip(loop.getResults(), newLoop.getResults().take_front(
1982  loop.getNumResults()))) {
1983  std::get<0>(it).replaceAllUsesWith(std::get<1>(it));
1984  }
1985  }
1986  return newLoop;
1987 }
1988 
1989 //===----------------------------------------------------------------------===//
1990 // AffineIfOp
1991 //===----------------------------------------------------------------------===//
1992 
1993 namespace {
1994 /// Remove else blocks that have nothing other than a zero value yield.
1995 struct SimplifyDeadElse : public OpRewritePattern<AffineIfOp> {
1997 
1998  LogicalResult matchAndRewrite(AffineIfOp ifOp,
1999  PatternRewriter &rewriter) const override {
2000  if (ifOp.elseRegion().empty() ||
2001  !llvm::hasSingleElement(*ifOp.getElseBlock()) || ifOp.getNumResults())
2002  return failure();
2003 
2004  rewriter.startRootUpdate(ifOp);
2005  rewriter.eraseBlock(ifOp.getElseBlock());
2006  rewriter.finalizeRootUpdate(ifOp);
2007  return success();
2008  }
2009 };
2010 
2011 /// Removes affine.if cond if the condition is always true or false in certain
2012 /// trivial cases. Promotes the then/else block in the parent operation block.
2013 struct AlwaysTrueOrFalseIf : public OpRewritePattern<AffineIfOp> {
2015 
2016  LogicalResult matchAndRewrite(AffineIfOp op,
2017  PatternRewriter &rewriter) const override {
2018 
2019  auto isTriviallyFalse = [](IntegerSet iSet) {
2020  return iSet.isEmptyIntegerSet();
2021  };
2022 
2023  auto isTriviallyTrue = [](IntegerSet iSet) {
2024  return (iSet.getNumEqualities() == 1 && iSet.getNumInequalities() == 0 &&
2025  iSet.getConstraint(0) == 0);
2026  };
2027 
2028  IntegerSet affineIfConditions = op.getIntegerSet();
2029  Block *blockToMove;
2030  if (isTriviallyFalse(affineIfConditions)) {
2031  // The absence, or equivalently, the emptiness of the else region need not
2032  // be checked when affine.if is returning results because if an affine.if
2033  // operation is returning results, it always has a non-empty else region.
2034  if (op.getNumResults() == 0 && !op.hasElse()) {
2035  // If the else region is absent, or equivalently, empty, remove the
2036  // affine.if operation (which is not returning any results).
2037  rewriter.eraseOp(op);
2038  return success();
2039  }
2040  blockToMove = op.getElseBlock();
2041  } else if (isTriviallyTrue(affineIfConditions)) {
2042  blockToMove = op.getThenBlock();
2043  } else {
2044  return failure();
2045  }
2046  Operation *blockToMoveTerminator = blockToMove->getTerminator();
2047  // Promote the "blockToMove" block to the parent operation block between the
2048  // prologue and epilogue of "op".
2049  rewriter.mergeBlockBefore(blockToMove, op);
2050  // Replace the "op" operation with the operands of the
2051  // "blockToMoveTerminator" operation. Note that "blockToMoveTerminator" is
2052  // the affine.yield operation present in the "blockToMove" block. It has no
2053  // operands when affine.if is not returning results and therefore, in that
2054  // case, replaceOp just erases "op". When affine.if is not returning
2055  // results, the affine.yield operation can be omitted. It gets inserted
2056  // implicitly.
2057  rewriter.replaceOp(op, blockToMoveTerminator->getOperands());
2058  // Erase the "blockToMoveTerminator" operation since it is now in the parent
2059  // operation block, which already has its own terminator.
2060  rewriter.eraseOp(blockToMoveTerminator);
2061  return success();
2062  }
2063 };
2064 } // namespace
2065 
2066 static LogicalResult verify(AffineIfOp op) {
2067  // Verify that we have a condition attribute.
2068  auto conditionAttr =
2069  op->getAttrOfType<IntegerSetAttr>(op.getConditionAttrName());
2070  if (!conditionAttr)
2071  return op.emitOpError(
2072  "requires an integer set attribute named 'condition'");
2073 
2074  // Verify that there are enough operands for the condition.
2075  IntegerSet condition = conditionAttr.getValue();
2076  if (op.getNumOperands() != condition.getNumInputs())
2077  return op.emitOpError(
2078  "operand count and condition integer set dimension and "
2079  "symbol count must match");
2080 
2081  // Verify that the operands are valid dimension/symbols.
2082  if (failed(verifyDimAndSymbolIdentifiers(op, op.getOperands(),
2083  condition.getNumDims())))
2084  return failure();
2085 
2086  return success();
2087 }
2088 
2090  OperationState &result) {
2091  // Parse the condition attribute set.
2092  IntegerSetAttr conditionAttr;
2093  unsigned numDims;
2094  if (parser.parseAttribute(conditionAttr, AffineIfOp::getConditionAttrName(),
2095  result.attributes) ||
2096  parseDimAndSymbolList(parser, result.operands, numDims))
2097  return failure();
2098 
2099  // Verify the condition operands.
2100  auto set = conditionAttr.getValue();
2101  if (set.getNumDims() != numDims)
2102  return parser.emitError(
2103  parser.getNameLoc(),
2104  "dim operand count and integer set dim count must match");
2105  if (numDims + set.getNumSymbols() != result.operands.size())
2106  return parser.emitError(
2107  parser.getNameLoc(),
2108  "symbol operand count and integer set symbol count must match");
2109 
2110  if (parser.parseOptionalArrowTypeList(result.types))
2111  return failure();
2112 
2113  // Create the regions for 'then' and 'else'. The latter must be created even
2114  // if it remains empty for the validity of the operation.
2115  result.regions.reserve(2);
2116  Region *thenRegion = result.addRegion();
2117  Region *elseRegion = result.addRegion();
2118 
2119  // Parse the 'then' region.
2120  if (parser.parseRegion(*thenRegion, {}, {}))
2121  return failure();
2122  AffineIfOp::ensureTerminator(*thenRegion, parser.getBuilder(),
2123  result.location);
2124 
2125  // If we find an 'else' keyword then parse the 'else' region.
2126  if (!parser.parseOptionalKeyword("else")) {
2127  if (parser.parseRegion(*elseRegion, {}, {}))
2128  return failure();
2129  AffineIfOp::ensureTerminator(*elseRegion, parser.getBuilder(),
2130  result.location);
2131  }
2132 
2133  // Parse the optional attribute list.
2134  if (parser.parseOptionalAttrDict(result.attributes))
2135  return failure();
2136 
2137  return success();
2138 }
2139 
2140 static void print(OpAsmPrinter &p, AffineIfOp op) {
2141  auto conditionAttr =
2142  op->getAttrOfType<IntegerSetAttr>(op.getConditionAttrName());
2143  p << " " << conditionAttr;
2144  printDimAndSymbolList(op.operand_begin(), op.operand_end(),
2145  conditionAttr.getValue().getNumDims(), p);
2146  p.printOptionalArrowTypeList(op.getResultTypes());
2147  p << ' ';
2148  p.printRegion(op.thenRegion(),
2149  /*printEntryBlockArgs=*/false,
2150  /*printBlockTerminators=*/op.getNumResults());
2151 
2152  // Print the 'else' regions if it has any blocks.
2153  auto &elseRegion = op.elseRegion();
2154  if (!elseRegion.empty()) {
2155  p << " else ";
2156  p.printRegion(elseRegion,
2157  /*printEntryBlockArgs=*/false,
2158  /*printBlockTerminators=*/op.getNumResults());
2159  }
2160 
2161  // Print the attribute list.
2162  p.printOptionalAttrDict(op->getAttrs(),
2163  /*elidedAttrs=*/op.getConditionAttrName());
2164 }
2165 
2166 IntegerSet AffineIfOp::getIntegerSet() {
2167  return (*this)
2168  ->getAttrOfType<IntegerSetAttr>(getConditionAttrName())
2169  .getValue();
2170 }
2171 
2172 void AffineIfOp::setIntegerSet(IntegerSet newSet) {
2173  (*this)->setAttr(getConditionAttrName(), IntegerSetAttr::get(newSet));
2174 }
2175 
2176 void AffineIfOp::setConditional(IntegerSet set, ValueRange operands) {
2177  setIntegerSet(set);
2178  (*this)->setOperands(operands);
2179 }
2180 
2181 void AffineIfOp::build(OpBuilder &builder, OperationState &result,
2182  TypeRange resultTypes, IntegerSet set, ValueRange args,
2183  bool withElseRegion) {
2184  assert(resultTypes.empty() || withElseRegion);
2185  result.addTypes(resultTypes);
2186  result.addOperands(args);
2187  result.addAttribute(getConditionAttrName(), IntegerSetAttr::get(set));
2188 
2189  Region *thenRegion = result.addRegion();
2190  thenRegion->push_back(new Block());
2191  if (resultTypes.empty())
2192  AffineIfOp::ensureTerminator(*thenRegion, builder, result.location);
2193 
2194  Region *elseRegion = result.addRegion();
2195  if (withElseRegion) {
2196  elseRegion->push_back(new Block());
2197  if (resultTypes.empty())
2198  AffineIfOp::ensureTerminator(*elseRegion, builder, result.location);
2199  }
2200 }
2201 
2202 void AffineIfOp::build(OpBuilder &builder, OperationState &result,
2203  IntegerSet set, ValueRange args, bool withElseRegion) {
2204  AffineIfOp::build(builder, result, /*resultTypes=*/{}, set, args,
2205  withElseRegion);
2206 }
2207 
2208 /// Canonicalize an affine if op's conditional (integer set + operands).
2209 LogicalResult AffineIfOp::fold(ArrayRef<Attribute>,
2211  auto set = getIntegerSet();
2212  SmallVector<Value, 4> operands(getOperands());
2213  canonicalizeSetAndOperands(&set, &operands);
2214 
2215  // Any canonicalization change always leads to either a reduction in the
2216  // number of operands or a change in the number of symbolic operands
2217  // (promotion of dims to symbols).
2218  if (operands.size() < getIntegerSet().getNumInputs() ||
2219  set.getNumSymbols() > getIntegerSet().getNumSymbols()) {
2220  setConditional(set, operands);
2221  return success();
2222  }
2223 
2224  return failure();
2225 }
2226 
2227 void AffineIfOp::getCanonicalizationPatterns(RewritePatternSet &results,
2228  MLIRContext *context) {
2229  results.add<SimplifyDeadElse, AlwaysTrueOrFalseIf>(context);
2230 }
2231 
2232 //===----------------------------------------------------------------------===//
2233 // AffineLoadOp
2234 //===----------------------------------------------------------------------===//
2235 
2236 void AffineLoadOp::build(OpBuilder &builder, OperationState &result,
2237  AffineMap map, ValueRange operands) {
2238  assert(operands.size() == 1 + map.getNumInputs() && "inconsistent operands");
2239  result.addOperands(operands);
2240  if (map)
2241  result.addAttribute(getMapAttrName(), AffineMapAttr::get(map));
2242  auto memrefType = operands[0].getType().cast<MemRefType>();
2243  result.types.push_back(memrefType.getElementType());
2244 }
2245 
2246 void AffineLoadOp::build(OpBuilder &builder, OperationState &result,
2247  Value memref, AffineMap map, ValueRange mapOperands) {
2248  assert(map.getNumInputs() == mapOperands.size() && "inconsistent index info");
2249  result.addOperands(memref);
2250  result.addOperands(mapOperands);
2251  auto memrefType = memref.getType().cast<MemRefType>();
2252  result.addAttribute(getMapAttrName(), AffineMapAttr::get(map));
2253  result.types.push_back(memrefType.getElementType());
2254 }
2255 
2256 void AffineLoadOp::build(OpBuilder &builder, OperationState &result,
2257  Value memref, ValueRange indices) {
2258  auto memrefType = memref.getType().cast<MemRefType>();
2259  int64_t rank = memrefType.getRank();
2260  // Create identity map for memrefs with at least one dimension or () -> ()
2261  // for zero-dimensional memrefs.
2262  auto map =
2263  rank ? builder.getMultiDimIdentityMap(rank) : builder.getEmptyAffineMap();
2264  build(builder, result, memref, map, indices);
2265 }
2266 
2268  OperationState &result) {
2269  auto &builder = parser.getBuilder();
2270  auto indexTy = builder.getIndexType();
2271 
2272  MemRefType type;
2273  OpAsmParser::OperandType memrefInfo;
2274  AffineMapAttr mapAttr;
2276  return failure(
2277  parser.parseOperand(memrefInfo) ||
2278  parser.parseAffineMapOfSSAIds(mapOperands, mapAttr,
2279  AffineLoadOp::getMapAttrName(),
2280  result.attributes) ||
2281  parser.parseOptionalAttrDict(result.attributes) ||
2282  parser.parseColonType(type) ||
2283  parser.resolveOperand(memrefInfo, type, result.operands) ||
2284  parser.resolveOperands(mapOperands, indexTy, result.operands) ||
2285  parser.addTypeToList(type.getElementType(), result.types));
2286 }
2287 
2288 static void print(OpAsmPrinter &p, AffineLoadOp op) {
2289  p << " " << op.getMemRef() << '[';
2290  if (AffineMapAttr mapAttr =
2291  op->getAttrOfType<AffineMapAttr>(op.getMapAttrName()))
2292  p.printAffineMapOfSSAIds(mapAttr, op.getMapOperands());
2293  p << ']';
2294  p.printOptionalAttrDict(op->getAttrs(),
2295  /*elidedAttrs=*/{op.getMapAttrName()});
2296  p << " : " << op.getMemRefType();
2297 }
2298 
2299 /// Verify common indexing invariants of affine.load, affine.store,
2300 /// affine.vector_load and affine.vector_store.
2301 static LogicalResult
2302 verifyMemoryOpIndexing(Operation *op, AffineMapAttr mapAttr,
2303  Operation::operand_range mapOperands,
2304  MemRefType memrefType, unsigned numIndexOperands) {
2305  if (mapAttr) {
2306  AffineMap map = mapAttr.getValue();
2307  if (map.getNumResults() != memrefType.getRank())
2308  return op->emitOpError("affine map num results must equal memref rank");
2309  if (map.getNumInputs() != numIndexOperands)
2310  return op->emitOpError("expects as many subscripts as affine map inputs");
2311  } else {
2312  if (memrefType.getRank() != numIndexOperands)
2313  return op->emitOpError(
2314  "expects the number of subscripts to be equal to memref rank");
2315  }
2316 
2317  Region *scope = getAffineScope(op);
2318  for (auto idx : mapOperands) {
2319  if (!idx.getType().isIndex())
2320  return op->emitOpError("index to load must have 'index' type");
2321  if (!isValidAffineIndexOperand(idx, scope))
2322  return op->emitOpError("index must be a dimension or symbol identifier");
2323  }
2324 
2325  return success();
2326 }
2327 
2328 LogicalResult verify(AffineLoadOp op) {
2329  auto memrefType = op.getMemRefType();
2330  if (op.getType() != memrefType.getElementType())
2331  return op.emitOpError("result type must match element type of memref");
2332 
2334  op.getOperation(),
2335  op->getAttrOfType<AffineMapAttr>(op.getMapAttrName()),
2336  op.getMapOperands(), memrefType,
2337  /*numIndexOperands=*/op.getNumOperands() - 1)))
2338  return failure();
2339 
2340  return success();
2341 }
2342 
2343 void AffineLoadOp::getCanonicalizationPatterns(RewritePatternSet &results,
2344  MLIRContext *context) {
2345  results.add<SimplifyAffineOp<AffineLoadOp>>(context);
2346 }
2347 
2348 OpFoldResult AffineLoadOp::fold(ArrayRef<Attribute> cstOperands) {
2349  /// load(memrefcast) -> load
2350  if (succeeded(foldMemRefCast(*this)))
2351  return getResult();
2352  return OpFoldResult();
2353 }
2354 
2355 //===----------------------------------------------------------------------===//
2356 // AffineStoreOp
2357 //===----------------------------------------------------------------------===//
2358 
2359 void AffineStoreOp::build(OpBuilder &builder, OperationState &result,
2360  Value valueToStore, Value memref, AffineMap map,
2361  ValueRange mapOperands) {
2362  assert(map.getNumInputs() == mapOperands.size() && "inconsistent index info");
2363  result.addOperands(valueToStore);
2364  result.addOperands(memref);
2365  result.addOperands(mapOperands);
2366  result.addAttribute(getMapAttrName(), AffineMapAttr::get(map));
2367 }
2368 
2369 // Use identity map.
2370 void AffineStoreOp::build(OpBuilder &builder, OperationState &result,
2371  Value valueToStore, Value memref,
2372  ValueRange indices) {
2373  auto memrefType = memref.getType().cast<MemRefType>();
2374  int64_t rank = memrefType.getRank();
2375  // Create identity map for memrefs with at least one dimension or () -> ()
2376  // for zero-dimensional memrefs.
2377  auto map =
2378  rank ? builder.getMultiDimIdentityMap(rank) : builder.getEmptyAffineMap();
2379  build(builder, result, valueToStore, memref, map, indices);
2380 }
2381 
2383  OperationState &result) {
2384  auto indexTy = parser.getBuilder().getIndexType();
2385 
2386  MemRefType type;
2387  OpAsmParser::OperandType storeValueInfo;
2388  OpAsmParser::OperandType memrefInfo;
2389  AffineMapAttr mapAttr;
2391  return failure(parser.parseOperand(storeValueInfo) || parser.parseComma() ||
2392  parser.parseOperand(memrefInfo) ||
2393  parser.parseAffineMapOfSSAIds(mapOperands, mapAttr,
2394  AffineStoreOp::getMapAttrName(),
2395  result.attributes) ||
2396  parser.parseOptionalAttrDict(result.attributes) ||
2397  parser.parseColonType(type) ||
2398  parser.resolveOperand(storeValueInfo, type.getElementType(),
2399  result.operands) ||
2400  parser.resolveOperand(memrefInfo, type, result.operands) ||
2401  parser.resolveOperands(mapOperands, indexTy, result.operands));
2402 }
2403 
2404 static void print(OpAsmPrinter &p, AffineStoreOp op) {
2405  p << " " << op.getValueToStore();
2406  p << ", " << op.getMemRef() << '[';
2407  if (AffineMapAttr mapAttr =
2408  op->getAttrOfType<AffineMapAttr>(op.getMapAttrName()))
2409  p.printAffineMapOfSSAIds(mapAttr, op.getMapOperands());
2410  p << ']';
2411  p.printOptionalAttrDict(op->getAttrs(),
2412  /*elidedAttrs=*/{op.getMapAttrName()});
2413  p << " : " << op.getMemRefType();
2414 }
2415 
2416 LogicalResult verify(AffineStoreOp op) {
2417  // The value to store must have the same type as memref element type.
2418  auto memrefType = op.getMemRefType();
2419  if (op.getValueToStore().getType() != memrefType.getElementType())
2420  return op.emitOpError(
2421  "value to store must have the same type as memref element type");
2422 
2424  op.getOperation(),
2425  op->getAttrOfType<AffineMapAttr>(op.getMapAttrName()),
2426  op.getMapOperands(), memrefType,
2427  /*numIndexOperands=*/op.getNumOperands() - 2)))
2428  return failure();
2429 
2430  return success();
2431 }
2432 
2433 void AffineStoreOp::getCanonicalizationPatterns(RewritePatternSet &results,
2434  MLIRContext *context) {
2435  results.add<SimplifyAffineOp<AffineStoreOp>>(context);
2436 }
2437 
2438 LogicalResult AffineStoreOp::fold(ArrayRef<Attribute> cstOperands,
2439  SmallVectorImpl<OpFoldResult> &results) {
2440  /// store(memrefcast) -> store
2441  return foldMemRefCast(*this, getValueToStore());
2442 }
2443 
2444 //===----------------------------------------------------------------------===//
2445 // AffineMinMaxOpBase
2446 //===----------------------------------------------------------------------===//
2447 
2448 template <typename T> static LogicalResult verifyAffineMinMaxOp(T op) {
2449  // Verify that operand count matches affine map dimension and symbol count.
2450  if (op.getNumOperands() != op.map().getNumDims() + op.map().getNumSymbols())
2451  return op.emitOpError(
2452  "operand count and affine map dimension and symbol count must match");
2453  return success();
2454 }
2455 
2456 template <typename T> static void printAffineMinMaxOp(OpAsmPrinter &p, T op) {
2457  p << ' ' << op->getAttr(T::getMapAttrName());
2458  auto operands = op.getOperands();
2459  unsigned numDims = op.map().getNumDims();
2460  p << '(' << operands.take_front(numDims) << ')';
2461 
2462  if (operands.size() != numDims)
2463  p << '[' << operands.drop_front(numDims) << ']';
2464  p.printOptionalAttrDict(op->getAttrs(),
2465  /*elidedAttrs=*/{T::getMapAttrName()});
2466 }
2467 
2468 template <typename T>
2470  OperationState &result) {
2471  auto &builder = parser.getBuilder();
2472  auto indexType = builder.getIndexType();
2475  AffineMapAttr mapAttr;
2476  return failure(
2477  parser.parseAttribute(mapAttr, T::getMapAttrName(), result.attributes) ||
2478  parser.parseOperandList(dimInfos, OpAsmParser::Delimiter::Paren) ||
2479  parser.parseOperandList(symInfos,
2481  parser.parseOptionalAttrDict(result.attributes) ||
2482  parser.resolveOperands(dimInfos, indexType, result.operands) ||
2483  parser.resolveOperands(symInfos, indexType, result.operands) ||
2484  parser.addTypeToList(indexType, result.types));
2485 }
2486 
2487 /// Fold an affine min or max operation with the given operands. The operand
2488 /// list may contain nulls, which are interpreted as the operand not being a
2489 /// constant.
2490 template <typename T>
2493  "expected affine min or max op");
2494 
2495  // Fold the affine map.
2496  // TODO: Fold more cases:
2497  // min(some_affine, some_affine + constant, ...), etc.
2498  SmallVector<int64_t, 2> results;
2499  auto foldedMap = op.map().partialConstantFold(operands, &results);
2500 
2501  // If some of the map results are not constant, try changing the map in-place.
2502  if (results.empty()) {
2503  // If the map is the same, report that folding did not happen.
2504  if (foldedMap == op.map())
2505  return {};
2506  op->setAttr("map", AffineMapAttr::get(foldedMap));
2507  return op.getResult();
2508  }
2509 
2510  // Otherwise, completely fold the op into a constant.
2511  auto resultIt = std::is_same<T, AffineMinOp>::value
2512  ? std::min_element(results.begin(), results.end())
2513  : std::max_element(results.begin(), results.end());
2514  if (resultIt == results.end())
2515  return {};
2516  return IntegerAttr::get(IndexType::get(op.getContext()), *resultIt);
2517 }
2518 
2519 /// Remove duplicated expressions in affine min/max ops.
2520 template <typename T>
2523 
2525  PatternRewriter &rewriter) const override {
2526  AffineMap oldMap = affineOp.getAffineMap();
2527 
2528  SmallVector<AffineExpr, 4> newExprs;
2529  for (AffineExpr expr : oldMap.getResults()) {
2530  // This is a linear scan over newExprs, but it should be fine given that
2531  // we typically just have a few expressions per op.
2532  if (!llvm::is_contained(newExprs, expr))
2533  newExprs.push_back(expr);
2534  }
2535 
2536  if (newExprs.size() == oldMap.getNumResults())
2537  return failure();
2538 
2539  auto newMap = AffineMap::get(oldMap.getNumDims(), oldMap.getNumSymbols(),
2540  newExprs, rewriter.getContext());
2541  rewriter.replaceOpWithNewOp<T>(affineOp, newMap, affineOp.getMapOperands());
2542 
2543  return success();
2544  }
2545 };
2546 
2547 /// Merge an affine min/max op to its consumers if its consumer is also an
2548 /// affine min/max op.
2549 ///
2550 /// This pattern requires the producer affine min/max op is bound to a
2551 /// dimension/symbol that is used as a standalone expression in the consumer
2552 /// affine op's map.
2553 ///
2554 /// For example, a pattern like the following:
2555 ///
2556 /// %0 = affine.min affine_map<()[s0] -> (s0 + 16, s0 * 8)> ()[%sym1]
2557 /// %1 = affine.min affine_map<(d0)[s0] -> (s0 + 4, d0)> (%0)[%sym2]
2558 ///
2559 /// Can be turned into:
2560 ///
2561 /// %1 = affine.min affine_map<
2562 /// ()[s0, s1] -> (s0 + 4, s1 + 16, s1 * 8)> ()[%sym2, %sym1]
2563 template <typename T> struct MergeAffineMinMaxOp : public OpRewritePattern<T> {
2565 
2567  PatternRewriter &rewriter) const override {
2568  AffineMap oldMap = affineOp.getAffineMap();
2569  ValueRange dimOperands =
2570  affineOp.getMapOperands().take_front(oldMap.getNumDims());
2571  ValueRange symOperands =
2572  affineOp.getMapOperands().take_back(oldMap.getNumSymbols());
2573 
2574  auto newDimOperands = llvm::to_vector<8>(dimOperands);
2575  auto newSymOperands = llvm::to_vector<8>(symOperands);
2576  SmallVector<AffineExpr, 4> newExprs;
2577  SmallVector<T, 4> producerOps;
2578 
2579  // Go over each expression to see whether it's a single dimension/symbol
2580  // with the corresponding operand which is the result of another affine
2581  // min/max op. If So it can be merged into this affine op.
2582  for (AffineExpr expr : oldMap.getResults()) {
2583  if (auto symExpr = expr.dyn_cast<AffineSymbolExpr>()) {
2584  Value symValue = symOperands[symExpr.getPosition()];
2585  if (auto producerOp = symValue.getDefiningOp<T>()) {
2586  producerOps.push_back(producerOp);
2587  continue;
2588  }
2589  } else if (auto dimExpr = expr.dyn_cast<AffineDimExpr>()) {
2590  Value dimValue = dimOperands[dimExpr.getPosition()];
2591  if (auto producerOp = dimValue.getDefiningOp<T>()) {
2592  producerOps.push_back(producerOp);
2593  continue;
2594  }
2595  }
2596  // For the above cases we will remove the expression by merging the
2597  // producer affine min/max's affine expressions. Otherwise we need to
2598  // keep the existing expression.
2599  newExprs.push_back(expr);
2600  }
2601 
2602  if (producerOps.empty())
2603  return failure();
2604 
2605  unsigned numUsedDims = oldMap.getNumDims();
2606  unsigned numUsedSyms = oldMap.getNumSymbols();
2607 
2608  // Now go over all producer affine ops and merge their expressions.
2609  for (T producerOp : producerOps) {
2610  AffineMap producerMap = producerOp.getAffineMap();
2611  unsigned numProducerDims = producerMap.getNumDims();
2612  unsigned numProducerSyms = producerMap.getNumSymbols();
2613 
2614  // Collect all dimension/symbol values.
2615  ValueRange dimValues =
2616  producerOp.getMapOperands().take_front(numProducerDims);
2617  ValueRange symValues =
2618  producerOp.getMapOperands().take_back(numProducerSyms);
2619  newDimOperands.append(dimValues.begin(), dimValues.end());
2620  newSymOperands.append(symValues.begin(), symValues.end());
2621 
2622  // For expressions we need to shift to avoid overlap.
2623  for (AffineExpr expr : producerMap.getResults()) {
2624  newExprs.push_back(expr.shiftDims(numProducerDims, numUsedDims)
2625  .shiftSymbols(numProducerSyms, numUsedSyms));
2626  }
2627 
2628  numUsedDims += numProducerDims;
2629  numUsedSyms += numProducerSyms;
2630  }
2631 
2632  auto newMap = AffineMap::get(numUsedDims, numUsedSyms, newExprs,
2633  rewriter.getContext());
2634  auto newOperands =
2635  llvm::to_vector<8>(llvm::concat<Value>(newDimOperands, newSymOperands));
2636  rewriter.replaceOpWithNewOp<T>(affineOp, newMap, newOperands);
2637 
2638  return success();
2639  }
2640 };
2641 
2642 template <typename T>
2645 
2647  PatternRewriter &rewriter) const override {
2648  if (affineOp.map().getNumResults() != 1)
2649  return failure();
2650  rewriter.replaceOpWithNewOp<AffineApplyOp>(affineOp, affineOp.map(),
2651  affineOp.getOperands());
2652  return success();
2653  }
2654 };
2655 
2656 //===----------------------------------------------------------------------===//
2657 // AffineMinOp
2658 //===----------------------------------------------------------------------===//
2659 //
2660 // %0 = affine.min (d0) -> (1000, d0 + 512) (%i0)
2661 //
2662 
2663 OpFoldResult AffineMinOp::fold(ArrayRef<Attribute> operands) {
2664  return foldMinMaxOp(*this, operands);
2665 }
2666 
2667 void AffineMinOp::getCanonicalizationPatterns(RewritePatternSet &patterns,
2668  MLIRContext *context) {
2671  MergeAffineMinMaxOp<AffineMinOp>, SimplifyAffineOp<AffineMinOp>>(
2672  context);
2673 }
2674 
2675 //===----------------------------------------------------------------------===//
2676 // AffineMaxOp
2677 //===----------------------------------------------------------------------===//
2678 //
2679 // %0 = affine.max (d0) -> (1000, d0 + 512) (%i0)
2680 //
2681 
2682 OpFoldResult AffineMaxOp::fold(ArrayRef<Attribute> operands) {
2683  return foldMinMaxOp(*this, operands);
2684 }
2685 
2686 void AffineMaxOp::getCanonicalizationPatterns(RewritePatternSet &patterns,
2687  MLIRContext *context) {
2690  MergeAffineMinMaxOp<AffineMaxOp>, SimplifyAffineOp<AffineMaxOp>>(
2691  context);
2692 }
2693 
2694 //===----------------------------------------------------------------------===//
2695 // AffinePrefetchOp
2696 //===----------------------------------------------------------------------===//
2697 
2698 //
2699 // affine.prefetch %0[%i, %j + 5], read, locality<3>, data : memref<400x400xi32>
2700 //
2702  OperationState &result) {
2703  auto &builder = parser.getBuilder();
2704  auto indexTy = builder.getIndexType();
2705 
2706  MemRefType type;
2707  OpAsmParser::OperandType memrefInfo;
2708  IntegerAttr hintInfo;
2709  auto i32Type = parser.getBuilder().getIntegerType(32);
2710  StringRef readOrWrite, cacheType;
2711 
2712  AffineMapAttr mapAttr;
2714  if (parser.parseOperand(memrefInfo) ||
2715  parser.parseAffineMapOfSSAIds(mapOperands, mapAttr,
2716  AffinePrefetchOp::getMapAttrName(),
2717  result.attributes) ||
2718  parser.parseComma() || parser.parseKeyword(&readOrWrite) ||
2719  parser.parseComma() || parser.parseKeyword("locality") ||
2720  parser.parseLess() ||
2721  parser.parseAttribute(hintInfo, i32Type,
2722  AffinePrefetchOp::getLocalityHintAttrName(),
2723  result.attributes) ||
2724  parser.parseGreater() || parser.parseComma() ||
2725  parser.parseKeyword(&cacheType) ||
2726  parser.parseOptionalAttrDict(result.attributes) ||
2727  parser.parseColonType(type) ||
2728  parser.resolveOperand(memrefInfo, type, result.operands) ||
2729  parser.resolveOperands(mapOperands, indexTy, result.operands))
2730  return failure();
2731 
2732  if (!readOrWrite.equals("read") && !readOrWrite.equals("write"))
2733  return parser.emitError(parser.getNameLoc(),
2734  "rw specifier has to be 'read' or 'write'");
2735  result.addAttribute(
2736  AffinePrefetchOp::getIsWriteAttrName(),
2737  parser.getBuilder().getBoolAttr(readOrWrite.equals("write")));
2738 
2739  if (!cacheType.equals("data") && !cacheType.equals("instr"))
2740  return parser.emitError(parser.getNameLoc(),
2741  "cache type has to be 'data' or 'instr'");
2742 
2743  result.addAttribute(
2744  AffinePrefetchOp::getIsDataCacheAttrName(),
2745  parser.getBuilder().getBoolAttr(cacheType.equals("data")));
2746 
2747  return success();
2748 }
2749 
2750 static void print(OpAsmPrinter &p, AffinePrefetchOp op) {
2751  p << " " << op.memref() << '[';
2752  AffineMapAttr mapAttr = op->getAttrOfType<AffineMapAttr>(op.getMapAttrName());
2753  if (mapAttr) {
2754  SmallVector<Value, 2> operands(op.getMapOperands());
2755  p.printAffineMapOfSSAIds(mapAttr, operands);
2756  }
2757  p << ']' << ", " << (op.isWrite() ? "write" : "read") << ", "
2758  << "locality<" << op.localityHint() << ">, "
2759  << (op.isDataCache() ? "data" : "instr");
2761  op->getAttrs(),
2762  /*elidedAttrs=*/{op.getMapAttrName(), op.getLocalityHintAttrName(),
2763  op.getIsDataCacheAttrName(), op.getIsWriteAttrName()});
2764  p << " : " << op.getMemRefType();
2765 }
2766 
2767 static LogicalResult verify(AffinePrefetchOp op) {
2768  auto mapAttr = op->getAttrOfType<AffineMapAttr>(op.getMapAttrName());
2769  if (mapAttr) {
2770  AffineMap map = mapAttr.getValue();
2771  if (map.getNumResults() != op.getMemRefType().getRank())
2772  return op.emitOpError("affine.prefetch affine map num results must equal"
2773  " memref rank");
2774  if (map.getNumInputs() + 1 != op.getNumOperands())
2775  return op.emitOpError("too few operands");
2776  } else {
2777  if (op.getNumOperands() != 1)
2778  return op.emitOpError("too few operands");
2779  }
2780 
2781  Region *scope = getAffineScope(op);
2782  for (auto idx : op.getMapOperands()) {
2783  if (!isValidAffineIndexOperand(idx, scope))
2784  return op.emitOpError("index must be a dimension or symbol identifier");
2785  }
2786  return success();
2787 }
2788 
2789 void AffinePrefetchOp::getCanonicalizationPatterns(RewritePatternSet &results,
2790  MLIRContext *context) {
2791  // prefetch(memrefcast) -> prefetch
2792  results.add<SimplifyAffineOp<AffinePrefetchOp>>(context);
2793 }
2794 
2795 LogicalResult AffinePrefetchOp::fold(ArrayRef<Attribute> cstOperands,
2796  SmallVectorImpl<OpFoldResult> &results) {
2797  /// prefetch(memrefcast) -> prefetch
2798  return foldMemRefCast(*this);
2799 }
2800 
2801 //===----------------------------------------------------------------------===//
2802 // AffineParallelOp
2803 //===----------------------------------------------------------------------===//
2804 
2805 void AffineParallelOp::build(OpBuilder &builder, OperationState &result,
2806  TypeRange resultTypes,
2807  ArrayRef<arith::AtomicRMWKind> reductions,
2808  ArrayRef<int64_t> ranges) {
2809  SmallVector<AffineMap> lbs(ranges.size(), builder.getConstantAffineMap(0));
2810  auto ubs = llvm::to_vector<4>(llvm::map_range(ranges, [&](int64_t value) {
2811  return builder.getConstantAffineMap(value);
2812  }));
2813  SmallVector<int64_t> steps(ranges.size(), 1);
2814  build(builder, result, resultTypes, reductions, lbs, /*lbArgs=*/{}, ubs,
2815  /*ubArgs=*/{}, steps);
2816 }
2817 
2818 void AffineParallelOp::build(OpBuilder &builder, OperationState &result,
2819  TypeRange resultTypes,
2820  ArrayRef<arith::AtomicRMWKind> reductions,
2821  ArrayRef<AffineMap> lbMaps, ValueRange lbArgs,
2822  ArrayRef<AffineMap> ubMaps, ValueRange ubArgs,
2823  ArrayRef<int64_t> steps) {
2824  assert(llvm::all_of(lbMaps,
2825  [lbMaps](AffineMap m) {
2826  return m.getNumDims() == lbMaps[0].getNumDims() &&
2827  m.getNumSymbols() == lbMaps[0].getNumSymbols();
2828  }) &&
2829  "expected all lower bounds maps to have the same number of dimensions "
2830  "and symbols");
2831  assert(llvm::all_of(ubMaps,
2832  [ubMaps](AffineMap m) {
2833  return m.getNumDims() == ubMaps[0].getNumDims() &&
2834  m.getNumSymbols() == ubMaps[0].getNumSymbols();
2835  }) &&
2836  "expected all upper bounds maps to have the same number of dimensions "
2837  "and symbols");
2838  assert((lbMaps.empty() || lbMaps[0].getNumInputs() == lbArgs.size()) &&
2839  "expected lower bound maps to have as many inputs as lower bound "
2840  "operands");
2841  assert((ubMaps.empty() || ubMaps[0].getNumInputs() == ubArgs.size()) &&
2842  "expected upper bound maps to have as many inputs as upper bound "
2843  "operands");
2844 
2845  result.addTypes(resultTypes);
2846 
2847  // Convert the reductions to integer attributes.
2848  SmallVector<Attribute, 4> reductionAttrs;
2849  for (arith::AtomicRMWKind reduction : reductions)
2850  reductionAttrs.push_back(
2851  builder.getI64IntegerAttr(static_cast<int64_t>(reduction)));
2852  result.addAttribute(getReductionsAttrName(),
2853  builder.getArrayAttr(reductionAttrs));
2854 
2855  // Concatenates maps defined in the same input space (same dimensions and
2856  // symbols), assumes there is at least one map.
2857  auto concatMapsSameInput = [&builder](ArrayRef<AffineMap> maps,
2858  SmallVectorImpl<int32_t> &groups) {
2859  if (maps.empty())
2860  return AffineMap::get(builder.getContext());
2862  groups.reserve(groups.size() + maps.size());
2863  exprs.reserve(maps.size());
2864  for (AffineMap m : maps) {
2865  llvm::append_range(exprs, m.getResults());
2866  groups.push_back(m.getNumResults());
2867  }
2868  return AffineMap::get(maps[0].getNumDims(), maps[0].getNumSymbols(), exprs,
2869  maps[0].getContext());
2870  };
2871 
2872  // Set up the bounds.
2873  SmallVector<int32_t> lbGroups, ubGroups;
2874  AffineMap lbMap = concatMapsSameInput(lbMaps, lbGroups);
2875  AffineMap ubMap = concatMapsSameInput(ubMaps, ubGroups);
2876  result.addAttribute(getLowerBoundsMapAttrName(), AffineMapAttr::get(lbMap));
2877  result.addAttribute(getLowerBoundsGroupsAttrName(),
2878  builder.getI32TensorAttr(lbGroups));
2879  result.addAttribute(getUpperBoundsMapAttrName(), AffineMapAttr::get(ubMap));
2880  result.addAttribute(getUpperBoundsGroupsAttrName(),
2881  builder.getI32TensorAttr(ubGroups));
2882  result.addAttribute(getStepsAttrName(), builder.getI64ArrayAttr(steps));
2883  result.addOperands(lbArgs);
2884  result.addOperands(ubArgs);
2885 
2886  // Create a region and a block for the body.
2887  auto *bodyRegion = result.addRegion();
2888  auto *body = new Block();
2889  // Add all the block arguments.
2890  for (unsigned i = 0, e = steps.size(); i < e; ++i)
2891  body->addArgument(IndexType::get(builder.getContext()), result.location);
2892  bodyRegion->push_back(body);
2893  if (resultTypes.empty())
2894  ensureTerminator(*bodyRegion, builder, result.location);
2895 }
2896 
2897 Region &AffineParallelOp::getLoopBody() { return region(); }
2898 
2899 bool AffineParallelOp::isDefinedOutsideOfLoop(Value value) {
2900  return !region().isAncestor(value.getParentRegion());
2901 }
2902 
2903 LogicalResult AffineParallelOp::moveOutOfLoop(ArrayRef<Operation *> ops) {
2904  for (Operation *op : ops)
2905  op->moveBefore(*this);
2906  return success();
2907 }
2908 
2909 unsigned AffineParallelOp::getNumDims() { return steps().size(); }
2910 
2911 AffineParallelOp::operand_range AffineParallelOp::getLowerBoundsOperands() {
2912  return getOperands().take_front(lowerBoundsMap().getNumInputs());
2913 }
2914 
2915 AffineParallelOp::operand_range AffineParallelOp::getUpperBoundsOperands() {
2916  return getOperands().drop_front(lowerBoundsMap().getNumInputs());
2917 }
2918 
2919 AffineMap AffineParallelOp::getLowerBoundMap(unsigned pos) {
2920  auto values = lowerBoundsGroups().getValues<int32_t>();
2921  unsigned start = 0;
2922  for (unsigned i = 0; i < pos; ++i)
2923  start += values[i];
2924  return lowerBoundsMap().getSliceMap(start, values[pos]);
2925 }
2926 
2927 AffineMap AffineParallelOp::getUpperBoundMap(unsigned pos) {
2928  auto values = upperBoundsGroups().getValues<int32_t>();
2929  unsigned start = 0;
2930  for (unsigned i = 0; i < pos; ++i)
2931  start += values[i];
2932  return upperBoundsMap().getSliceMap(start, values[pos]);
2933 }
2934 
2935 AffineValueMap AffineParallelOp::getLowerBoundsValueMap() {
2936  return AffineValueMap(lowerBoundsMap(), getLowerBoundsOperands());
2937 }
2938 
2939 AffineValueMap AffineParallelOp::getUpperBoundsValueMap() {
2940  return AffineValueMap(upperBoundsMap(), getUpperBoundsOperands());
2941 }
2942 
2943 Optional<SmallVector<int64_t, 8>> AffineParallelOp::getConstantRanges() {
2944  if (hasMinMaxBounds())
2945  return llvm::None;
2946 
2947  // Try to convert all the ranges to constant expressions.
2949  AffineValueMap rangesValueMap;
2950  AffineValueMap::difference(getUpperBoundsValueMap(), getLowerBoundsValueMap(),
2951  &rangesValueMap);
2952  out.reserve(rangesValueMap.getNumResults());
2953  for (unsigned i = 0, e = rangesValueMap.getNumResults(); i < e; ++i) {
2954  auto expr = rangesValueMap.getResult(i);
2955  auto cst = expr.dyn_cast<AffineConstantExpr>();
2956  if (!cst)
2957  return llvm::None;
2958  out.push_back(cst.getValue());
2959  }
2960  return out;
2961 }
2962 
2963 Block *AffineParallelOp::getBody() { return &region().front(); }
2964 
2965 OpBuilder AffineParallelOp::getBodyBuilder() {
2966  return OpBuilder(getBody(), std::prev(getBody()->end()));
2967 }
2968 
2969 void AffineParallelOp::setLowerBounds(ValueRange lbOperands, AffineMap map) {
2970  assert(lbOperands.size() == map.getNumInputs() &&
2971  "operands to map must match number of inputs");
2972 
2973  auto ubOperands = getUpperBoundsOperands();
2974 
2975  SmallVector<Value, 4> newOperands(lbOperands);
2976  newOperands.append(ubOperands.begin(), ubOperands.end());
2977  (*this)->setOperands(newOperands);
2978 
2979  lowerBoundsMapAttr(AffineMapAttr::get(map));
2980 }
2981 
2982 void AffineParallelOp::setUpperBounds(ValueRange ubOperands, AffineMap map) {
2983  assert(ubOperands.size() == map.getNumInputs() &&
2984  "operands to map must match number of inputs");
2985 
2986  SmallVector<Value, 4> newOperands(getLowerBoundsOperands());
2987  newOperands.append(ubOperands.begin(), ubOperands.end());
2988  (*this)->setOperands(newOperands);
2989 
2990  upperBoundsMapAttr(AffineMapAttr::get(map));
2991 }
2992 
2993 void AffineParallelOp::setLowerBoundsMap(AffineMap map) {
2994  AffineMap lbMap = lowerBoundsMap();
2995  assert(lbMap.getNumDims() == map.getNumDims() &&
2996  lbMap.getNumSymbols() == map.getNumSymbols());
2997  (void)lbMap;
2998  lowerBoundsMapAttr(AffineMapAttr::get(map));
2999 }
3000 
3001 void AffineParallelOp::setUpperBoundsMap(AffineMap map) {
3002  AffineMap ubMap = upperBoundsMap();
3003  assert(ubMap.getNumDims() == map.getNumDims() &&
3004  ubMap.getNumSymbols() == map.getNumSymbols());
3005  (void)ubMap;
3006  upperBoundsMapAttr(AffineMapAttr::get(map));
3007 }
3008 
3009 SmallVector<int64_t, 8> AffineParallelOp::getSteps() {
3010  SmallVector<int64_t, 8> result;
3011  for (Attribute attr : steps()) {
3012  result.push_back(attr.cast<IntegerAttr>().getInt());
3013  }
3014  return result;
3015 }
3016 
3017 void AffineParallelOp::setSteps(ArrayRef<int64_t> newSteps) {
3018  stepsAttr(getBodyBuilder().getI64ArrayAttr(newSteps));
3019 }
3020 
3021 static LogicalResult verify(AffineParallelOp op) {
3022  auto numDims = op.getNumDims();
3023  if (op.lowerBoundsGroups().getNumElements() != numDims ||
3024  op.upperBoundsGroups().getNumElements() != numDims ||
3025  op.steps().size() != numDims ||
3026  op.getBody()->getNumArguments() != numDims) {
3027  return op.emitOpError()
3028  << "the number of region arguments ("
3029  << op.getBody()->getNumArguments()
3030  << ") and the number of map groups for lower ("
3031  << op.lowerBoundsGroups().getNumElements() << ") and upper bound ("
3032  << op.upperBoundsGroups().getNumElements()
3033  << "), and the number of steps (" << op.steps().size()
3034  << ") must all match";
3035  }
3036 
3037  unsigned expectedNumLBResults = 0;
3038  for (APInt v : op.lowerBoundsGroups())
3039  expectedNumLBResults += v.getZExtValue();
3040  if (expectedNumLBResults != op.lowerBoundsMap().getNumResults())
3041  return op.emitOpError() << "expected lower bounds map to have "
3042  << expectedNumLBResults << " results";
3043  unsigned expectedNumUBResults = 0;
3044  for (APInt v : op.upperBoundsGroups())
3045  expectedNumUBResults += v.getZExtValue();
3046  if (expectedNumUBResults != op.upperBoundsMap().getNumResults())
3047  return op.emitOpError() << "expected upper bounds map to have "
3048  << expectedNumUBResults << " results";
3049 
3050  if (op.reductions().size() != op.getNumResults())
3051  return op.emitOpError("a reduction must be specified for each output");
3052 
3053  // Verify reduction ops are all valid
3054  for (Attribute attr : op.reductions()) {
3055  auto intAttr = attr.dyn_cast<IntegerAttr>();
3056  if (!intAttr || !arith::symbolizeAtomicRMWKind(intAttr.getInt()))
3057  return op.emitOpError("invalid reduction attribute");
3058  }
3059 
3060  // Verify that the bound operands are valid dimension/symbols.
3061  /// Lower bounds.
3062  if (failed(verifyDimAndSymbolIdentifiers(op, op.getLowerBoundsOperands(),
3063  op.lowerBoundsMap().getNumDims())))
3064  return failure();
3065  /// Upper bounds.
3066  if (failed(verifyDimAndSymbolIdentifiers(op, op.getUpperBoundsOperands(),
3067  op.upperBoundsMap().getNumDims())))
3068  return failure();
3069  return success();
3070 }
3071 
3073  SmallVector<Value, 4> newOperands{operands};
3074  auto newMap = getAffineMap();
3075  composeAffineMapAndOperands(&newMap, &newOperands);
3076  if (newMap == getAffineMap() && newOperands == operands)
3077  return failure();
3078  reset(newMap, newOperands);
3079  return success();
3080 }
3081 
3082 /// Canonicalize the bounds of the given loop.
3083 static LogicalResult canonicalizeLoopBounds(AffineParallelOp op) {
3084  AffineValueMap lb = op.getLowerBoundsValueMap();
3085  bool lbCanonicalized = succeeded(lb.canonicalize());
3086 
3087  AffineValueMap ub = op.getUpperBoundsValueMap();
3088  bool ubCanonicalized = succeeded(ub.canonicalize());
3089 
3090  // Any canonicalization change always leads to updated map(s).
3091  if (!lbCanonicalized && !ubCanonicalized)
3092  return failure();
3093 
3094  if (lbCanonicalized)
3095  op.setLowerBounds(lb.getOperands(), lb.getAffineMap());
3096  if (ubCanonicalized)
3097  op.setUpperBounds(ub.getOperands(), ub.getAffineMap());
3098 
3099  return success();
3100 }
3101 
3102 LogicalResult AffineParallelOp::fold(ArrayRef<Attribute> operands,
3103  SmallVectorImpl<OpFoldResult> &results) {
3104  return canonicalizeLoopBounds(*this);
3105 }
3106 
3107 /// Prints a lower(upper) bound of an affine parallel loop with max(min)
3108 /// conditions in it. `mapAttr` is a flat list of affine expressions and `group`
3109 /// identifies which of the those expressions form max/min groups. `operands`
3110 /// are the SSA values of dimensions and symbols and `keyword` is either "min"
3111 /// or "max".
3112 static void printMinMaxBound(OpAsmPrinter &p, AffineMapAttr mapAttr,
3113  DenseIntElementsAttr group, ValueRange operands,
3114  StringRef keyword) {
3115  AffineMap map = mapAttr.getValue();
3116  unsigned numDims = map.getNumDims();
3117  ValueRange dimOperands = operands.take_front(numDims);
3118  ValueRange symOperands = operands.drop_front(numDims);
3119  unsigned start = 0;
3120  for (llvm::APInt groupSize : group) {
3121  if (start != 0)
3122  p << ", ";
3123 
3124  unsigned size = groupSize.getZExtValue();
3125  if (size == 1) {
3126  p.printAffineExprOfSSAIds(map.getResult(start), dimOperands, symOperands);
3127  ++start;
3128  } else {
3129  p << keyword << '(';
3130  AffineMap submap = map.getSliceMap(start, size);
3131  p.printAffineMapOfSSAIds(AffineMapAttr::get(submap), operands);
3132  p << ')';
3133  start += size;
3134  }
3135  }
3136 }
3137 
3138 static void print(OpAsmPrinter &p, AffineParallelOp op) {
3139  p << " (" << op.getBody()->getArguments() << ") = (";
3140  printMinMaxBound(p, op.lowerBoundsMapAttr(), op.lowerBoundsGroupsAttr(),
3141  op.getLowerBoundsOperands(), "max");
3142  p << ") to (";
3143  printMinMaxBound(p, op.upperBoundsMapAttr(), op.upperBoundsGroupsAttr(),
3144  op.getUpperBoundsOperands(), "min");
3145  p << ')';
3146  SmallVector<int64_t, 8> steps = op.getSteps();
3147  bool elideSteps = llvm::all_of(steps, [](int64_t step) { return step == 1; });
3148  if (!elideSteps) {
3149  p << " step (";
3150  llvm::interleaveComma(steps, p);
3151  p << ')';
3152  }
3153  if (op.getNumResults()) {
3154  p << " reduce (";
3155  llvm::interleaveComma(op.reductions(), p, [&](auto &attr) {
3156  arith::AtomicRMWKind sym = *arith::symbolizeAtomicRMWKind(
3157  attr.template cast<IntegerAttr>().getInt());
3158  p << "\"" << arith::stringifyAtomicRMWKind(sym) << "\"";
3159  });
3160  p << ") -> (" << op.getResultTypes() << ")";
3161  }
3162 
3163  p << ' ';
3164  p.printRegion(op.region(), /*printEntryBlockArgs=*/false,
3165  /*printBlockTerminators=*/op.getNumResults());
3166  p.printOptionalAttrDict(
3167  op->getAttrs(),
3168  /*elidedAttrs=*/{AffineParallelOp::getReductionsAttrName(),
3169  AffineParallelOp::getLowerBoundsMapAttrName(),
3170  AffineParallelOp::getLowerBoundsGroupsAttrName(),
3171  AffineParallelOp::getUpperBoundsMapAttrName(),
3172  AffineParallelOp::getUpperBoundsGroupsAttrName(),
3173  AffineParallelOp::getStepsAttrName()});
3174 }
3175 
3176 /// Given a list of lists of parsed operands, populates `uniqueOperands` with
3177 /// unique operands. Also populates `replacements with affine expressions of
3178 /// `kind` that can be used to update affine maps previously accepting a
3179 /// `operands` to accept `uniqueOperands` instead.
3183  SmallVectorImpl<Value> &uniqueOperands,
3184  SmallVectorImpl<AffineExpr> &replacements, AffineExprKind kind) {
3185  assert((kind == AffineExprKind::DimId || kind == AffineExprKind::SymbolId) &&
3186  "expected operands to be dim or symbol expression");
3187 
3188  Type indexType = parser.getBuilder().getIndexType();
3189  for (const auto &list : operands) {
3190  SmallVector<Value> valueOperands;
3191  parser.resolveOperands(list, indexType, valueOperands);
3192  for (Value operand : valueOperands) {
3193  unsigned pos = std::distance(uniqueOperands.begin(),
3194  llvm::find(uniqueOperands, operand));
3195  if (pos == uniqueOperands.size())
3196  uniqueOperands.push_back(operand);
3197  replacements.push_back(
3198  kind == AffineExprKind::DimId
3199  ? getAffineDimExpr(pos, parser.getContext())
3200  : getAffineSymbolExpr(pos, parser.getContext()));
3201  }
3202  }
3203 }
3204 
3205 namespace {
3206 enum class MinMaxKind { Min, Max };
3207 } // namespace
3208 
3209 /// Parses an affine map that can contain a min/max for groups of its results,
3210 /// e.g., max(expr-1, expr-2), expr-3, max(expr-4, expr-5, expr-6). Populates
3211 /// `result` attributes with the map (flat list of expressions) and the grouping
3212 /// (list of integers that specify how many expressions to put into each
3213 /// min/max) attributes. Deduplicates repeated operands.
3214 ///
3215 /// parallel-bound ::= `(` parallel-group-list `)`
3216 /// parallel-group-list ::= parallel-group (`,` parallel-group-list)?
3217 /// parallel-group ::= simple-group | min-max-group
3218 /// simple-group ::= expr-of-ssa-ids
3219 /// min-max-group ::= ( `min` | `max` ) `(` expr-of-ssa-ids-list `)`
3220 /// expr-of-ssa-ids-list ::= expr-of-ssa-ids (`,` expr-of-ssa-id-list)?
3221 ///
3222 /// Examples:
3223 /// (%0, min(%1 + %2, %3), %4, min(%5 floordiv 32, %6))
3224 /// (%0, max(%1 - 2 * %2))
3226  OperationState &result,
3227  MinMaxKind kind) {
3228  constexpr llvm::StringLiteral tmpAttrName = "__pseudo_bound_map";
3229 
3230  StringRef mapName = kind == MinMaxKind::Min
3231  ? AffineParallelOp::getUpperBoundsMapAttrName()
3232  : AffineParallelOp::getLowerBoundsMapAttrName();
3233  StringRef groupsName = kind == MinMaxKind::Min
3234  ? AffineParallelOp::getUpperBoundsGroupsAttrName()
3235  : AffineParallelOp::getLowerBoundsGroupsAttrName();
3236 
3237  if (failed(parser.parseLParen()))
3238  return failure();
3239 
3240  if (succeeded(parser.parseOptionalRParen())) {
3241  result.addAttribute(
3242  mapName, AffineMapAttr::get(parser.getBuilder().getEmptyAffineMap()));
3243  result.addAttribute(groupsName, parser.getBuilder().getI32TensorAttr({}));
3244  return success();
3245  }
3246 
3247  SmallVector<AffineExpr> flatExprs;
3250  SmallVector<int32_t> numMapsPerGroup;
3252  do {
3253  if (succeeded(parser.parseOptionalKeyword(
3254  kind == MinMaxKind::Min ? "min" : "max"))) {
3255  mapOperands.clear();
3256  AffineMapAttr map;
3257  if (failed(parser.parseAffineMapOfSSAIds(mapOperands, map, tmpAttrName,
3258  result.attributes,
3260  return failure();
3261  result.attributes.erase(tmpAttrName);
3262  llvm::append_range(flatExprs, map.getValue().getResults());
3263  auto operandsRef = llvm::makeArrayRef(mapOperands);
3264  auto dimsRef = operandsRef.take_front(map.getValue().getNumDims());
3265  SmallVector<OpAsmParser::OperandType> dims(dimsRef.begin(),
3266  dimsRef.end());
3267  auto symsRef = operandsRef.drop_front(map.getValue().getNumDims());
3268  SmallVector<OpAsmParser::OperandType> syms(symsRef.begin(),
3269  symsRef.end());
3270  flatDimOperands.append(map.getValue().getNumResults(), dims);
3271  flatSymOperands.append(map.getValue().getNumResults(), syms);
3272  numMapsPerGroup.push_back(map.getValue().getNumResults());
3273  } else {
3274  if (failed(parser.parseAffineExprOfSSAIds(flatDimOperands.emplace_back(),
3275  flatSymOperands.emplace_back(),
3276  flatExprs.emplace_back())))
3277  return failure();
3278  numMapsPerGroup.push_back(1);
3279  }
3280  } while (succeeded(parser.parseOptionalComma()));
3281 
3282  if (failed(parser.parseRParen()))
3283  return failure();
3284 
3285  unsigned totalNumDims = 0;
3286  unsigned totalNumSyms = 0;
3287  for (unsigned i = 0, e = flatExprs.size(); i < e; ++i) {
3288  unsigned numDims = flatDimOperands[i].size();
3289  unsigned numSyms = flatSymOperands[i].size();
3290  flatExprs[i] = flatExprs[i]
3291  .shiftDims(numDims, totalNumDims)
3292  .shiftSymbols(numSyms, totalNumSyms);
3293  totalNumDims += numDims;
3294  totalNumSyms += numSyms;
3295  }
3296 
3297  // Deduplicate map operands.
3298  SmallVector<Value> dimOperands, symOperands;
3299  SmallVector<AffineExpr> dimRplacements, symRepacements;
3300  deduplicateAndResolveOperands(parser, flatDimOperands, dimOperands,
3301  dimRplacements, AffineExprKind::DimId);
3302  deduplicateAndResolveOperands(parser, flatSymOperands, symOperands,
3303  symRepacements, AffineExprKind::SymbolId);
3304 
3305  result.operands.append(dimOperands.begin(), dimOperands.end());
3306  result.operands.append(symOperands.begin(), symOperands.end());
3307 
3308  Builder &builder = parser.getBuilder();
3309  auto flatMap = AffineMap::get(totalNumDims, totalNumSyms, flatExprs,
3310  parser.getContext());
3311  flatMap = flatMap.replaceDimsAndSymbols(
3312  dimRplacements, symRepacements, dimOperands.size(), symOperands.size());
3313 
3314  result.addAttribute(mapName, AffineMapAttr::get(flatMap));
3315  result.addAttribute(groupsName, builder.getI32TensorAttr(numMapsPerGroup));
3316  return success();
3317 }
3318 
3319 //
3320 // operation ::= `affine.parallel` `(` ssa-ids `)` `=` parallel-bound
3321 // `to` parallel-bound steps? region attr-dict?
3322 // steps ::= `steps` `(` integer-literals `)`
3323 //
3325  OperationState &result) {
3326  auto &builder = parser.getBuilder();
3327  auto indexType = builder.getIndexType();
3329  if (parser.parseRegionArgumentList(ivs, /*requiredOperandCount=*/-1,
3331  parser.parseEqual() ||
3332  parseAffineMapWithMinMax(parser, result, MinMaxKind::Max) ||
3333  parser.parseKeyword("to") ||
3334  parseAffineMapWithMinMax(parser, result, MinMaxKind::Min))
3335  return failure();
3336 
3337  AffineMapAttr stepsMapAttr;
3338  NamedAttrList stepsAttrs;
3340  if (failed(parser.parseOptionalKeyword("step"))) {
3341  SmallVector<int64_t, 4> steps(ivs.size(), 1);
3342  result.addAttribute(AffineParallelOp::getStepsAttrName(),
3343  builder.getI64ArrayAttr(steps));
3344  } else {
3345  if (parser.parseAffineMapOfSSAIds(stepsMapOperands, stepsMapAttr,
3346  AffineParallelOp::getStepsAttrName(),
3347  stepsAttrs,
3349  return failure();
3350 
3351  // Convert steps from an AffineMap into an I64ArrayAttr.
3353  auto stepsMap = stepsMapAttr.getValue();
3354  for (const auto &result : stepsMap.getResults()) {
3355  auto constExpr = result.dyn_cast<AffineConstantExpr>();
3356  if (!constExpr)
3357  return parser.emitError(parser.getNameLoc(),
3358  "steps must be constant integers");
3359  steps.push_back(constExpr.getValue());
3360  }
3361  result.addAttribute(AffineParallelOp::getStepsAttrName(),
3362  builder.getI64ArrayAttr(steps));
3363  }
3364 
3365  // Parse optional clause of the form: `reduce ("addf", "maxf")`, where the
3366  // quoted strings are a member of the enum AtomicRMWKind.
3367  SmallVector<Attribute, 4> reductions;
3368  if (succeeded(parser.parseOptionalKeyword("reduce"))) {
3369  if (parser.parseLParen())
3370  return failure();
3371  do {
3372  // Parse a single quoted string via the attribute parsing, and then
3373  // verify it is a member of the enum and convert to it's integer
3374  // representation.
3375  StringAttr attrVal;
3376  NamedAttrList attrStorage;
3377  auto loc = parser.getCurrentLocation();
3378  if (parser.parseAttribute(attrVal, builder.getNoneType(), "reduce",
3379  attrStorage))
3380  return failure();
3382  arith::symbolizeAtomicRMWKind(attrVal.getValue());
3383  if (!reduction)
3384  return parser.emitError(loc, "invalid reduction value: ") << attrVal;
3385  reductions.push_back(builder.getI64IntegerAttr(
3386  static_cast<int64_t>(reduction.getValue())));
3387  // While we keep getting commas, keep parsing.
3388  } while (succeeded(parser.parseOptionalComma()));
3389  if (parser.parseRParen())
3390  return failure();
3391  }
3392  result.addAttribute(AffineParallelOp::getReductionsAttrName(),
3393  builder.getArrayAttr(reductions));
3394 
3395  // Parse return types of reductions (if any)
3396  if (parser.parseOptionalArrowTypeList(result.types))
3397  return failure();
3398 
3399  // Now parse the body.
3400  Region *body = result.addRegion();
3401  SmallVector<Type, 4> types(ivs.size(), indexType);
3402  if (parser.parseRegion(*body, ivs, types) ||
3403  parser.parseOptionalAttrDict(result.attributes))
3404  return failure();
3405 
3406  // Add a terminator if none was parsed.
3407  AffineParallelOp::ensureTerminator(*body, builder, result.location);
3408  return success();
3409 }
3410 
3411 //===----------------------------------------------------------------------===//
3412 // AffineYieldOp
3413 //===----------------------------------------------------------------------===//
3414 
3415 static LogicalResult verify(AffineYieldOp op) {
3416  auto *parentOp = op->getParentOp();
3417  auto results = parentOp->getResults();
3418  auto operands = op.getOperands();
3419 
3420  if (!isa<AffineParallelOp, AffineIfOp, AffineForOp>(parentOp))
3421  return op.emitOpError() << "only terminates affine.if/for/parallel regions";
3422  if (parentOp->getNumResults() != op.getNumOperands())
3423  return op.emitOpError() << "parent of yield must have same number of "
3424  "results as the yield operands";
3425  for (auto it : llvm::zip(results, operands)) {
3426  if (std::get<0>(it).getType() != std::get<1>(it).getType())
3427  return op.emitOpError()
3428  << "types mismatch between yield op and its parent";
3429  }
3430 
3431  return success();
3432 }
3433 
3434 //===----------------------------------------------------------------------===//
3435 // AffineVectorLoadOp
3436 //===----------------------------------------------------------------------===//
3437 
3438 void AffineVectorLoadOp::build(OpBuilder &builder, OperationState &result,
3439  VectorType resultType, AffineMap map,
3440  ValueRange operands) {
3441  assert(operands.size() == 1 + map.getNumInputs() && "inconsistent operands");
3442  result.addOperands(operands);
3443  if (map)
3444  result.addAttribute(getMapAttrName(), AffineMapAttr::get(map));
3445  result.types.push_back(resultType);
3446 }
3447 
3448 void AffineVectorLoadOp::build(OpBuilder &builder, OperationState &result,
3449  VectorType resultType, Value memref,
3450  AffineMap map, ValueRange mapOperands) {
3451  assert(map.getNumInputs() == mapOperands.size() && "inconsistent index info");
3452  result.addOperands(memref);
3453  result.addOperands(mapOperands);
3454  result.addAttribute(getMapAttrName(), AffineMapAttr::get(map));
3455  result.types.push_back(resultType);
3456 }
3457 
3458 void AffineVectorLoadOp::build(OpBuilder &builder, OperationState &result,
3459  VectorType resultType, Value memref,
3460  ValueRange indices) {
3461  auto memrefType = memref.getType().cast<MemRefType>();
3462  int64_t rank = memrefType.getRank();
3463  // Create identity map for memrefs with at least one dimension or () -> ()
3464  // for zero-dimensional memrefs.
3465  auto map =
3466  rank ? builder.getMultiDimIdentityMap(rank) : builder.getEmptyAffineMap();
3467  build(builder, result, resultType, memref, map, indices);
3468 }
3469 
3470 void AffineVectorLoadOp::getCanonicalizationPatterns(RewritePatternSet &results,
3471  MLIRContext *context) {
3472  results.add<SimplifyAffineOp<AffineVectorLoadOp>>(context);
3473 }
3474 
3476  OperationState &result) {
3477  auto &builder = parser.getBuilder();
3478  auto indexTy = builder.getIndexType();
3479 
3480  MemRefType memrefType;
3481  VectorType resultType;
3482  OpAsmParser::OperandType memrefInfo;
3483  AffineMapAttr mapAttr;
3485  return failure(
3486  parser.parseOperand(memrefInfo) ||
3487  parser.parseAffineMapOfSSAIds(mapOperands, mapAttr,
3488  AffineVectorLoadOp::getMapAttrName(),
3489  result.attributes) ||
3490  parser.parseOptionalAttrDict(result.attributes) ||
3491  parser.parseColonType(memrefType) || parser.parseComma() ||
3492  parser.parseType(resultType) ||
3493  parser.resolveOperand(memrefInfo, memrefType, result.operands) ||
3494  parser.resolveOperands(mapOperands, indexTy, result.operands) ||
3495  parser.addTypeToList(resultType, result.types));
3496 }
3497 
3498 static void print(OpAsmPrinter &p, AffineVectorLoadOp op) {
3499  p << " " << op.getMemRef() << '[';
3500  if (AffineMapAttr mapAttr =
3501  op->getAttrOfType<AffineMapAttr>(op.getMapAttrName()))
3502  p.printAffineMapOfSSAIds(mapAttr, op.getMapOperands());
3503  p << ']';
3504  p.printOptionalAttrDict(op->getAttrs(),
3505  /*elidedAttrs=*/{op.getMapAttrName()});
3506  p << " : " << op.getMemRefType() << ", " << op.getType();
3507 }
3508 
3509 /// Verify common invariants of affine.vector_load and affine.vector_store.
3510 static LogicalResult verifyVectorMemoryOp(Operation *op, MemRefType memrefType,
3511  VectorType vectorType) {
3512  // Check that memref and vector element types match.
3513  if (memrefType.getElementType() != vectorType.getElementType())
3514  return op->emitOpError(
3515  "requires memref and vector types of the same elemental type");
3516  return success();
3517 }
3518 
3519 static LogicalResult verify(AffineVectorLoadOp op) {
3520  MemRefType memrefType = op.getMemRefType();
3522  op.getOperation(),
3523  op->getAttrOfType<AffineMapAttr>(op.getMapAttrName()),
3524  op.getMapOperands(), memrefType,
3525  /*numIndexOperands=*/op.getNumOperands() - 1)))
3526  return failure();
3527 
3528  if (failed(verifyVectorMemoryOp(op.getOperation(), memrefType,
3529  op.getVectorType())))
3530  return failure();
3531 
3532  return success();
3533 }
3534 
3535 //===----------------------------------------------------------------------===//
3536 // AffineVectorStoreOp
3537 //===----------------------------------------------------------------------===//
3538 
3539 void AffineVectorStoreOp::build(OpBuilder &builder, OperationState &result,
3540  Value valueToStore, Value memref, AffineMap map,
3541  ValueRange mapOperands) {
3542  assert(map.getNumInputs() == mapOperands.size() && "inconsistent index info");
3543  result.addOperands(valueToStore);
3544  result.addOperands(memref);
3545  result.addOperands(mapOperands);
3546  result.addAttribute(getMapAttrName(), AffineMapAttr::get(map));
3547 }
3548 
3549 // Use identity map.
3550 void AffineVectorStoreOp::build(OpBuilder &builder, OperationState &result,
3551  Value valueToStore, Value memref,
3552  ValueRange indices) {
3553  auto memrefType = memref.getType().cast<MemRefType>();
3554  int64_t rank = memrefType.getRank();
3555  // Create identity map for memrefs with at least one dimension or () -> ()
3556  // for zero-dimensional memrefs.
3557  auto map =
3558  rank ? builder.getMultiDimIdentityMap(rank) : builder.getEmptyAffineMap();
3559  build(builder, result, valueToStore, memref, map, indices);
3560 }
3561 void AffineVectorStoreOp::getCanonicalizationPatterns(
3562  RewritePatternSet &results, MLIRContext *context) {
3563  results.add<SimplifyAffineOp<AffineVectorStoreOp>>(context);
3564 }
3565 
3567  OperationState &result) {
3568  auto indexTy = parser.getBuilder().getIndexType();
3569 
3570  MemRefType memrefType;
3571  VectorType resultType;
3572  OpAsmParser::OperandType storeValueInfo;
3573  OpAsmParser::OperandType memrefInfo;
3574  AffineMapAttr mapAttr;
3576  return failure(
3577  parser.parseOperand(storeValueInfo) || parser.parseComma() ||
3578  parser.parseOperand(memrefInfo) ||
3579  parser.parseAffineMapOfSSAIds(mapOperands, mapAttr,
3580  AffineVectorStoreOp::getMapAttrName(),
3581  result.attributes) ||
3582  parser.parseOptionalAttrDict(result.attributes) ||
3583  parser.parseColonType(memrefType) || parser.parseComma() ||
3584  parser.parseType(resultType) ||
3585  parser.resolveOperand(storeValueInfo, resultType, result.operands) ||
3586  parser.resolveOperand(memrefInfo, memrefType, result.operands) ||
3587  parser.resolveOperands(mapOperands, indexTy, result.operands));
3588 }
3589 
3590 static void print(OpAsmPrinter &p, AffineVectorStoreOp op) {
3591  p << " " << op.getValueToStore();
3592  p << ", " << op.getMemRef() << '[';
3593  if (AffineMapAttr mapAttr =
3594  op->getAttrOfType<AffineMapAttr>(op.getMapAttrName()))
3595  p.printAffineMapOfSSAIds(mapAttr, op.getMapOperands());
3596  p << ']';
3597  p.printOptionalAttrDict(op->getAttrs(),
3598  /*elidedAttrs=*/{op.getMapAttrName()});
3599  p << " : " << op.getMemRefType() << ", " << op.getValueToStore().getType();
3600 }
3601 
3602 static LogicalResult verify(AffineVectorStoreOp op) {
3603  MemRefType memrefType = op.getMemRefType();
3605  op.getOperation(),
3606  op->getAttrOfType<AffineMapAttr>(op.getMapAttrName()),
3607  op.getMapOperands(), memrefType,
3608  /*numIndexOperands=*/op.getNumOperands() - 2)))
3609  return failure();
3610 
3611  if (failed(verifyVectorMemoryOp(op.getOperation(), memrefType,
3612  op.getVectorType())))
3613  return failure();
3614 
3615  return success();
3616 }
3617 
3618 //===----------------------------------------------------------------------===//
3619 // TableGen'd op method definitions
3620 //===----------------------------------------------------------------------===//
3621 
3622 #define GET_OP_CLASSES
3623 #include "mlir/Dialect/Affine/IR/AffineOps.cpp.inc"
static bool isLegalToInline(InlinerInterface &interface, Region *src, Region *insertRegion, bool shouldCloneInlinedRegion, BlockAndValueMapping &valueMapping)
Utility to check that all of the operations within &#39;src&#39; can be inlined.
operand_range::iterator operand_iterator
Definition: Operation.h:241
virtual ParseResult parseOperand(OperandType &result)=0
Parse a single operand.
This is the representation of an operand reference.
Include the generated interface declarations.
OpTy create(Location location, Args &&...args)
Create an operation of specific op type at the current insertion point.
Definition: Builders.h:430
AffineMap getEmptyAffineMap()
Returns a zero result affine map with no dimensions or symbols: () -> ().
Definition: Builders.cpp:297
static LogicalResult verifyVectorMemoryOp(Operation *op, MemRefType memrefType, VectorType vectorType)
Verify common invariants of affine.vector_load and affine.vector_store.
Definition: AffineOps.cpp:3510
This class contains a list of basic blocks and a link to the parent operation it is attached to...
Definition: Region.h:26
ParseResult resolveOperands(ArrayRef< OperandType > operands, Type type, SmallVectorImpl< Value > &result)
Resolve a list of operands to SSA values, emitting an error on failure, or appending the results to t...
virtual ParseResult parseLParen()=0
Parse a ( token.
static ParseResult parseAffineLoadOp(OpAsmParser &parser, OperationState &result)
Definition: AffineOps.cpp:2267
MLIRContext * getContext() const
Definition: Builders.h:54
void createOrFold(SmallVectorImpl< Value > &results, Location location, Args &&...args)
Create an operation of specific op type at the current insertion point, and immediately try to fold i...
Definition: Builders.h:444
A special type of RewriterBase that coordinates the application of a rewrite pattern on the current I...
Definition: PatternMatch.h:881
AffineMap getMultiDimIdentityMap(unsigned rank)
Definition: Builders.cpp:308
NamedAttrList is array of NamedAttributes that tracks whether it is sorted and does some basic work t...
Operation is a basic unit of execution within MLIR.
Definition: Operation.h:28
unsigned getNumSymbols() const
Definition: AffineMap.cpp:298
unsigned getNumDims() const
Definition: AffineMap.cpp:294
static bool isMemRefSizeValidSymbol(AnyMemRefDefOp memrefDefOp, unsigned index, Region *region)
Returns true if the &#39;index&#39; dimension of the memref defined by memrefDefOp is a statically shaped one...
Definition: AffineOps.cpp:317
ParseResult parseDimAndSymbolList(OpAsmParser &parser, SmallVectorImpl< Value > &operands, unsigned &numDims)
Parses dimension and symbol list.
Definition: AffineOps.cpp:452
operand_range getOperands()
Returns an iterator on the underlying Value&#39;s.
Definition: Operation.h:247
AffineExpr getAffineConstantExpr(int64_t constant, MLIRContext *context)
Definition: AffineExpr.cpp:516
static void printDimAndSymbolList(Operation::operand_iterator begin, Operation::operand_iterator end, unsigned numDims, OpAsmPrinter &printer)
Prints dimension and symbol list.
Definition: AffineOps.cpp:442
Block represents an ordered list of Operations.
Definition: Block.h:29
Block & front()
Definition: Region.h:65
void setInsertionPoint(Block *block, Block::iterator insertPoint)
Set the insertion point to the specified location.
Definition: Builders.h:329
static bool hasTrivialZeroTripCount(AffineForOp op)
Returns true if the affine.for has zero iterations in trivial cases.
Definition: AffineOps.cpp:1690
LogicalResult matchAndRewrite(T affineOp, PatternRewriter &rewriter) const override
Definition: AffineOps.cpp:2646
virtual void printAffineExprOfSSAIds(AffineExpr expr, ValueRange dimOperands, ValueRange symOperands)=0
Prints an affine expression of SSA ids with SSA id names used instead of dims and symbols...
virtual ParseResult parseAffineExprOfSSAIds(SmallVectorImpl< OperandType > &dimOperands, SmallVectorImpl< OperandType > &symbOperands, AffineExpr &expr)=0
Parses an affine expression where dims and symbols are SSA operands.
virtual void eraseOp(Operation *op)
This method erases an operation that is known to have no uses.
This class represents a single result from folding an operation.
Definition: OpDefinition.h:244
unsigned getNumInputs() const
Definition: IntegerSet.cpp:17
static Operation::operand_range getUpperBoundOperands(AffineForOp forOp)
Definition: SCFToGPU.cpp:79
LogicalResult verify(Operation *op)
Perform (potentially expensive) checks of invariants, used to detect compiler bugs, on this operation and any nested operations.
Definition: Verifier.cpp:353
virtual ParseResult parseRegionArgument(OperandType &argument)=0
Parse a region argument, this argument is resolved when calling &#39;parseRegion&#39;.
A trait of region holding operations that defines a new scope for polyhedral optimization purposes...
static LogicalResult foldMemRefCast(Operation *op, Value ignore=nullptr)
This is a common class used for patterns of the form "someop(memrefcast) -> someop".
Definition: AffineOps.cpp:987
void push_back(Block *block)
Definition: Region.h:61
bool failed(LogicalResult result)
Utility function that returns true if the provided LogicalResult corresponds to a failure value...
Definition: LogicalResult.h:72
static bool isValidAffineIndexOperand(Value value, Region *region)
Definition: AffineOps.cpp:437
AffineExpr getResult(unsigned i)
static void printAffineMinMaxOp(OpAsmPrinter &p, T op)
Definition: AffineOps.cpp:2456
NoneType getNoneType()
Definition: Builders.cpp:75
ParseResult parseAssignmentList(SmallVectorImpl< OperandType > &lhs, SmallVectorImpl< OperandType > &rhs)
Parse a list of assignments of the form (x1 = y1, x2 = y2, ...)
static ParseResult parseAffineVectorStoreOp(OpAsmParser &parser, OperationState &result)
Definition: AffineOps.cpp:3566
static LogicalResult verifyAffineMinMaxOp(T op)
Definition: AffineOps.cpp:2448
ParseResult addTypeToList(Type type, SmallVectorImpl< Type > &result)
Add the specified type to the end of the specified type list and return success.
AffineApplyOp makeComposedAffineApply(OpBuilder &b, Location loc, AffineMap map, ValueRange operands)
Returns a composed AffineApplyOp by composing map and operands with other AffineApplyOps supplying th...
Definition: AffineOps.cpp:714
bool succeeded(LogicalResult result)
Utility function that returns true if the provided LogicalResult corresponds to a success value...
Definition: LogicalResult.h:68
virtual ParseResult parseTrailingOperandList(SmallVectorImpl< OperandType > &result, int requiredOperandCount=-1, Delimiter delimiter=Delimiter::None)=0
Parse zero or more trailing SSA comma-separated trailing operand references with a specified surround...
T lookup(T from) const
Lookup a mapped value within the map.
The OpAsmParser has methods for interacting with the asm parser: parsing things from it...
virtual Builder & getBuilder() const =0
Return a builder which provides useful access to MLIRContext, global objects like types and attribute...
static void printBound(AffineMapAttr boundMap, Operation::operand_range boundOperands, const char *prefix, OpAsmPrinter &p)
Definition: AffineOps.cpp:1510
ArrayAttr getI64ArrayAttr(ArrayRef< int64_t > values)
Definition: Builders.cpp:220
AffineMap shiftDims(unsigned shift, unsigned offset=0) const
Replace dims[offset ...
Definition: AffineMap.h:216
virtual void startRootUpdate(Operation *op)
This method is used to notify the rewriter that an in-place operation modification is about to happen...
Definition: PatternMatch.h:774
Attribute erase(StringAttr name)
Erase the attribute with the given name from the list.
static void difference(const AffineValueMap &a, const AffineValueMap &b, AffineValueMap *res)
Return the value map that is the difference of value maps &#39;a&#39; and &#39;b&#39;, represented as an affine map a...
bool isFunctionOfSymbol(unsigned position) const
Return true if any affine expression involves AffineSymbolExpr position.
Definition: AffineMap.h:177
virtual ParseResult parseOptionalKeyword(StringRef keyword)=0
Parse the given keyword if present.
An integer constant appearing in affine expression.
Definition: AffineExpr.h:232
Region * getParent() const
Provide a &#39;getParent&#39; method for ilist_node_with_parent methods.
Definition: Block.cpp:26
virtual ParseResult parseComma()=0
Parse a , token.
virtual llvm::SMLoc getNameLoc() const =0
Return the location of the original name token.
void extractForInductionVars(ArrayRef< AffineForOp > forInsts, SmallVectorImpl< Value > *ivs)
Extracts the induction variables from a list of AffineForOps and places them in the output argument i...
Definition: AffineOps.cpp:1857
static constexpr const bool value
static void printMinMaxBound(OpAsmPrinter &p, AffineMapAttr mapAttr, DenseIntElementsAttr group, ValueRange operands, StringRef keyword)
Prints a lower(upper) bound of an affine parallel loop with max(min) conditions in it...
Definition: AffineOps.cpp:3112
virtual ParseResult parseOptionalComma()=0
Parse a , token if present.
SmallVector< Value, 4 > operands
This class defines the main interface for locations in MLIR and acts as a non-nullable wrapper around...
Definition: Location.h:48
virtual ParseResult parseOperandList(SmallVectorImpl< OperandType > &result, int requiredOperandCount=-1, Delimiter delimiter=Delimiter::None)=0
Parse zero or more SSA comma-separated operand references with a specified surrounding delimiter...
AffineExpr getResult(unsigned idx) const
Definition: AffineMap.cpp:315
static AffineMap getConstantMap(int64_t val, MLIRContext *context)
Returns a single constant result affine map.
Definition: AffineMap.cpp:95
unsigned getNumInputs() const
Definition: AffineMap.cpp:306
Block * getOwner() const
Returns the block that owns this argument.
Definition: Value.h:307
MutableArrayRef< OpOperand > getOpOperands()
Definition: Operation.h:252
LogicalResult matchAndRewrite(T affineOp, PatternRewriter &rewriter) const override
Definition: AffineOps.cpp:2566
static void canonicalizeMapOrSetAndOperands(MapOrSet *mapOrSet, SmallVectorImpl< Value > *operands)
Definition: AffineOps.cpp:803
MinMaxKind
Definition: AffineOps.cpp:3206
U dyn_cast() const
Definition: AffineExpr.h:281
virtual ParseResult parseArrowTypeList(SmallVectorImpl< Type > &result)=0
Parse an arrow followed by a type list.
LogicalResult success(bool isSuccess=true)
Utility function to generate a LogicalResult.
Definition: LogicalResult.h:56
Merge an affine min/max op to its consumers if its consumer is also an affine min/max op...
Definition: AffineOps.cpp:2563
This class represents an efficient way to signal success or failure.
Definition: LogicalResult.h:26
AffineMap removeDuplicateExprs(AffineMap map)
Returns a map with the same dimension and symbol count as map, but whose results are the unique affin...
Definition: AffineMap.cpp:664
static ParseResult parseAffineIfOp(OpAsmParser &parser, OperationState &result)
Definition: AffineOps.cpp:2089
virtual ParseResult resolveOperand(const OperandType &operand, Type type, SmallVectorImpl< Value > &result)=0
Resolve an operand to an SSA value, emitting an error on failure.
LogicalResult failure(bool isFailure=true)
Utility function to generate a LogicalResult.
Definition: LogicalResult.h:62
AffineMap getSliceMap(unsigned start, unsigned length) const
Returns the map consisting of length expressions starting from start.
Definition: AffineMap.cpp:527
bool isValidDim(Value value)
Returns true if the given Value can be used as a dimension id in the region of the closest surroundin...
Definition: AffineOps.cpp:264
static LogicalResult verifyMemoryOpIndexing(Operation *op, AffineMapAttr mapAttr, Operation::operand_range mapOperands, MemRefType memrefType, unsigned numIndexOperands)
Verify common indexing invariants of affine.load, affine.store, affine.vector_load and affine...
Definition: AffineOps.cpp:2302
virtual void replaceOp(Operation *op, ValueRange newValues)
This method replaces the results of the operation with the specified list of values.
static ParseResult parseBound(bool isLower, OperationState &result, OpAsmParser &p)
Parse a for operation loop bounds.
Definition: AffineOps.cpp:1350
bool isStrided(MemRefType t)
Return true if the layout for t is compatible with strided semantics.
virtual ParseResult parseRegion(Region &region, ArrayRef< OperandType > arguments={}, ArrayRef< Type > argTypes={}, ArrayRef< Location > argLocations={}, bool enableNameShadowing=false)=0
Parses a region.
virtual ParseResult parseGreater()=0
Parse a &#39;>&#39; token.
static AffineMap get(MLIRContext *context)
Returns a zero result affine map with no dimensions or symbols: () -> ().
Region * getParentRegion()
Return the Region in which this Value is defined.
Definition: Value.cpp:41
void addOperands(ValueRange newOperands)
virtual void printAffineMapOfSSAIds(AffineMapAttr mapAttr, ValueRange operands)=0
Prints an affine map of SSA ids, where SSA id names are used in place of dims/symbols.
virtual llvm::SMLoc getCurrentLocation()=0
Get the location of the next token and store it into the argument.
IntegerAttr getIntegerAttr(Type type, int64_t value)
Definition: Builders.cpp:170
void printOptionalArrowTypeList(TypeRange &&types)
Print an optional arrow followed by a type list.
IntegerAttr getI64IntegerAttr(int64_t value)
Definition: Builders.cpp:99
bool isValidSymbol(Value value)
Returns true if the given value can be used as a symbol in the region of the closest surrounding op t...
Definition: AffineOps.cpp:361
static AffineForOp buildAffineLoopFromValues(OpBuilder &builder, Location loc, Value lb, Value ub, int64_t step, AffineForOp::BodyBuilderFn bodyBuilderFn)
Creates an affine loop from the bounds that may or may not be constants.
Definition: AffineOps.cpp:1917
AffineExpr getAffineSymbolExpr(unsigned position, MLIRContext *context)
Definition: AffineExpr.cpp:501
Attributes are known-constant values of operations.
Definition: Attributes.h:24
void fullyComposeAffineMapAndOperands(AffineMap *map, SmallVectorImpl< Value > *operands)
Given an affine map map and its input operands, this method composes into map, maps of AffineApplyOps...
Definition: AffineOps.cpp:705
AffineMap replaceDimsAndSymbols(ArrayRef< AffineExpr > dimReplacements, ArrayRef< AffineExpr > symReplacements, unsigned numResultDims, unsigned numResultSyms) const
This method substitutes any uses of dimensions and symbols (e.g.
Definition: AffineMap.cpp:397
U dyn_cast() const
Definition: Value.h:99
static ParseResult parseAffineStoreOp(OpAsmParser &parser, OperationState &result)
Definition: AffineOps.cpp:2382
bool hasTrait()
Returns true if the operation was registered with a particular trait, e.g.
Definition: Operation.h:470
Operation * getParentOp()
Returns the closest surrounding operation that contains this operation or nullptr if this is a top-le...
Definition: Operation.h:117
constexpr void enumerate(std::tuple< Tys... > &tuple, CallbackT &&callback)
Definition: Matchers.h:206
DialectInlinerInterface(Dialect *dialect)
Definition: InliningUtils.h:44
static bool isDimOpValidSymbol(OpTy dimOp, Region *region)
Returns true if the result of the dim op is a valid symbol for region.
Definition: AffineOps.cpp:331
void print(OpAsmPrinter &p)
Definition: AffineOps.cpp:1180
IntegerType getIntegerType(unsigned width)
Definition: Builders.cpp:58
bool isIndex() const
Definition: Types.cpp:28
virtual ParseResult parseRParen()=0
Parse a ) token.
This is the interface that must be implemented by the dialects of operations to be inlined...
Definition: InliningUtils.h:41
Base type for affine expression.
Definition: AffineExpr.h:68
virtual ParseResult parseAffineMapOfSSAIds(SmallVectorImpl< OperandType > &operands, Attribute &map, StringRef attrName, NamedAttrList &attrs, Delimiter delimiter=Delimiter::Square)=0
Parses an affine map attribute where dims and symbols are SSA operands.
void canonicalizeMapAndOperands(AffineMap *map, SmallVectorImpl< Value > *operands)
Modifies both map and operands in-place so as to:
Definition: AffineOps.cpp:879
void print(OpAsmPrinter &p)
Definition: AffineOps.cpp:1027
LogicalResult constantFold(ArrayRef< Attribute > operandConstants, SmallVectorImpl< Attribute > &results) const
Folds the results of the application of an affine map on the provided operands to a constant if possi...
Definition: AffineMap.cpp:336
void canonicalizeSetAndOperands(IntegerSet *set, SmallVectorImpl< Value > *operands)
Canonicalizes an integer set the same way canonicalizeMapAndOperands does for affine maps...
Definition: AffineOps.cpp:884
This class provides an abstraction over the various different ranges of value types.
Definition: TypeRange.h:38
static void build(OpBuilder &builder, OperationState &result, Value tagMemRef, AffineMap tagMap, ValueRange tagIndices, Value numElements)
Definition: AffineOps.cpp:1171
void addTypes(ArrayRef< Type > newTypes)
static void canonicalizePromotedSymbols(MapOrSet *mapOrSet, SmallVectorImpl< Value > *operands)
Definition: AffineOps.cpp:760
ParseResult parseKeyword(StringRef keyword, const Twine &msg="")
Parse a given keyword.
virtual ParseResult parseLess()=0
Parse a &#39;<&#39; token.
SmallVector< Value, 4 > applyMapToValues(OpBuilder &b, Location loc, AffineMap map, ValueRange values)
Returns the values obtained by applying map to the list of values.
Definition: AffineOps.cpp:742
This is a pure-virtual base class that exposes the asmprinter hooks necessary to implement a custom p...
unsigned getNumResults() const
Definition: AffineMap.cpp:302
This represents an operation in an abstracted form, suitable for use with the builder APIs...
void mergeBlockBefore(Block *source, Operation *op, ValueRange argValues=llvm::None)
A multi-dimensional affine map Affine map&#39;s are immutable like Type&#39;s, and they are uniqued...
Definition: AffineMap.h:38
bool isForInductionVar(Value val)
Returns true if the provided value is the induction variable of a AffineForOp.
Definition: AffineOps.cpp:1838
AffineDmaWaitOp blocks until the completion of a DMA operation associated with the tag element &#39;tag[i...
Definition: AffineOps.h:266
DenseIntElementsAttr getI32TensorAttr(ArrayRef< int32_t > values)
Tensor-typed DenseIntElementsAttr getters.
Definition: Builders.cpp:127
Parens surrounding zero or more operands.
static Operation::operand_range getLowerBoundOperands(AffineForOp forOp)
Definition: SCFToGPU.cpp:74
unsigned getNumDims() const
Definition: IntegerSet.cpp:15
detail::constant_op_matcher m_Constant()
Matches a constant foldable operation.
Definition: Matchers.h:231
AffineMap getAffineMap() const
BlockArgListType getArguments()
Definition: Block.h:76
AffineBound represents a lower or upper bound in the for operation.
Definition: AffineOps.h:432
This class represents an argument of a Block.
Definition: Value.h:298
static void build(OpBuilder &builder, OperationState &result, Value srcMemRef, AffineMap srcMap, ValueRange srcIndices, Value destMemRef, AffineMap dstMap, ValueRange destIndices, Value tagMemRef, AffineMap tagMap, ValueRange tagIndices, Value numElements, Value stride=nullptr, Value elementsPerStride=nullptr)
Definition: AffineOps.cpp:1005
auto getType() const
ArrayRef< AffineExpr > getResults() const
Definition: AffineMap.cpp:311
AffineMap replace(AffineExpr expr, AffineExpr replacement, unsigned numResultDims, unsigned numResultSyms) const
Sparse replace method.
Definition: AffineMap.cpp:412
static LogicalResult canonicalizeLoopBounds(AffineForOp forOp)
Canonicalize the bounds of the given loop.
Definition: AffineOps.cpp:1638
AffineExpr getAffineDimExpr(unsigned position, MLIRContext *context)
These free functions allow clients of the API to not use classes in detail.
Definition: AffineExpr.cpp:491
static ParseResult parse(OpAsmParser &parser, OperationState &result)
Definition: AffineOps.cpp:1194
virtual void finalizeRootUpdate(Operation *op)
This method is used to signal the end of a root update on the given operation.
Definition: PatternMatch.h:779
Remove duplicated expressions in affine min/max ops.
Definition: AffineOps.cpp:2521
LogicalResult verify()
Definition: AffineOps.cpp:1122
Instances of the Type class are uniqued, have an immutable identifier and an optional mutable compone...
Definition: Types.h:72
static void buildAffineLoopNestImpl(OpBuilder &builder, Location loc, BoundListTy lbs, BoundListTy ubs, ArrayRef< int64_t > steps, function_ref< void(OpBuilder &, Location, ValueRange)> bodyBuilderFn, LoopCreatorTy &&loopCreatorFn)
Builds an affine loop nest, using "loopCreatorFn" to create individual loop operations.
Definition: AffineOps.cpp:1867
Operation * getParentOp()
Return the parent operation this region is attached to.
Definition: Region.h:200
virtual ParseResult parseOptionalRParen()=0
Parse a ) token if present.
AffineForOp getForInductionVarOwner(Value val)
Returns the loop parent of an induction variable.
Definition: AffineOps.cpp:1844
static ParseResult parse(OpAsmParser &parser, OperationState &result)
Definition: AffineOps.cpp:1049
This class represents an instance of an SSA value in the MLIR system, representing a computable value...
Definition: Value.h:84
void addAttribute(StringRef name, Attribute attr)
Add an attribute with the specified name.
static LogicalResult verifyDimAndSymbolIdentifiers(OpTy &op, Operation::operand_range operands, unsigned numDims)
Utility function to verify that a set of operands are valid dimension and symbol identifiers.
Definition: AffineOps.cpp:475
static ParseResult parseAffineParallelOp(OpAsmParser &parser, OperationState &result)
Definition: AffineOps.cpp:3324
static LogicalResult foldLoopBounds(AffineForOp forOp)
Fold the constant bounds of a loop.
Definition: AffineOps.cpp:1592
bool isTopLevelValue(Value value)
A utility function to check if a value is defined at the top level of an op with trait AffineScope or...
Definition: AffineOps.cpp:233
Operation * getTerminator()
Get the terminator operation of this block.
Definition: Block.cpp:230
static Value createFoldedComposedAffineApply(OpBuilder &b, Location loc, AffineMap map, ValueRange operandsRef)
Fully compose map with operands and canonicalize the result.
Definition: AffineOps.cpp:733
NamedAttrList attributes
LogicalResult verify()
Definition: AffineOps.cpp:1224
OpRewritePattern is a wrapper around RewritePattern that allows for matching and rewriting against an...
Definition: PatternMatch.h:355
void setInsertionPointToStart(Block *block)
Sets the insertion point to the start of the specified block.
Definition: Builders.h:362
static ParseResult parseAffineVectorLoadOp(OpAsmParser &parser, OperationState &result)
Definition: AffineOps.cpp:3475
virtual InFlightDiagnostic emitError(llvm::SMLoc loc, const Twine &message={})=0
Emit a diagnostic at the specified location and return failure.
bool isa() const
Definition: Value.h:89
void pop_back()
Pop last element from list.
static ParseResult parseAffinePrefetchOp(OpAsmParser &parser, OperationState &result)
Definition: AffineOps.cpp:2701
virtual void printOptionalAttrDict(ArrayRef< NamedAttribute > attrs, ArrayRef< StringRef > elidedAttrs={})=0
If the specified operation has attributes, print out an attribute dictionary with their values...
RAII guard to reset the insertion point of the builder when destroyed.
Definition: Builders.h:279
LogicalResult fold(ArrayRef< Attribute > cstOperands, SmallVectorImpl< OpFoldResult > &results)
Definition: AffineOps.cpp:1237
Region * addRegion()
Create a region that should be attached to the operation.
OpTy replaceOpWithNewOp(Operation *op, Args &&... args)
Replaces the result op with a new op that is created without verification.
Definition: PatternMatch.h:741
This class is a general helper class for creating context-global objects like types, attributes, and affine expressions.
Definition: Builders.h:49
AffineMap simplifyAffineMap(AffineMap map)
Simplifies an affine map by simplifying its underlying AffineExpr results.
Definition: AffineMap.cpp:654
Type getType() const
Return the type of this value.
Definition: Value.h:117
static void print(OpAsmPrinter &p, AffineApplyOp op)
Definition: AffineOps.cpp:520
RewritePatternSet & add(ConstructorArg &&arg, ConstructorArgs &&... args)
Add an instance of each of the pattern types &#39;Ts&#39; to the pattern list with the given arguments...
Definition: PatternMatch.h:930
IndexType getIndexType()
Definition: Builders.cpp:48
static ParseResult parseAffineMinMaxOp(OpAsmParser &parser, OperationState &result)
Definition: AffineOps.cpp:2469
This class provides the API for ops that are known to be isolated from above.
AffineMap getDimIdentityMap()
Definition: Builders.cpp:304
U dyn_cast() const
Definition: Attributes.h:117
bool matchPattern(Value value, const Pattern &pattern)
Entry point for matching a pattern over a Value.
Definition: Matchers.h:266
static int64_t getNumElements(ShapedType type)
Definition: TensorOps.cpp:678
Symbolic identifier.
LogicalResult matchAndRewrite(T affineOp, PatternRewriter &rewriter) const override
Definition: AffineOps.cpp:2524
A dimensional identifier appearing in an affine expression.
Definition: AffineExpr.h:216
Specialization of arith.constant op that returns an integer of index type.
Definition: Arithmetic.h:78
static VectorType vectorType(CodeGen &codegen, Type etp)
Constructs vector type.
virtual void printOperand(Value value)=0
Print implementations for various things an operation contains.
bool isFunctionOfDim(unsigned position) const
Return true if any affine expression involves AffineDimExpr position.
Definition: AffineMap.h:170
BoolAttr getBoolAttr(bool value)
Definition: Builders.cpp:87
virtual ParseResult parseType(Type &result)=0
Parse a type.
static LogicalResult replaceDimOrSym(AffineMap *map, unsigned dimOrSymbolPosition, SmallVectorImpl< Value > &dims, SmallVectorImpl< Value > &syms)
Replace all occurrences of AffineExpr at position pos in map by the defining AffineApplyOp expression...
Definition: AffineOps.cpp:600
Operation * getDefiningOp() const
If this value is the result of an operation, return the operation that defines it.
Definition: Value.cpp:20
AffineExprKind
Definition: AffineExpr.h:40
static void deduplicateAndResolveOperands(OpAsmParser &parser, ArrayRef< SmallVector< OpAsmParser::OperandType >> operands, SmallVectorImpl< Value > &uniqueOperands, SmallVectorImpl< AffineExpr > &replacements, AffineExprKind kind)
Given a list of lists of parsed operands, populates uniqueOperands with unique operands.
Definition: AffineOps.cpp:3180
MLIRContext is the top-level object for a collection of MLIR operations.
Definition: MLIRContext.h:55
This class represents an operand of an operation.
Definition: Value.h:249
AffineForOp replaceForOpWithNewYields(OpBuilder &b, AffineForOp loop, ValueRange newIterOperands, ValueRange newYieldedValues, ValueRange newIterArgs, bool replaceLoopResults=true)
Replace loop with a new loop where newIterOperands are appended with new initialization values and ne...
Definition: AffineOps.cpp:1946
AffineDmaStartOp starts a non-blocking DMA operation that transfers data from a source memref to a de...
Definition: AffineOps.h:74
This class implements the operand iterators for the Operation class.
U cast() const
Definition: Value.h:107
static Region * getAffineScope(Operation *op)
Returns the closest region enclosing op that is held by an operation with trait AffineScope; nullptr ...
Definition: AffineOps.cpp:249
Region * getParentRegion()
Returns the region to which the instruction belongs.
Definition: Operation.h:113
static ParseResult parseAffineMapWithMinMax(OpAsmParser &parser, OperationState &result, MinMaxKind kind)
Parses an affine map that can contain a min/max for groups of its results, e.g., max(expr-1, expr-2), expr-3, max(expr-4, expr-5, expr-6).
Definition: AffineOps.cpp:3225
AffineMap getConstantAffineMap(int64_t val)
Returns a single constant result affine map with 0 dimensions and 0 symbols.
Definition: Builders.cpp:299
An AffineValueMap is an affine map plus its ML value operands and results for analysis purposes...
virtual ParseResult parseOptionalAttrDict(NamedAttrList &result)=0
Parse a named dictionary into &#39;result&#39; if it is present.
virtual ParseResult parseEqual()=0
Parse a = token.
MLIRContext * getContext() const
Definition: AffineMap.cpp:253
virtual ParseResult parseColonTypeList(SmallVectorImpl< Type > &result)=0
Parse a colon followed by a type list, which must have at least one type.
SmallVector< std::unique_ptr< Region >, 1 > regions
Regions that the op will hold.
LogicalResult canonicalize()
Attempts to canonicalize the map and operands.
Definition: AffineOps.cpp:3072
InFlightDiagnostic emitOpError(const Twine &message={})
Emit an error with the op name prefixed, like "&#39;dim&#39; op " which is convenient for verifiers...
Definition: Operation.cpp:518
static ParseResult parseAffineForOp(OpAsmParser &parser, OperationState &result)
Definition: AffineOps.cpp:1441
bool isa() const
Definition: Types.h:234
static SmallVector< AffineMap, 4 > inferFromExprList(ArrayRef< ArrayRef< AffineExpr >> exprsList)
Returns a vector of AffineMaps; each with as many results as exprs.size(), as many dims as the larges...
Definition: AffineMap.cpp:235
BlockArgument addArgument(Type type, Location loc)
Add one value to the argument list.
Definition: Block.cpp:141
Square brackets supporting zero or more ops, or nothing.
MLIRContext * getContext() const
Definition: AsmPrinter.cpp:61
This class represents success/failure for operation parsing.
Definition: OpDefinition.h:36
static bool remainsLegalAfterInline(Value value, Region *src, Region *dest, const BlockAndValueMapping &mapping, function_ref< bool(Value, Region *)> legalityCheck)
Checks if value known to be a legal affine dimension or symbol in src region remains legal if the ope...
Definition: AffineOps.cpp:49
ArrayRef< Value > getOperands() const
virtual ParseResult parseAttribute(Attribute &result, Type type={})=0
Parse an arbitrary attribute of a given type and return it in result.
This class helps build Operations.
Definition: Builders.h:177
ArrayAttr getArrayAttr(ArrayRef< Attribute > value)
Definition: Builders.cpp:205
This class provides an abstraction over the different types of ranges over Values.
static void composeAffineMapAndOperands(AffineMap *map, SmallVectorImpl< Value > *operands)
Iterate over operands and fold away all those produced by an AffineApplyOp iteratively.
Definition: AffineOps.cpp:643
static ParseResult parseAffineApplyOp(OpAsmParser &parser, OperationState &result)
Definition: AffineOps.cpp:497
virtual ParseResult parseRegionArgumentList(SmallVectorImpl< OperandType > &result, int requiredOperandCount=-1, Delimiter delimiter=Delimiter::None)=0
Parse zero or more region arguments with a specified surrounding delimiter, and an optional required ...
Dimensional identifier.
virtual ParseResult parseOptionalArrowTypeList(SmallVectorImpl< Type > &result)=0
Parse an optional arrow followed by a type list.
bool isSingleConstant() const
Returns true if this affine map is a single result constant function.
Definition: AffineMap.cpp:271
static AffineForOp buildAffineLoopFromConstants(OpBuilder &builder, Location loc, int64_t lb, int64_t ub, int64_t step, AffineForOp::BodyBuilderFn bodyBuilderFn)
Creates an affine loop from the bounds known to be constants.
Definition: AffineOps.cpp:1908
virtual ParseResult parseColonType(Type &result)=0
Parse a colon followed by a type.
bool isAncestor(Region *other)
Return true if this region is ancestor of the other region.
Definition: Region.h:222
static Operation * materializeConstant(Dialect *dialect, OpBuilder &builder, Attribute value, Type type, Location loc)
A utility function used to materialize a constant for a given attribute and type. ...
Definition: FoldUtils.cpp:50
LogicalResult fold(ArrayRef< Attribute > cstOperands, SmallVectorImpl< OpFoldResult > &results)
Definition: AffineOps.cpp:1160
static OpFoldResult foldMinMaxOp(T op, ArrayRef< Attribute > operands)
Fold an affine min or max operation with the given operands.
Definition: AffineOps.cpp:2491
An attribute that represents a reference to a dense integer vector or tensor object.
U cast() const
Definition: Types.h:250
A symbolic identifier appearing in an affine expression.
Definition: AffineExpr.h:224
virtual void printRegion(Region &blocks, bool printEntryBlockArgs=true, bool printBlockTerminators=true, bool printEmptyBlock=false)=0
Prints a region.
SmallVector< Type, 4 > types
Types of the results of this operation.
virtual void eraseBlock(Block *block)
This method erases all operations in a block.
void buildAffineLoopNest(OpBuilder &builder, Location loc, ArrayRef< int64_t > lbs, ArrayRef< int64_t > ubs, ArrayRef< int64_t > steps, function_ref< void(OpBuilder &, Location, ValueRange)> bodyBuilderFn=nullptr)
Builds a perfect nest of affine.for loops, i.e., each loop except the innermost one contains only ano...
Definition: AffineOps.cpp:1930
unsigned getNumResults() const
An integer set representing a conjunction of one or more affine equalities and inequalities.
Definition: IntegerSet.h:44