MLIR  21.0.0git
Operation.cpp
Go to the documentation of this file.
1 //===- Operation.cpp - Operation support code -----------------------------===//
2 //
3 // Part of the LLVM Project, under the Apache License v2.0 with LLVM Exceptions.
4 // See https://llvm.org/LICENSE.txt for license information.
5 // SPDX-License-Identifier: Apache-2.0 WITH LLVM-exception
6 //
7 //===----------------------------------------------------------------------===//
8 
9 #include "mlir/IR/Operation.h"
10 #include "mlir/IR/Attributes.h"
12 #include "mlir/IR/BuiltinTypes.h"
13 #include "mlir/IR/Dialect.h"
14 #include "mlir/IR/IRMapping.h"
15 #include "mlir/IR/Matchers.h"
18 #include "mlir/IR/PatternMatch.h"
19 #include "mlir/IR/TypeUtilities.h"
21 #include "llvm/ADT/SmallVector.h"
22 #include "llvm/Support/ErrorHandling.h"
23 #include <numeric>
24 #include <optional>
25 
26 using namespace mlir;
27 
28 //===----------------------------------------------------------------------===//
29 // Operation
30 //===----------------------------------------------------------------------===//
31 
32 /// Create a new Operation from operation state.
34  Operation *op =
35  create(state.location, state.name, state.types, state.operands,
36  state.attributes.getDictionary(state.getContext()),
37  state.properties, state.successors, state.regions);
38  if (LLVM_UNLIKELY(state.propertiesAttr)) {
39  assert(!state.properties);
40  LogicalResult result =
41  op->setPropertiesFromAttribute(state.propertiesAttr,
42  /*diagnostic=*/nullptr);
43  assert(result.succeeded() && "invalid properties in op creation");
44  (void)result;
45  }
46  return op;
47 }
48 
49 /// Create a new Operation with the specific fields.
51  TypeRange resultTypes, ValueRange operands,
52  NamedAttrList &&attributes,
53  OpaqueProperties properties, BlockRange successors,
54  RegionRange regions) {
55  unsigned numRegions = regions.size();
56  Operation *op =
57  create(location, name, resultTypes, operands, std::move(attributes),
58  properties, successors, numRegions);
59  for (unsigned i = 0; i < numRegions; ++i)
60  if (regions[i])
61  op->getRegion(i).takeBody(*regions[i]);
62  return op;
63 }
64 
65 /// Create a new Operation with the specific fields.
67  TypeRange resultTypes, ValueRange operands,
68  NamedAttrList &&attributes,
69  OpaqueProperties properties, BlockRange successors,
70  unsigned numRegions) {
71  // Populate default attributes.
72  name.populateDefaultAttrs(attributes);
73 
74  return create(location, name, resultTypes, operands,
75  attributes.getDictionary(location.getContext()), properties,
76  successors, numRegions);
77 }
78 
79 /// Overload of create that takes an existing DictionaryAttr to avoid
80 /// unnecessarily uniquing a list of attributes.
82  TypeRange resultTypes, ValueRange operands,
83  DictionaryAttr attributes,
84  OpaqueProperties properties, BlockRange successors,
85  unsigned numRegions) {
86  assert(llvm::all_of(resultTypes, [](Type t) { return t; }) &&
87  "unexpected null result type");
88 
89  // We only need to allocate additional memory for a subset of results.
90  unsigned numTrailingResults = OpResult::getNumTrailing(resultTypes.size());
91  unsigned numInlineResults = OpResult::getNumInline(resultTypes.size());
92  unsigned numSuccessors = successors.size();
93  unsigned numOperands = operands.size();
94  unsigned numResults = resultTypes.size();
95  int opPropertiesAllocSize = llvm::alignTo<8>(name.getOpPropertyByteSize());
96 
97  // If the operation is known to have no operands, don't allocate an operand
98  // storage.
99  bool needsOperandStorage =
100  operands.empty() ? !name.hasTrait<OpTrait::ZeroOperands>() : true;
101 
102  // Compute the byte size for the operation and the operand storage. This takes
103  // into account the size of the operation, its trailing objects, and its
104  // prefixed objects.
105  size_t byteSize =
108  needsOperandStorage ? 1 : 0, opPropertiesAllocSize, numSuccessors,
109  numRegions, numOperands);
110  size_t prefixByteSize = llvm::alignTo(
111  Operation::prefixAllocSize(numTrailingResults, numInlineResults),
112  alignof(Operation));
113  char *mallocMem = reinterpret_cast<char *>(malloc(byteSize + prefixByteSize));
114  void *rawMem = mallocMem + prefixByteSize;
115 
116  // Create the new Operation.
117  Operation *op = ::new (rawMem) Operation(
118  location, name, numResults, numSuccessors, numRegions,
119  opPropertiesAllocSize, attributes, properties, needsOperandStorage);
120 
121  assert((numSuccessors == 0 || op->mightHaveTrait<OpTrait::IsTerminator>()) &&
122  "unexpected successors in a non-terminator operation");
123 
124  // Initialize the results.
125  auto resultTypeIt = resultTypes.begin();
126  for (unsigned i = 0; i < numInlineResults; ++i, ++resultTypeIt)
127  new (op->getInlineOpResult(i)) detail::InlineOpResult(*resultTypeIt, i);
128  for (unsigned i = 0; i < numTrailingResults; ++i, ++resultTypeIt) {
129  new (op->getOutOfLineOpResult(i))
130  detail::OutOfLineOpResult(*resultTypeIt, i);
131  }
132 
133  // Initialize the regions.
134  for (unsigned i = 0; i != numRegions; ++i)
135  new (&op->getRegion(i)) Region(op);
136 
137  // Initialize the operands.
138  if (needsOperandStorage) {
139  new (&op->getOperandStorage()) detail::OperandStorage(
140  op, op->getTrailingObjects<OpOperand>(), operands);
141  }
142 
143  // Initialize the successors.
144  auto blockOperands = op->getBlockOperands();
145  for (unsigned i = 0; i != numSuccessors; ++i)
146  new (&blockOperands[i]) BlockOperand(op, successors[i]);
147 
148  // This must be done after properties are initalized.
149  op->setAttrs(attributes);
150 
151  return op;
152 }
153 
154 Operation::Operation(Location location, OperationName name, unsigned numResults,
155  unsigned numSuccessors, unsigned numRegions,
156  int fullPropertiesStorageSize, DictionaryAttr attributes,
157  OpaqueProperties properties, bool hasOperandStorage)
158  : location(location), numResults(numResults), numSuccs(numSuccessors),
159  numRegions(numRegions), hasOperandStorage(hasOperandStorage),
160  propertiesStorageSize((fullPropertiesStorageSize + 7) / 8), name(name) {
161  assert(attributes && "unexpected null attribute dictionary");
162  assert(fullPropertiesStorageSize <= propertiesCapacity &&
163  "Properties size overflow");
164 #ifndef NDEBUG
165  if (!getDialect() && !getContext()->allowsUnregisteredDialects())
166  llvm::report_fatal_error(
167  name.getStringRef() +
168  " created with unregistered dialect. If this is intended, please call "
169  "allowUnregisteredDialects() on the MLIRContext, or use "
170  "-allow-unregistered-dialect with the MLIR tool used.");
171 #endif
172  if (fullPropertiesStorageSize)
173  name.initOpProperties(getPropertiesStorage(), properties);
174 }
175 
176 // Operations are deleted through the destroy() member because they are
177 // allocated via malloc.
178 Operation::~Operation() {
179  assert(block == nullptr && "operation destroyed but still in a block");
180 #ifndef NDEBUG
181  if (!use_empty()) {
182  {
184  emitOpError("operation destroyed but still has uses");
185  for (Operation *user : getUsers())
186  diag.attachNote(user->getLoc()) << "- use: " << *user << "\n";
187  }
188  llvm::report_fatal_error("operation destroyed but still has uses");
189  }
190 #endif
191  // Explicitly run the destructors for the operands.
192  if (hasOperandStorage)
193  getOperandStorage().~OperandStorage();
194 
195  // Explicitly run the destructors for the successors.
196  for (auto &successor : getBlockOperands())
197  successor.~BlockOperand();
198 
199  // Explicitly destroy the regions.
200  for (auto &region : getRegions())
201  region.~Region();
202  if (propertiesStorageSize)
204 }
205 
206 /// Destroy this operation or one of its subclasses.
208  // Operations may have additional prefixed allocation, which needs to be
209  // accounted for here when computing the address to free.
210  char *rawMem = reinterpret_cast<char *>(this) -
211  llvm::alignTo(prefixAllocSize(), alignof(Operation));
212  this->~Operation();
213  free(rawMem);
214 }
215 
216 /// Return true if this operation is a proper ancestor of the `other`
217 /// operation.
219  while ((other = other->getParentOp()))
220  if (this == other)
221  return true;
222  return false;
223 }
224 
225 /// Replace any uses of 'from' with 'to' within this operation.
227  if (from == to)
228  return;
229  for (auto &operand : getOpOperands())
230  if (operand.get() == from)
231  operand.set(to);
232 }
233 
234 /// Replace the current operands of this operation with the ones provided in
235 /// 'operands'.
237  if (LLVM_LIKELY(hasOperandStorage))
238  return getOperandStorage().setOperands(this, operands);
239  assert(operands.empty() && "setting operands without an operand storage");
240 }
241 
242 /// Replace the operands beginning at 'start' and ending at 'start' + 'length'
243 /// with the ones provided in 'operands'. 'operands' may be smaller or larger
244 /// than the range pointed to by 'start'+'length'.
245 void Operation::setOperands(unsigned start, unsigned length,
246  ValueRange operands) {
247  assert((start + length) <= getNumOperands() &&
248  "invalid operand range specified");
249  if (LLVM_LIKELY(hasOperandStorage))
250  return getOperandStorage().setOperands(this, start, length, operands);
251  assert(operands.empty() && "setting operands without an operand storage");
252 }
253 
254 /// Insert the given operands into the operand list at the given 'index'.
255 void Operation::insertOperands(unsigned index, ValueRange operands) {
256  if (LLVM_LIKELY(hasOperandStorage))
257  return setOperands(index, /*length=*/0, operands);
258  assert(operands.empty() && "inserting operands without an operand storage");
259 }
260 
261 //===----------------------------------------------------------------------===//
262 // Diagnostics
263 //===----------------------------------------------------------------------===//
264 
265 /// Emit an error about fatal conditions with this operation, reporting up to
266 /// any diagnostic handlers that may be listening.
269  if (getContext()->shouldPrintOpOnDiagnostic()) {
270  diag.attachNote(getLoc())
271  .append("see current operation: ")
272  .appendOp(*this, OpPrintingFlags().printGenericOpForm());
273  }
274  return diag;
275 }
276 
277 /// Emit a warning about this operation, reporting up to any diagnostic
278 /// handlers that may be listening.
281  if (getContext()->shouldPrintOpOnDiagnostic())
282  diag.attachNote(getLoc()) << "see current operation: " << *this;
283  return diag;
284 }
285 
286 /// Emit a remark about this operation, reporting up to any diagnostic
287 /// handlers that may be listening.
290  if (getContext()->shouldPrintOpOnDiagnostic())
291  diag.attachNote(getLoc()) << "see current operation: " << *this;
292  return diag;
293 }
294 
296  if (getPropertiesStorageSize()) {
297  NamedAttrList attrsList = attrs;
298  getName().populateInherentAttrs(this, attrsList);
299  return attrsList.getDictionary(getContext());
300  }
301  return attrs;
302 }
303 
304 void Operation::setAttrs(DictionaryAttr newAttrs) {
305  assert(newAttrs && "expected valid attribute dictionary");
306  if (getPropertiesStorageSize()) {
307  // We're spliting the providing DictionaryAttr by removing the inherentAttr
308  // which will be stored in the properties.
309  SmallVector<NamedAttribute> discardableAttrs;
310  discardableAttrs.reserve(newAttrs.size());
311  for (NamedAttribute attr : newAttrs) {
312  if (getInherentAttr(attr.getName()))
313  setInherentAttr(attr.getName(), attr.getValue());
314  else
315  discardableAttrs.push_back(attr);
316  }
317  if (discardableAttrs.size() != newAttrs.size())
318  newAttrs = DictionaryAttr::get(getContext(), discardableAttrs);
319  }
320  attrs = newAttrs;
321 }
323  if (getPropertiesStorageSize()) {
324  // We're spliting the providing array of attributes by removing the
325  // inherentAttr which will be stored in the properties.
326  SmallVector<NamedAttribute> discardableAttrs;
327  discardableAttrs.reserve(newAttrs.size());
328  for (NamedAttribute attr : newAttrs) {
329  if (getInherentAttr(attr.getName()))
330  setInherentAttr(attr.getName(), attr.getValue());
331  else
332  discardableAttrs.push_back(attr);
333  }
334  attrs = DictionaryAttr::get(getContext(), discardableAttrs);
335  return;
336  }
337  attrs = DictionaryAttr::get(getContext(), newAttrs);
338 }
339 
340 std::optional<Attribute> Operation::getInherentAttr(StringRef name) {
341  return getName().getInherentAttr(this, name);
342 }
343 
344 void Operation::setInherentAttr(StringAttr name, Attribute value) {
345  getName().setInherentAttr(this, name, value);
346 }
347 
349  std::optional<RegisteredOperationName> info = getRegisteredInfo();
350  if (LLVM_UNLIKELY(!info))
351  return *getPropertiesStorage().as<Attribute *>();
352  return info->getOpPropertiesAsAttribute(this);
353 }
356  std::optional<RegisteredOperationName> info = getRegisteredInfo();
357  if (LLVM_UNLIKELY(!info)) {
358  *getPropertiesStorage().as<Attribute *>() = attr;
359  return success();
360  }
361  return info->setOpPropertiesFromAttribute(
362  this->getName(), this->getPropertiesStorage(), attr, emitError);
363 }
364 
367 }
368 
369 llvm::hash_code Operation::hashProperties() {
370  return name.hashOpProperties(getPropertiesStorage());
371 }
372 
373 //===----------------------------------------------------------------------===//
374 // Operation Ordering
375 //===----------------------------------------------------------------------===//
376 
377 constexpr unsigned Operation::kInvalidOrderIdx;
378 constexpr unsigned Operation::kOrderStride;
379 
380 /// Given an operation 'other' that is within the same parent block, return
381 /// whether the current operation is before 'other' in the operation list
382 /// of the parent block.
383 /// Note: This function has an average complexity of O(1), but worst case may
384 /// take O(N) where N is the number of operations within the parent block.
386  assert(block && "Operations without parent blocks have no order.");
387  assert(other && other->block == block &&
388  "Expected other operation to have the same parent block.");
389  // If the order of the block is already invalid, directly recompute the
390  // parent.
391  if (!block->isOpOrderValid()) {
392  block->recomputeOpOrder();
393  } else {
394  // Update the order either operation if necessary.
395  updateOrderIfNecessary();
396  other->updateOrderIfNecessary();
397  }
398 
399  return orderIndex < other->orderIndex;
400 }
401 
402 /// Update the order index of this operation of this operation if necessary,
403 /// potentially recomputing the order of the parent block.
404 void Operation::updateOrderIfNecessary() {
405  assert(block && "expected valid parent");
406 
407  // If the order is valid for this operation there is nothing to do.
408  if (hasValidOrder() || llvm::hasSingleElement(*block))
409  return;
410  Operation *blockFront = &block->front();
411  Operation *blockBack = &block->back();
412 
413  // This method is expected to only be invoked on blocks with more than one
414  // operation.
415  assert(blockFront != blockBack && "expected more than one operation");
416 
417  // If the operation is at the end of the block.
418  if (this == blockBack) {
419  Operation *prevNode = getPrevNode();
420  if (!prevNode->hasValidOrder())
421  return block->recomputeOpOrder();
422 
423  // Add the stride to the previous operation.
424  orderIndex = prevNode->orderIndex + kOrderStride;
425  return;
426  }
427 
428  // If this is the first operation try to use the next operation to compute the
429  // ordering.
430  if (this == blockFront) {
431  Operation *nextNode = getNextNode();
432  if (!nextNode->hasValidOrder())
433  return block->recomputeOpOrder();
434  // There is no order to give this operation.
435  if (nextNode->orderIndex == 0)
436  return block->recomputeOpOrder();
437 
438  // If we can't use the stride, just take the middle value left. This is safe
439  // because we know there is at least one valid index to assign to.
440  if (nextNode->orderIndex <= kOrderStride)
441  orderIndex = (nextNode->orderIndex / 2);
442  else
443  orderIndex = kOrderStride;
444  return;
445  }
446 
447  // Otherwise, this operation is between two others. Place this operation in
448  // the middle of the previous and next if possible.
449  Operation *prevNode = getPrevNode(), *nextNode = getNextNode();
450  if (!prevNode->hasValidOrder() || !nextNode->hasValidOrder())
451  return block->recomputeOpOrder();
452  unsigned prevOrder = prevNode->orderIndex, nextOrder = nextNode->orderIndex;
453 
454  // Check to see if there is a valid order between the two.
455  if (prevOrder + 1 == nextOrder)
456  return block->recomputeOpOrder();
457  orderIndex = prevOrder + ((nextOrder - prevOrder) / 2);
458 }
459 
460 //===----------------------------------------------------------------------===//
461 // ilist_traits for Operation
462 //===----------------------------------------------------------------------===//
463 
464 auto llvm::ilist_detail::SpecificNodeAccess<
465  typename llvm::ilist_detail::compute_node_options<
466  ::mlir::Operation>::type>::getNodePtr(pointer n) -> node_type * {
467  return NodeAccess::getNodePtr<OptionsT>(n);
468 }
469 
470 auto llvm::ilist_detail::SpecificNodeAccess<
471  typename llvm::ilist_detail::compute_node_options<
472  ::mlir::Operation>::type>::getNodePtr(const_pointer n)
473  -> const node_type * {
474  return NodeAccess::getNodePtr<OptionsT>(n);
475 }
476 
477 auto llvm::ilist_detail::SpecificNodeAccess<
478  typename llvm::ilist_detail::compute_node_options<
479  ::mlir::Operation>::type>::getValuePtr(node_type *n) -> pointer {
480  return NodeAccess::getValuePtr<OptionsT>(n);
481 }
482 
483 auto llvm::ilist_detail::SpecificNodeAccess<
484  typename llvm::ilist_detail::compute_node_options<
485  ::mlir::Operation>::type>::getValuePtr(const node_type *n)
486  -> const_pointer {
487  return NodeAccess::getValuePtr<OptionsT>(n);
488 }
489 
491  op->destroy();
492 }
493 
494 Block *llvm::ilist_traits<::mlir::Operation>::getContainingBlock() {
495  size_t offset(size_t(&((Block *)nullptr->*Block::getSublistAccess(nullptr))));
496  iplist<Operation> *anchor(static_cast<iplist<Operation> *>(this));
497  return reinterpret_cast<Block *>(reinterpret_cast<char *>(anchor) - offset);
498 }
499 
500 /// This is a trait method invoked when an operation is added to a block. We
501 /// keep the block pointer up to date.
503  assert(!op->getBlock() && "already in an operation block!");
504  op->block = getContainingBlock();
505 
506  // Invalidate the order on the operation.
507  op->orderIndex = Operation::kInvalidOrderIdx;
508 }
509 
510 /// This is a trait method invoked when an operation is removed from a block.
511 /// We keep the block pointer up to date.
513  assert(op->block && "not already in an operation block!");
514  op->block = nullptr;
515 }
516 
517 /// This is a trait method invoked when an operation is moved from one block
518 /// to another. We keep the block pointer up to date.
520  ilist_traits<Operation> &otherList, op_iterator first, op_iterator last) {
521  Block *curParent = getContainingBlock();
522 
523  // Invalidate the ordering of the parent block.
524  curParent->invalidateOpOrder();
525 
526  // If we are transferring operations within the same block, the block
527  // pointer doesn't need to be updated.
528  if (curParent == otherList.getContainingBlock())
529  return;
530 
531  // Update the 'block' member of each operation.
532  for (; first != last; ++first)
533  first->block = curParent;
534 }
535 
536 /// Remove this operation (and its descendants) from its Block and delete
537 /// all of them.
539  if (auto *parent = getBlock())
540  parent->getOperations().erase(this);
541  else
542  destroy();
543 }
544 
545 /// Remove the operation from its parent block, but don't delete it.
547  if (Block *parent = getBlock())
548  parent->getOperations().remove(this);
549 }
550 
551 /// Unlink this operation from its current block and insert it right before
552 /// `existingOp` which may be in the same or another block in the same
553 /// function.
554 void Operation::moveBefore(Operation *existingOp) {
555  moveBefore(existingOp->getBlock(), existingOp->getIterator());
556 }
557 
558 /// Unlink this operation from its current basic block and insert it right
559 /// before `iterator` in the specified basic block.
561  llvm::iplist<Operation>::iterator iterator) {
562  assert(getBlock() &&
563  "cannot move an operation that isn't contained in a block");
564  block->getOperations().splice(iterator, getBlock()->getOperations(),
565  getIterator());
566 }
567 
568 /// Unlink this operation from its current block and insert it right after
569 /// `existingOp` which may be in the same or another block in the same function.
570 void Operation::moveAfter(Operation *existingOp) {
571  moveAfter(existingOp->getBlock(), existingOp->getIterator());
572 }
573 
574 /// Unlink this operation from its current block and insert it right after
575 /// `iterator` in the specified block.
577  llvm::iplist<Operation>::iterator iterator) {
578  assert(iterator != block->end() && "cannot move after end of block");
579  moveBefore(block, std::next(iterator));
580 }
581 
582 /// This drops all operand uses from this operation, which is an essential
583 /// step in breaking cyclic dependences between references when they are to
584 /// be deleted.
586  for (auto &op : getOpOperands())
587  op.drop();
588 
589  for (auto &region : getRegions())
590  region.dropAllReferences();
591 
592  for (auto &dest : getBlockOperands())
593  dest.drop();
594 }
595 
596 /// This drops all uses of any values defined by this operation or its nested
597 /// regions, wherever they are located.
599  dropAllUses();
600 
601  for (auto &region : getRegions())
602  for (auto &block : region)
603  block.dropAllDefinedValueUses();
604 }
605 
606 void Operation::setSuccessor(Block *block, unsigned index) {
607  assert(index < getNumSuccessors());
608  getBlockOperands()[index].set(block);
609 }
610 
611 #ifndef NDEBUG
612 /// Assert that the folded results (in case of values) have the same type as
613 /// the results of the given op.
616  if (results.empty())
617  return;
618 
619  for (auto [ofr, opResult] : llvm::zip_equal(results, op->getResults())) {
620  if (auto value = dyn_cast<Value>(ofr)) {
621  if (value.getType() != opResult.getType()) {
622  op->emitOpError() << "folder produced a value of incorrect type: "
623  << value.getType()
624  << ", expected: " << opResult.getType();
625  assert(false && "incorrect fold result type");
626  }
627  }
628  }
629 }
630 #endif // NDEBUG
631 
632 /// Attempt to fold this operation using the Op's registered foldHook.
633 LogicalResult Operation::fold(ArrayRef<Attribute> operands,
635  // If we have a registered operation definition matching this one, use it to
636  // try to constant fold the operation.
637  if (succeeded(name.foldHook(this, operands, results))) {
638 #ifndef NDEBUG
639  checkFoldResultTypes(this, results);
640 #endif // NDEBUG
641  return success();
642  }
643 
644  // Otherwise, fall back on the dialect hook to handle it.
645  Dialect *dialect = getDialect();
646  if (!dialect)
647  return failure();
648 
649  auto *interface = dyn_cast<DialectFoldInterface>(dialect);
650  if (!interface)
651  return failure();
652 
653  LogicalResult status = interface->fold(this, operands, results);
654 #ifndef NDEBUG
655  if (succeeded(status))
656  checkFoldResultTypes(this, results);
657 #endif // NDEBUG
658  return status;
659 }
660 
662  // Check if any operands are constants.
663  SmallVector<Attribute> constants;
664  constants.assign(getNumOperands(), Attribute());
665  for (unsigned i = 0, e = getNumOperands(); i != e; ++i)
666  matchPattern(getOperand(i), m_Constant(&constants[i]));
667  return fold(constants, results);
668 }
669 
670 /// Emit an error with the op name prefixed, like "'dim' op " which is
671 /// convenient for verifiers.
673  return emitError() << "'" << getName() << "' op " << message;
674 }
675 
676 //===----------------------------------------------------------------------===//
677 // Operation Cloning
678 //===----------------------------------------------------------------------===//
679 
681  : cloneRegionsFlag(false), cloneOperandsFlag(false) {}
682 
683 Operation::CloneOptions::CloneOptions(bool cloneRegions, bool cloneOperands)
684  : cloneRegionsFlag(cloneRegions), cloneOperandsFlag(cloneOperands) {}
685 
688 }
689 
691  cloneRegionsFlag = enable;
692  return *this;
693 }
694 
696  cloneOperandsFlag = enable;
697  return *this;
698 }
699 
700 /// Create a deep copy of this operation but keep the operation regions empty.
701 /// Operands are remapped using `mapper` (if present), and `mapper` is updated
702 /// to contain the results. The `mapResults` flag specifies whether the results
703 /// of the cloned operation should be added to the map.
705  return clone(mapper, CloneOptions::all().cloneRegions(false));
706 }
707 
709  IRMapping mapper;
710  return cloneWithoutRegions(mapper);
711 }
712 
713 /// Create a deep copy of this operation, remapping any operands that use
714 /// values outside of the operation using the map that is provided (leaving
715 /// them alone if no entry is present). Replaces references to cloned
716 /// sub-operations to the corresponding operation that is copied, and adds
717 /// those mappings to the map.
719  SmallVector<Value, 8> operands;
720  SmallVector<Block *, 2> successors;
721 
722  // Remap the operands.
723  if (options.shouldCloneOperands()) {
724  operands.reserve(getNumOperands());
725  for (auto opValue : getOperands())
726  operands.push_back(mapper.lookupOrDefault(opValue));
727  }
728 
729  // Remap the successors.
730  successors.reserve(getNumSuccessors());
731  for (Block *successor : getSuccessors())
732  successors.push_back(mapper.lookupOrDefault(successor));
733 
734  // Create the new operation.
735  auto *newOp = create(getLoc(), getName(), getResultTypes(), operands, attrs,
736  getPropertiesStorage(), successors, getNumRegions());
737  mapper.map(this, newOp);
738 
739  // Clone the regions.
740  if (options.shouldCloneRegions()) {
741  for (unsigned i = 0; i != numRegions; ++i)
742  getRegion(i).cloneInto(&newOp->getRegion(i), mapper);
743  }
744 
745  // Remember the mapping of any results.
746  for (unsigned i = 0, e = getNumResults(); i != e; ++i)
747  mapper.map(getResult(i), newOp->getResult(i));
748 
749  return newOp;
750 }
751 
753  IRMapping mapper;
754  return clone(mapper, options);
755 }
756 
757 //===----------------------------------------------------------------------===//
758 // OpState trait class.
759 //===----------------------------------------------------------------------===//
760 
761 // The fallback for the parser is to try for a dialect operation parser.
762 // Otherwise, reject the custom assembly form.
763 ParseResult OpState::parse(OpAsmParser &parser, OperationState &result) {
764  if (auto parseFn = result.name.getDialect()->getParseOperationHook(
765  result.name.getStringRef()))
766  return (*parseFn)(parser, result);
767  return parser.emitError(parser.getNameLoc(), "has no custom assembly form");
768 }
769 
770 // The fallback for the printer is to try for a dialect operation printer.
771 // Otherwise, it prints the generic form.
772 void OpState::print(Operation *op, OpAsmPrinter &p, StringRef defaultDialect) {
773  if (auto printFn = op->getDialect()->getOperationPrinter(op)) {
774  printOpName(op, p, defaultDialect);
775  printFn(op, p);
776  } else {
777  p.printGenericOp(op);
778  }
779 }
780 
781 /// Print an operation name, eliding the dialect prefix if necessary and doesn't
782 /// lead to ambiguities.
784  StringRef defaultDialect) {
785  StringRef name = op->getName().getStringRef();
786  if (name.starts_with((defaultDialect + ".").str()) && name.count('.') == 1)
787  name = name.drop_front(defaultDialect.size() + 1);
788  p.getStream() << name;
789 }
790 
791 /// Parse properties as a Attribute.
793  Attribute &result) {
794  if (succeeded(parser.parseOptionalLess())) { // The less is optional.
795  if (parser.parseAttribute(result) || parser.parseGreater())
796  return failure();
797  }
798  return success();
799 }
800 
801 /// Print the properties as a Attribute with names not included within
802 /// 'elidedProps'
804  ArrayRef<StringRef> elidedProps) {
805  if (!properties)
806  return;
807  auto dictAttr = dyn_cast_or_null<::mlir::DictionaryAttr>(properties);
808  if (dictAttr && !elidedProps.empty()) {
809  ArrayRef<NamedAttribute> attrs = dictAttr.getValue();
810  llvm::SmallDenseSet<StringRef> elidedAttrsSet(elidedProps.begin(),
811  elidedProps.end());
812  auto filteredAttrs =
813  llvm::make_filter_range(attrs, [&](NamedAttribute attr) {
814  return !elidedAttrsSet.contains(attr.getName().strref());
815  });
816  if (!filteredAttrs.empty()) {
817  p << "<{";
818  interleaveComma(filteredAttrs, p, [&](NamedAttribute attr) {
819  p.printNamedAttribute(attr);
820  });
821  p << "}>";
822  }
823  } else {
824  p << "<" << properties << ">";
825  }
826 }
827 
828 /// Emit an error about fatal conditions with this operation, reporting up to
829 /// any diagnostic handlers that may be listening.
830 InFlightDiagnostic OpState::emitError(const Twine &message) {
831  return getOperation()->emitError(message);
832 }
833 
834 /// Emit an error with the op name prefixed, like "'dim' op " which is
835 /// convenient for verifiers.
837  return getOperation()->emitOpError(message);
838 }
839 
840 /// Emit a warning about this operation, reporting up to any diagnostic
841 /// handlers that may be listening.
843  return getOperation()->emitWarning(message);
844 }
845 
846 /// Emit a remark about this operation, reporting up to any diagnostic
847 /// handlers that may be listening.
849  return getOperation()->emitRemark(message);
850 }
851 
852 //===----------------------------------------------------------------------===//
853 // Op Trait implementations
854 //===----------------------------------------------------------------------===//
855 
856 LogicalResult
859  // Nothing to fold if there are not at least 2 operands.
860  if (op->getNumOperands() < 2)
861  return failure();
862  // Move all constant operands to the end.
863  OpOperand *operandsBegin = op->getOpOperands().begin();
864  auto isNonConstant = [&](OpOperand &o) {
865  return !static_cast<bool>(operands[std::distance(operandsBegin, &o)]);
866  };
867  auto *firstConstantIt = llvm::find_if_not(op->getOpOperands(), isNonConstant);
868  auto *newConstantIt = std::stable_partition(
869  firstConstantIt, op->getOpOperands().end(), isNonConstant);
870  // Return success if the op was modified.
871  return success(firstConstantIt != newConstantIt);
872 }
873 
875  if (op->getNumOperands() == 1) {
876  auto *argumentOp = op->getOperand(0).getDefiningOp();
877  if (argumentOp && op->getName() == argumentOp->getName()) {
878  // Replace the outer operation output with the inner operation.
879  return op->getOperand(0);
880  }
881  } else if (op->getOperand(0) == op->getOperand(1)) {
882  return op->getOperand(0);
883  }
884 
885  return {};
886 }
887 
889  auto *argumentOp = op->getOperand(0).getDefiningOp();
890  if (argumentOp && op->getName() == argumentOp->getName()) {
891  // Replace the outer involutions output with inner's input.
892  return argumentOp->getOperand(0);
893  }
894 
895  return {};
896 }
897 
899  if (op->getNumOperands() != 0)
900  return op->emitOpError() << "requires zero operands";
901  return success();
902 }
903 
905  if (op->getNumOperands() != 1)
906  return op->emitOpError() << "requires a single operand";
907  return success();
908 }
909 
911  unsigned numOperands) {
912  if (op->getNumOperands() != numOperands) {
913  return op->emitOpError() << "expected " << numOperands
914  << " operands, but found " << op->getNumOperands();
915  }
916  return success();
917 }
918 
920  unsigned numOperands) {
921  if (op->getNumOperands() < numOperands)
922  return op->emitOpError()
923  << "expected " << numOperands << " or more operands, but found "
924  << op->getNumOperands();
925  return success();
926 }
927 
928 /// If this is a vector type, or a tensor type, return the scalar element type
929 /// that it is built around, otherwise return the type unmodified.
931  if (auto vec = llvm::dyn_cast<VectorType>(type))
932  return vec.getElementType();
933 
934  // Look through tensor<vector<...>> to find the underlying element type.
935  if (auto tensor = llvm::dyn_cast<TensorType>(type))
936  return getTensorOrVectorElementType(tensor.getElementType());
937  return type;
938 }
939 
941  // FIXME: Add back check for no side effects on operation.
942  // Currently adding it would cause the shared library build
943  // to fail since there would be a dependency of IR on SideEffectInterfaces
944  // which is cyclical.
945  return success();
946 }
947 
949  // FIXME: Add back check for no side effects on operation.
950  // Currently adding it would cause the shared library build
951  // to fail since there would be a dependency of IR on SideEffectInterfaces
952  // which is cyclical.
953  return success();
954 }
955 
956 LogicalResult
958  for (auto opType : op->getOperandTypes()) {
959  auto type = getTensorOrVectorElementType(opType);
960  if (!type.isSignlessIntOrIndex())
961  return op->emitOpError() << "requires an integer or index type";
962  }
963  return success();
964 }
965 
967  for (auto opType : op->getOperandTypes()) {
968  auto type = getTensorOrVectorElementType(opType);
969  if (!llvm::isa<FloatType>(type))
970  return op->emitOpError("requires a float type");
971  }
972  return success();
973 }
974 
976  // Zero or one operand always have the "same" type.
977  unsigned nOperands = op->getNumOperands();
978  if (nOperands < 2)
979  return success();
980 
981  auto type = op->getOperand(0).getType();
982  for (auto opType : llvm::drop_begin(op->getOperandTypes(), 1))
983  if (opType != type)
984  return op->emitOpError() << "requires all operands to have the same type";
985  return success();
986 }
987 
989  if (op->getNumRegions() != 0)
990  return op->emitOpError() << "requires zero regions";
991  return success();
992 }
993 
995  if (op->getNumRegions() != 1)
996  return op->emitOpError() << "requires one region";
997  return success();
998 }
999 
1001  unsigned numRegions) {
1002  if (op->getNumRegions() != numRegions)
1003  return op->emitOpError() << "expected " << numRegions << " regions";
1004  return success();
1005 }
1006 
1008  unsigned numRegions) {
1009  if (op->getNumRegions() < numRegions)
1010  return op->emitOpError() << "expected " << numRegions << " or more regions";
1011  return success();
1012 }
1013 
1015  if (op->getNumResults() != 0)
1016  return op->emitOpError() << "requires zero results";
1017  return success();
1018 }
1019 
1021  if (op->getNumResults() != 1)
1022  return op->emitOpError() << "requires one result";
1023  return success();
1024 }
1025 
1027  unsigned numOperands) {
1028  if (op->getNumResults() != numOperands)
1029  return op->emitOpError() << "expected " << numOperands << " results";
1030  return success();
1031 }
1032 
1034  unsigned numOperands) {
1035  if (op->getNumResults() < numOperands)
1036  return op->emitOpError()
1037  << "expected " << numOperands << " or more results";
1038  return success();
1039 }
1040 
1042  if (failed(verifyAtLeastNOperands(op, 1)))
1043  return failure();
1044 
1045  if (failed(verifyCompatibleShapes(op->getOperandTypes())))
1046  return op->emitOpError() << "requires the same shape for all operands";
1047 
1048  return success();
1049 }
1050 
1052  if (failed(verifyAtLeastNOperands(op, 1)) ||
1053  failed(verifyAtLeastNResults(op, 1)))
1054  return failure();
1055 
1057  types.append(llvm::to_vector<4>(op->getResultTypes()));
1058 
1059  if (failed(verifyCompatibleShapes(types)))
1060  return op->emitOpError()
1061  << "requires the same shape for all operands and results";
1062 
1063  return success();
1064 }
1065 
1067  if (failed(verifyAtLeastNOperands(op, 1)))
1068  return failure();
1069  auto elementType = getElementTypeOrSelf(op->getOperand(0));
1070 
1071  for (auto operand : llvm::drop_begin(op->getOperands(), 1)) {
1072  if (getElementTypeOrSelf(operand) != elementType)
1073  return op->emitOpError("requires the same element type for all operands");
1074  }
1075 
1076  return success();
1077 }
1078 
1079 LogicalResult
1081  if (failed(verifyAtLeastNOperands(op, 1)) ||
1082  failed(verifyAtLeastNResults(op, 1)))
1083  return failure();
1084 
1085  auto elementType = getElementTypeOrSelf(op->getResult(0));
1086 
1087  // Verify result element type matches first result's element type.
1088  for (auto result : llvm::drop_begin(op->getResults(), 1)) {
1089  if (getElementTypeOrSelf(result) != elementType)
1090  return op->emitOpError(
1091  "requires the same element type for all operands and results");
1092  }
1093 
1094  // Verify operand's element type matches first result's element type.
1095  for (auto operand : op->getOperands()) {
1096  if (getElementTypeOrSelf(operand) != elementType)
1097  return op->emitOpError(
1098  "requires the same element type for all operands and results");
1099  }
1100 
1101  return success();
1102 }
1103 
1105  if (failed(verifyAtLeastNOperands(op, 1)) ||
1106  failed(verifyAtLeastNResults(op, 1)))
1107  return failure();
1108 
1109  auto type = op->getResult(0).getType();
1110  auto elementType = getElementTypeOrSelf(type);
1111  Attribute encoding = nullptr;
1112  if (auto rankedType = dyn_cast<RankedTensorType>(type))
1113  encoding = rankedType.getEncoding();
1114  for (auto resultType : llvm::drop_begin(op->getResultTypes())) {
1115  if (getElementTypeOrSelf(resultType) != elementType ||
1116  failed(verifyCompatibleShape(resultType, type)))
1117  return op->emitOpError()
1118  << "requires the same type for all operands and results";
1119  if (encoding)
1120  if (auto rankedType = dyn_cast<RankedTensorType>(resultType);
1121  encoding != rankedType.getEncoding())
1122  return op->emitOpError()
1123  << "requires the same encoding for all operands and results";
1124  }
1125  for (auto opType : op->getOperandTypes()) {
1126  if (getElementTypeOrSelf(opType) != elementType ||
1127  failed(verifyCompatibleShape(opType, type)))
1128  return op->emitOpError()
1129  << "requires the same type for all operands and results";
1130  if (encoding)
1131  if (auto rankedType = dyn_cast<RankedTensorType>(opType);
1132  encoding != rankedType.getEncoding())
1133  return op->emitOpError()
1134  << "requires the same encoding for all operands and results";
1135  }
1136  return success();
1137 }
1138 
1140  if (failed(verifyAtLeastNOperands(op, 1)))
1141  return failure();
1142 
1143  // delegate function that returns true if type is a shaped type with known
1144  // rank
1145  auto hasRank = [](const Type type) {
1146  if (auto shapedType = dyn_cast<ShapedType>(type))
1147  return shapedType.hasRank();
1148 
1149  return false;
1150  };
1151 
1152  auto rankedOperandTypes =
1153  llvm::make_filter_range(op->getOperandTypes(), hasRank);
1154  auto rankedResultTypes =
1155  llvm::make_filter_range(op->getResultTypes(), hasRank);
1156 
1157  // If all operands and results are unranked, then no further verification.
1158  if (rankedOperandTypes.empty() && rankedResultTypes.empty())
1159  return success();
1160 
1161  // delegate function that returns rank of shaped type with known rank
1162  auto getRank = [](const Type type) {
1163  return cast<ShapedType>(type).getRank();
1164  };
1165 
1166  auto rank = !rankedOperandTypes.empty() ? getRank(*rankedOperandTypes.begin())
1167  : getRank(*rankedResultTypes.begin());
1168 
1169  for (const auto type : rankedOperandTypes) {
1170  if (rank != getRank(type)) {
1171  return op->emitOpError("operands don't have matching ranks");
1172  }
1173  }
1174 
1175  for (const auto type : rankedResultTypes) {
1176  if (rank != getRank(type)) {
1177  return op->emitOpError("result type has different rank than operands");
1178  }
1179  }
1180 
1181  return success();
1182 }
1183 
1185  Block *block = op->getBlock();
1186  // Verify that the operation is at the end of the respective parent block.
1187  if (!block || &block->back() != op)
1188  return op->emitOpError("must be the last operation in the parent block");
1189  return success();
1190 }
1191 
1192 static LogicalResult verifyTerminatorSuccessors(Operation *op) {
1193  auto *parent = op->getParentRegion();
1194 
1195  // Verify that the operands lines up with the BB arguments in the successor.
1196  for (Block *succ : op->getSuccessors())
1197  if (succ->getParent() != parent)
1198  return op->emitError("reference to block defined in another region");
1199  return success();
1200 }
1201 
1203  if (op->getNumSuccessors() != 0) {
1204  return op->emitOpError("requires 0 successors but found ")
1205  << op->getNumSuccessors();
1206  }
1207  return success();
1208 }
1209 
1211  if (op->getNumSuccessors() != 1) {
1212  return op->emitOpError("requires 1 successor but found ")
1213  << op->getNumSuccessors();
1214  }
1215  return verifyTerminatorSuccessors(op);
1216 }
1218  unsigned numSuccessors) {
1219  if (op->getNumSuccessors() != numSuccessors) {
1220  return op->emitOpError("requires ")
1221  << numSuccessors << " successors but found "
1222  << op->getNumSuccessors();
1223  }
1224  return verifyTerminatorSuccessors(op);
1225 }
1227  unsigned numSuccessors) {
1228  if (op->getNumSuccessors() < numSuccessors) {
1229  return op->emitOpError("requires at least ")
1230  << numSuccessors << " successors but found "
1231  << op->getNumSuccessors();
1232  }
1233  return verifyTerminatorSuccessors(op);
1234 }
1235 
1237  for (auto resultType : op->getResultTypes()) {
1238  auto elementType = getTensorOrVectorElementType(resultType);
1239  bool isBoolType = elementType.isInteger(1);
1240  if (!isBoolType)
1241  return op->emitOpError() << "requires a bool result type";
1242  }
1243 
1244  return success();
1245 }
1246 
1248  for (auto resultType : op->getResultTypes())
1249  if (!llvm::isa<FloatType>(getTensorOrVectorElementType(resultType)))
1250  return op->emitOpError() << "requires a floating point type";
1251 
1252  return success();
1253 }
1254 
1255 LogicalResult
1257  for (auto resultType : op->getResultTypes())
1258  if (!getTensorOrVectorElementType(resultType).isSignlessIntOrIndex())
1259  return op->emitOpError() << "requires an integer or index type";
1260  return success();
1261 }
1262 
1264  StringRef attrName,
1265  StringRef valueGroupName,
1266  size_t expectedCount) {
1267  auto sizeAttr = op->getAttrOfType<DenseI32ArrayAttr>(attrName);
1268  if (!sizeAttr)
1269  return op->emitOpError("requires dense i32 array attribute '")
1270  << attrName << "'";
1271 
1272  ArrayRef<int32_t> sizes = sizeAttr.asArrayRef();
1273  if (llvm::any_of(sizes, [](int32_t element) { return element < 0; }))
1274  return op->emitOpError("'")
1275  << attrName << "' attribute cannot have negative elements";
1276 
1277  size_t totalCount =
1278  std::accumulate(sizes.begin(), sizes.end(), 0,
1279  [](unsigned all, int32_t one) { return all + one; });
1280 
1281  if (totalCount != expectedCount)
1282  return op->emitOpError()
1283  << valueGroupName << " count (" << expectedCount
1284  << ") does not match with the total size (" << totalCount
1285  << ") specified in attribute '" << attrName << "'";
1286  return success();
1287 }
1288 
1290  StringRef attrName) {
1291  return verifyValueSizeAttr(op, attrName, "operand", op->getNumOperands());
1292 }
1293 
1295  StringRef attrName) {
1296  return verifyValueSizeAttr(op, attrName, "result", op->getNumResults());
1297 }
1298 
1300  for (Region &region : op->getRegions()) {
1301  if (region.empty())
1302  continue;
1303 
1304  if (region.getNumArguments() != 0) {
1305  if (op->getNumRegions() > 1)
1306  return op->emitOpError("region #")
1307  << region.getRegionNumber() << " should have no arguments";
1308  return op->emitOpError("region should have no arguments");
1309  }
1310  }
1311  return success();
1312 }
1313 
1315  auto isMappableType = llvm::IsaPred<VectorType, TensorType>;
1316  auto resultMappableTypes =
1317  llvm::filter_to_vector<1>(op->getResultTypes(), isMappableType);
1318  auto operandMappableTypes =
1319  llvm::filter_to_vector<2>(op->getOperandTypes(), isMappableType);
1320 
1321  // If the op only has scalar operand/result types, then we have nothing to
1322  // check.
1323  if (resultMappableTypes.empty() && operandMappableTypes.empty())
1324  return success();
1325 
1326  if (!resultMappableTypes.empty() && operandMappableTypes.empty())
1327  return op->emitOpError("if a result is non-scalar, then at least one "
1328  "operand must be non-scalar");
1329 
1330  assert(!operandMappableTypes.empty());
1331 
1332  if (resultMappableTypes.empty())
1333  return op->emitOpError("if an operand is non-scalar, then there must be at "
1334  "least one non-scalar result");
1335 
1336  if (resultMappableTypes.size() != op->getNumResults())
1337  return op->emitOpError(
1338  "if an operand is non-scalar, then all results must be non-scalar");
1339 
1340  SmallVector<Type, 4> types = llvm::to_vector<2>(
1341  llvm::concat<Type>(operandMappableTypes, resultMappableTypes));
1342  TypeID expectedBaseTy = types.front().getTypeID();
1343  if (!llvm::all_of(types,
1344  [&](Type t) { return t.getTypeID() == expectedBaseTy; }) ||
1345  failed(verifyCompatibleShapes(types))) {
1346  return op->emitOpError() << "all non-scalar operands/results must have the "
1347  "same shape and base type";
1348  }
1349 
1350  return success();
1351 }
1352 
1353 /// Check for any values used by operations regions attached to the
1354 /// specified "IsIsolatedFromAbove" operation defined outside of it.
1356  assert(isolatedOp->hasTrait<OpTrait::IsIsolatedFromAbove>() &&
1357  "Intended to check IsolatedFromAbove ops");
1358 
1359  // List of regions to analyze. Each region is processed independently, with
1360  // respect to the common `limit` region, so we can look at them in any order.
1361  // Therefore, use a simple vector and push/pop back the current region.
1362  SmallVector<Region *, 8> pendingRegions;
1363  for (auto &region : isolatedOp->getRegions()) {
1364  pendingRegions.push_back(&region);
1365 
1366  // Traverse all operations in the region.
1367  while (!pendingRegions.empty()) {
1368  for (Operation &op : pendingRegions.pop_back_val()->getOps()) {
1369  for (Value operand : op.getOperands()) {
1370  // Check that any value that is used by an operation is defined in the
1371  // same region as either an operation result.
1372  auto *operandRegion = operand.getParentRegion();
1373  if (!operandRegion)
1374  return op.emitError("operation's operand is unlinked");
1375  if (!region.isAncestor(operandRegion)) {
1376  return op.emitOpError("using value defined outside the region")
1377  .attachNote(isolatedOp->getLoc())
1378  << "required by region isolation constraints";
1379  }
1380  }
1381 
1382  // Schedule any regions in the operation for further checking. Don't
1383  // recurse into other IsolatedFromAbove ops, because they will check
1384  // themselves.
1385  if (op.getNumRegions() &&
1386  !op.hasTrait<OpTrait::IsIsolatedFromAbove>()) {
1387  for (Region &subRegion : op.getRegions())
1388  pendingRegions.push_back(&subRegion);
1389  }
1390  }
1391  }
1392  }
1393 
1394  return success();
1395 }
1396 
1398  return op->hasTrait<Elementwise>() && op->hasTrait<Scalarizable>() &&
1399  op->hasTrait<Vectorizable>() && op->hasTrait<Tensorizable>();
1400 }
1401 
1402 //===----------------------------------------------------------------------===//
1403 // Misc. utils
1404 //===----------------------------------------------------------------------===//
1405 
1406 /// Insert an operation, generated by `buildTerminatorOp`, at the end of the
1407 /// region's only block if it does not have a terminator already. If the region
1408 /// is empty, insert a new block first. `buildTerminatorOp` should return the
1409 /// terminator operation to insert.
1411  Region &region, OpBuilder &builder, Location loc,
1412  function_ref<Operation *(OpBuilder &, Location)> buildTerminatorOp) {
1413  OpBuilder::InsertionGuard guard(builder);
1414  if (region.empty())
1415  builder.createBlock(&region);
1416 
1417  Block &block = region.back();
1418  if (!block.empty() && block.back().hasTrait<OpTrait::IsTerminator>())
1419  return;
1420 
1421  builder.setInsertionPointToEnd(&block);
1422  builder.insert(buildTerminatorOp(builder, loc));
1423 }
1424 
1425 /// Create a simple OpBuilder and forward to the OpBuilder version of this
1426 /// function.
1428  Region &region, Builder &builder, Location loc,
1429  function_ref<Operation *(OpBuilder &, Location)> buildTerminatorOp) {
1430  OpBuilder opBuilder(builder.getContext());
1431  ensureRegionTerminator(region, opBuilder, loc, buildTerminatorOp);
1432 }
static LogicalResult verifyTerminatorSuccessors(Operation *op)
Definition: Operation.cpp:1192
static Type getTensorOrVectorElementType(Type type)
If this is a vector type, or a tensor type, return the scalar element type that it is built around,...
Definition: Operation.cpp:930
static void checkFoldResultTypes(Operation *op, SmallVectorImpl< OpFoldResult > &results)
Assert that the folded results (in case of values) have the same type as the results of the given op.
Definition: Operation.cpp:614
static std::string diag(const llvm::Value &value)
static llvm::ManagedStatic< PassManagerOptions > options
virtual InFlightDiagnostic emitError(SMLoc loc, const Twine &message={})=0
Emit a diagnostic at the specified location and return failure.
virtual SMLoc getNameLoc() const =0
Return the location of the original name token.
virtual ParseResult parseOptionalLess()=0
Parse a '<' token if present.
virtual ParseResult parseGreater()=0
Parse a '>' token.
virtual ParseResult parseAttribute(Attribute &result, Type type={})=0
Parse an arbitrary attribute of a given type and return it in result.
virtual raw_ostream & getStream() const
Return the raw output stream used by this printer.
virtual void printNamedAttribute(NamedAttribute attr)
Print the given named attribute.
Attributes are known-constant values of operations.
Definition: Attributes.h:25
A block operand represents an operand that holds a reference to a Block, e.g.
Definition: BlockSupport.h:30
This class provides an abstraction over the different types of ranges over Blocks.
Definition: BlockSupport.h:106
Block represents an ordered list of Operations.
Definition: Block.h:33
void recomputeOpOrder()
Recomputes the ordering of child operations within the block.
Definition: Block.cpp:136
bool empty()
Definition: Block.h:148
Operation & back()
Definition: Block.h:152
bool isOpOrderValid()
Returns true if the ordering of the child operations is valid, false otherwise.
Definition: Block.cpp:104
void dropAllDefinedValueUses()
This drops all uses of values defined in this block or in the blocks of nested regions wherever the u...
Definition: Block.cpp:94
void invalidateOpOrder()
Invalidates the current ordering of operations.
Definition: Block.cpp:107
OpListType & getOperations()
Definition: Block.h:137
Operation & front()
Definition: Block.h:153
iterator end()
Definition: Block.h:144
static OpListType Block::* getSublistAccess(Operation *)
Returns pointer to member of operation list.
Definition: Block.h:390
This class is a general helper class for creating context-global objects like types,...
Definition: Builders.h:50
MLIRContext * getContext() const
Definition: Builders.h:55
Define a fold interface to allow for dialects to control specific aspects of the folding behavior for...
Dialects are groups of MLIR operations, types and attributes, as well as behavior associated with the...
Definition: Dialect.h:38
virtual std::optional< ParseOpHook > getParseOperationHook(StringRef opName) const
Return the hook to parse an operation registered to this dialect, if any.
Definition: Dialect.cpp:82
virtual llvm::unique_function< void(Operation *, OpAsmPrinter &printer)> getOperationPrinter(Operation *op) const
Print an operation registered to this dialect.
Definition: Dialect.cpp:87
This is a utility class for mapping one set of IR entities to another.
Definition: IRMapping.h:26
auto lookupOrDefault(T from) const
Lookup a mapped value within the map.
Definition: IRMapping.h:65
void map(Value from, Value to)
Inserts a new mapping for 'from' to 'to'.
Definition: IRMapping.h:30
This class represents a diagnostic that is inflight and set to be reported.
Definition: Diagnostics.h:314
This class defines the main interface for locations in MLIR and acts as a non-nullable wrapper around...
Definition: Location.h:76
MLIRContext * getContext() const
Return the context this location is uniqued in.
Definition: Location.h:86
NamedAttrList is array of NamedAttributes that tracks whether it is sorted and does some basic work t...
DictionaryAttr getDictionary(MLIRContext *context) const
Return a dictionary attribute for the underlying dictionary.
NamedAttribute represents a combination of a name and an Attribute value.
Definition: Attributes.h:164
StringAttr getName() const
Return the name of the attribute.
Definition: Attributes.cpp:55
The OpAsmParser has methods for interacting with the asm parser: parsing things from it,...
This is a pure-virtual base class that exposes the asmprinter hooks necessary to implement a custom p...
virtual void printGenericOp(Operation *op, bool printOpName=true)=0
Print the entire operation with the default generic assembly form.
RAII guard to reset the insertion point of the builder when destroyed.
Definition: Builders.h:346
This class helps build Operations.
Definition: Builders.h:205
Block * createBlock(Region *parent, Region::iterator insertPt={}, TypeRange argTypes={}, ArrayRef< Location > locs={})
Add new block with 'argTypes' arguments and set the insertion point to the end of it.
Definition: Builders.cpp:425
void setInsertionPointToEnd(Block *block)
Sets the insertion point to the end of the specified block.
Definition: Builders.h:434
Operation * insert(Operation *op)
Insert the given operation at the current insertion point and return it.
Definition: Builders.cpp:416
This class represents a single result from folding an operation.
Definition: OpDefinition.h:271
This class represents an operand of an operation.
Definition: Value.h:257
Set of flags used to control the behavior of the various IR print methods (e.g.
static void genericPrintProperties(OpAsmPrinter &p, Attribute properties, ArrayRef< StringRef > elidedProps={})
Print the properties as a Attribute with names not included within 'elidedProps'.
Definition: Operation.cpp:803
static void printOpName(Operation *op, OpAsmPrinter &p, StringRef defaultDialect)
Print an operation name, eliding the dialect prefix if necessary.
Definition: Operation.cpp:783
InFlightDiagnostic emitError(const Twine &message={})
Emit an error about fatal conditions with this operation, reporting up to any diagnostic handlers tha...
Definition: Operation.cpp:830
InFlightDiagnostic emitOpError(const Twine &message={})
Emit an error with the op name prefixed, like "'dim' op " which is convenient for verifiers.
Definition: Operation.cpp:836
InFlightDiagnostic emitWarning(const Twine &message={})
Emit a warning about this operation, reporting up to any diagnostic handlers that may be listening.
Definition: Operation.cpp:842
static ParseResult genericParseProperties(OpAsmParser &parser, Attribute &result)
Parse properties as a Attribute.
Definition: Operation.cpp:792
static ParseResult parse(OpAsmParser &parser, OperationState &result)
Parse the custom form of an operation.
Definition: Operation.cpp:763
InFlightDiagnostic emitRemark(const Twine &message={})
Emit a remark about this operation, reporting up to any diagnostic handlers that may be listening.
Definition: Operation.cpp:848
void print(raw_ostream &os, OpPrintingFlags flags=std::nullopt)
Print the operation to the given stream.
Definition: OpDefinition.h:117
This class provides the API for ops that are known to be isolated from above.
This class provides the API for ops that are known to be terminators.
Definition: OpDefinition.h:772
This class provides the API for ops that are known to have no SSA operand.
Definition: OpDefinition.h:446
Simple wrapper around a void* in order to express generically how to pass in op properties through AP...
void populateInherentAttrs(Operation *op, NamedAttrList &attrs) const
StringRef getStringRef() const
Return the name of this operation. This always succeeds.
void setInherentAttr(Operation *op, StringAttr name, Attribute value) const
bool hasTrait() const
Returns true if the operation was registered with a particular trait, e.g.
std::optional< Attribute > getInherentAttr(Operation *op, StringRef name) const
Lookup an inherent attribute by name, this method isn't recommended and may be removed in the future.
Dialect * getDialect() const
Return the dialect this operation is registered to if the dialect is loaded in the context,...
void initOpProperties(OpaqueProperties storage, OpaqueProperties init) const
Initialize the op properties.
llvm::hash_code hashOpProperties(OpaqueProperties properties) const
LogicalResult foldHook(Operation *op, ArrayRef< Attribute > operands, SmallVectorImpl< OpFoldResult > &results) const
This hook implements a generalized folder for this operation.
void populateDefaultAttrs(NamedAttrList &attrs) const
This hook implements the method to populate defaults attributes that are unset.
void destroyOpProperties(OpaqueProperties properties) const
This hooks destroy the op properties.
int getOpPropertyByteSize() const
This hooks return the number of bytes to allocate for the op properties.
void copyOpProperties(OpaqueProperties lhs, OpaqueProperties rhs) const
Class encompassing various options related to cloning an operation.
Definition: Operation.h:143
CloneOptions()
Default constructs an option with all flags set to false.
Definition: Operation.cpp:680
static CloneOptions all()
Returns an instance with all flags set to true.
Definition: Operation.cpp:686
CloneOptions & cloneRegions(bool enable=true)
Configures whether cloning should traverse into any of the regions of the operation.
Definition: Operation.cpp:690
CloneOptions & cloneOperands(bool enable=true)
Configures whether operation' operands should be cloned.
Definition: Operation.cpp:695
Operation is the basic unit of execution within MLIR.
Definition: Operation.h:88
void setInherentAttr(StringAttr name, Attribute value)
Set an inherent attribute by name.
Definition: Operation.cpp:344
void replaceUsesOfWith(Value from, Value to)
Replace any uses of 'from' with 'to' within this operation.
Definition: Operation.cpp:226
DictionaryAttr getAttrDictionary()
Return all of the attributes on this operation as a DictionaryAttr.
Definition: Operation.cpp:295
LogicalResult fold(ArrayRef< Attribute > operands, SmallVectorImpl< OpFoldResult > &results)
Attempt to fold this operation with the specified constant operand values.
Definition: Operation.cpp:633
bool use_empty()
Returns true if this operation has no uses.
Definition: Operation.h:852
Value getOperand(unsigned idx)
Definition: Operation.h:350
bool hasTrait()
Returns true if the operation was registered with a particular trait, e.g.
Definition: Operation.h:749
Operation * cloneWithoutRegions()
Create a partial copy of this operation without traversing into attached regions.
Definition: Operation.cpp:708
Dialect * getDialect()
Return the dialect this operation is associated with, or nullptr if the associated dialect is not loa...
Definition: Operation.h:220
void insertOperands(unsigned index, ValueRange operands)
Insert the given operands into the operand list at the given 'index'.
Definition: Operation.cpp:255
void dropAllUses()
Drop all uses of results of this operation.
Definition: Operation.h:834
AttrClass getAttrOfType(StringAttr name)
Definition: Operation.h:550
void setAttrs(DictionaryAttr newAttrs)
Set the attributes from a dictionary on this operation.
Definition: Operation.cpp:304
unsigned getNumSuccessors()
Definition: Operation.h:706
bool isBeforeInBlock(Operation *other)
Given an operation 'other' that is within the same parent block, return whether the current operation...
Definition: Operation.cpp:385
void dropAllReferences()
This drops all operand uses from this operation, which is an essential step in breaking cyclic depend...
Definition: Operation.cpp:585
InFlightDiagnostic emitWarning(const Twine &message={})
Emit a warning about this operation, reporting up to any diagnostic handlers that may be listening.
Definition: Operation.cpp:279
Operation * clone(IRMapping &mapper, CloneOptions options=CloneOptions::all())
Create a deep copy of this operation, remapping any operands that use values outside of the operation...
Definition: Operation.cpp:718
bool mightHaveTrait()
Returns true if the operation might have the provided trait.
Definition: Operation.h:757
OpResult getResult(unsigned idx)
Get the 'idx'th result of this operation.
Definition: Operation.h:407
std::optional< Attribute > getInherentAttr(StringRef name)
Access an inherent attribute by name: returns an empty optional if there is no inherent attribute wit...
Definition: Operation.cpp:340
MLIRContext * getContext()
Return the context this operation is associated with.
Definition: Operation.h:216
unsigned getNumRegions()
Returns the number of regions held by this operation.
Definition: Operation.h:674
std::optional< RegisteredOperationName > getRegisteredInfo()
If this operation has a registered operation description, return it.
Definition: Operation.h:123
Location getLoc()
The source location the operation was defined or derived from.
Definition: Operation.h:223
void dropAllDefinedValueUses()
Drop uses of all values defined by this operation or its nested regions.
Definition: Operation.cpp:598
unsigned getNumOperands()
Definition: Operation.h:346
static Operation * create(Location location, OperationName name, TypeRange resultTypes, ValueRange operands, NamedAttrList &&attributes, OpaqueProperties properties, BlockRange successors, unsigned numRegions)
Create a new Operation with the specific fields.
Definition: Operation.cpp:66
Attribute getPropertiesAsAttribute()
Return the properties converted to an attribute.
Definition: Operation.cpp:348
Operation * getParentOp()
Returns the closest surrounding operation that contains this operation or nullptr if this is a top-le...
Definition: Operation.h:234
InFlightDiagnostic emitError(const Twine &message={})
Emit an error about fatal conditions with this operation, reporting up to any diagnostic handlers tha...
Definition: Operation.cpp:267
Block * getBlock()
Returns the operation block that contains this operation.
Definition: Operation.h:213
Region & getRegion(unsigned index)
Returns the region held by this operation at position 'index'.
Definition: Operation.h:686
MutableArrayRef< Region > getRegions()
Returns the regions held by this operation.
Definition: Operation.h:677
void destroy()
Destroys this operation and its subclass data.
Definition: Operation.cpp:207
OperationName getName()
The name of an operation is the key identifier for it.
Definition: Operation.h:119
void remove()
Remove the operation from its parent block, but don't delete it.
Definition: Operation.cpp:546
LogicalResult setPropertiesFromAttribute(Attribute attr, function_ref< InFlightDiagnostic()> emitError)
Set the properties from the provided attribute.
Definition: Operation.cpp:354
MutableArrayRef< BlockOperand > getBlockOperands()
Definition: Operation.h:695
operand_type_range getOperandTypes()
Definition: Operation.h:397
MutableArrayRef< OpOperand > getOpOperands()
Definition: Operation.h:383
result_type_range getResultTypes()
Definition: Operation.h:428
operand_range getOperands()
Returns an iterator on the underlying Value's.
Definition: Operation.h:378
void setSuccessor(Block *block, unsigned index)
Definition: Operation.cpp:606
void moveBefore(Operation *existingOp)
Unlink this operation from its current block and insert it right before existingOp which may be in th...
Definition: Operation.cpp:554
void setOperands(ValueRange operands)
Replace the current operands of this operation with the ones provided in 'operands'.
Definition: Operation.cpp:236
user_range getUsers()
Returns a range of all users.
Definition: Operation.h:873
SuccessorRange getSuccessors()
Definition: Operation.h:703
Region * getParentRegion()
Returns the region to which the instruction belongs.
Definition: Operation.h:230
result_range getResults()
Definition: Operation.h:415
int getPropertiesStorageSize() const
Returns the properties storage size.
Definition: Operation.h:896
bool isProperAncestor(Operation *other)
Return true if this operation is a proper ancestor of the other operation.
Definition: Operation.cpp:218
InFlightDiagnostic emitRemark(const Twine &message={})
Emit a remark about this operation, reporting up to any diagnostic handlers that may be listening.
Definition: Operation.cpp:288
void moveAfter(Operation *existingOp)
Unlink this operation from its current block and insert it right after existingOp which may be in the...
Definition: Operation.cpp:570
llvm::hash_code hashProperties()
Compute a hash for the op properties (if any).
Definition: Operation.cpp:369
InFlightDiagnostic emitOpError(const Twine &message={})
Emit an error with the op name prefixed, like "'dim' op " which is convenient for verifiers.
Definition: Operation.cpp:672
OpaqueProperties getPropertiesStorage()
Returns the properties storage.
Definition: Operation.h:900
void erase()
Remove this operation from its parent block and delete it.
Definition: Operation.cpp:538
void copyProperties(OpaqueProperties rhs)
Copy properties from an existing other properties object.
Definition: Operation.cpp:365
unsigned getNumResults()
Return the number of results held by this operation.
Definition: Operation.h:404
This class provides an abstraction over the different types of ranges over Regions.
Definition: Region.h:346
This class contains a list of basic blocks and a link to the parent operation it is attached to.
Definition: Region.h:26
bool empty()
Definition: Region.h:60
void cloneInto(Region *dest, IRMapping &mapper)
Clone the internal blocks from this region into dest.
Definition: Region.cpp:70
Block & back()
Definition: Region.h:64
void takeBody(Region &other)
Takes body of another region (that region will have no body after this operation completes).
Definition: Region.h:241
This class provides an efficient unique identifier for a specific C++ type.
Definition: TypeID.h:107
This class provides an abstraction over the various different ranges of value types.
Definition: TypeRange.h:37
Instances of the Type class are uniqued, have an immutable identifier and an optional mutable compone...
Definition: Types.h:74
TypeID getTypeID()
Return a unique identifier for the concrete type.
Definition: Types.h:101
This class provides an abstraction over the different types of ranges over Values.
Definition: ValueRange.h:387
This class represents an instance of an SSA value in the MLIR system, representing a computable value...
Definition: Value.h:96
Type getType() const
Return the type of this value.
Definition: Value.h:105
Operation * getDefiningOp() const
If this value is the result of an operation, return the operation that defines it.
Definition: Value.cpp:20
This class handles the management of operation operands.
void setOperands(Operation *owner, ValueRange values)
Replace the operands contained in the storage with the ones provided in 'values'.
This class provides the implementation for an operation result whose index cannot be represented "inl...
Definition: Value.h:404
OpFoldResult foldIdempotent(Operation *op)
Definition: Operation.cpp:874
LogicalResult verifyResultsAreFloatLike(Operation *op)
Definition: Operation.cpp:1247
LogicalResult verifyAtLeastNResults(Operation *op, unsigned numOperands)
Definition: Operation.cpp:1033
LogicalResult verifyIsIdempotent(Operation *op)
Definition: Operation.cpp:940
LogicalResult verifyOperandsAreSignlessIntegerLike(Operation *op)
Definition: Operation.cpp:957
LogicalResult verifyNOperands(Operation *op, unsigned numOperands)
Definition: Operation.cpp:910
LogicalResult verifyNoRegionArguments(Operation *op)
Definition: Operation.cpp:1299
LogicalResult verifyResultsAreSignlessIntegerLike(Operation *op)
Definition: Operation.cpp:1256
LogicalResult verifyIsInvolution(Operation *op)
Definition: Operation.cpp:948
LogicalResult verifyOperandsAreFloatLike(Operation *op)
Definition: Operation.cpp:966
LogicalResult foldCommutative(Operation *op, ArrayRef< Attribute > operands, SmallVectorImpl< OpFoldResult > &results)
Definition: Operation.cpp:857
LogicalResult verifyZeroRegions(Operation *op)
Definition: Operation.cpp:988
LogicalResult verifyNSuccessors(Operation *op, unsigned numSuccessors)
Definition: Operation.cpp:1217
LogicalResult verifyOperandSizeAttr(Operation *op, StringRef sizeAttrName)
Definition: Operation.cpp:1289
LogicalResult verifyAtLeastNRegions(Operation *op, unsigned numRegions)
Definition: Operation.cpp:1007
LogicalResult verifyValueSizeAttr(Operation *op, StringRef attrName, StringRef valueGroupName, size_t expectedCount)
Definition: Operation.cpp:1263
LogicalResult verifyZeroResults(Operation *op)
Definition: Operation.cpp:1014
LogicalResult verifySameOperandsAndResultType(Operation *op)
Definition: Operation.cpp:1104
LogicalResult verifySameOperandsShape(Operation *op)
Definition: Operation.cpp:1041
LogicalResult verifyAtLeastNSuccessors(Operation *op, unsigned numSuccessors)
Definition: Operation.cpp:1226
LogicalResult verifyIsTerminator(Operation *op)
Definition: Operation.cpp:1184
LogicalResult verifyAtLeastNOperands(Operation *op, unsigned numOperands)
Definition: Operation.cpp:919
LogicalResult verifyZeroOperands(Operation *op)
Definition: Operation.cpp:898
LogicalResult verifyElementwise(Operation *op)
Definition: Operation.cpp:1314
LogicalResult verifyOneRegion(Operation *op)
Definition: Operation.cpp:994
LogicalResult verifySameOperandsAndResultRank(Operation *op)
Definition: Operation.cpp:1139
LogicalResult verifyOneOperand(Operation *op)
Definition: Operation.cpp:904
LogicalResult verifyIsIsolatedFromAbove(Operation *op)
Check for any values used by operations regions attached to the specified "IsIsolatedFromAbove" opera...
Definition: Operation.cpp:1355
LogicalResult verifyZeroSuccessors(Operation *op)
Definition: Operation.cpp:1202
LogicalResult verifySameOperandsElementType(Operation *op)
Definition: Operation.cpp:1066
LogicalResult verifyOneSuccessor(Operation *op)
Definition: Operation.cpp:1210
LogicalResult verifySameOperandsAndResultElementType(Operation *op)
Definition: Operation.cpp:1080
OpFoldResult foldInvolution(Operation *op)
Definition: Operation.cpp:888
LogicalResult verifyResultsAreBoolLike(Operation *op)
Definition: Operation.cpp:1236
LogicalResult verifyNResults(Operation *op, unsigned numOperands)
Definition: Operation.cpp:1026
LogicalResult verifyResultSizeAttr(Operation *op, StringRef sizeAttrName)
Definition: Operation.cpp:1294
LogicalResult verifyNRegions(Operation *op, unsigned numRegions)
Definition: Operation.cpp:1000
LogicalResult verifyOneResult(Operation *op)
Definition: Operation.cpp:1020
LogicalResult verifySameTypeOperands(Operation *op)
Definition: Operation.cpp:975
LogicalResult verifySameOperandsAndResultShape(Operation *op)
Definition: Operation.cpp:1051
bool hasElementwiseMappableTraits(Operation *op)
Together, Elementwise, Scalarizable, Vectorizable, and Tensorizable provide an easy way for scalar op...
Definition: Operation.cpp:1397
bool isMappableType(mlir::Type type)
Used to check whether the provided type implements the MappableType interface.
Definition: OpenACC.h:168
OpProperties
This is a "tag" used for mapping the properties storage in llvm::TrailingObjects.
Definition: Operation.h:28
void ensureRegionTerminator(Region &region, OpBuilder &builder, Location loc, function_ref< Operation *(OpBuilder &, Location)> buildTerminatorOp)
Insert an operation, generated by buildTerminatorOp, at the end of the region's only block if it does...
Definition: Operation.cpp:1410
Include the generated interface declarations.
bool matchPattern(Value value, const Pattern &pattern)
Entry point for matching a pattern over a Value.
Definition: Matchers.h:490
InFlightDiagnostic emitWarning(Location loc)
Utility method to emit a warning message using this location.
LogicalResult verifyCompatibleShapes(TypeRange types1, TypeRange types2)
Returns success if the given two arrays have the same number of elements and each pair wise entries h...
InFlightDiagnostic emitError(Location loc)
Utility method to emit an error message using this location.
Type getElementTypeOrSelf(Type type)
Return the element type or return the type itself.
InFlightDiagnostic emitRemark(Location loc)
Utility method to emit a remark message using this location.
auto get(MLIRContext *context, Ts &&...params)
Helper method that injects context only if needed, this helps unify some of the attribute constructio...
LogicalResult verifyCompatibleShape(ArrayRef< int64_t > shape1, ArrayRef< int64_t > shape2)
Returns success if the given two shapes are compatible.
detail::constant_op_matcher m_Constant()
Matches a constant foldable operation.
Definition: Matchers.h:369
void removeNodeFromList(Operation *op)
This is a trait method invoked when an operation is removed from a block.
Definition: Operation.cpp:512
void transferNodesFromList(ilist_traits< Operation > &otherList, op_iterator first, op_iterator last)
This is a trait method invoked when an operation is moved from one block to another.
Definition: Operation.cpp:519
void addNodeToList(Operation *op)
This is a trait method invoked when an operation is added to a block.
Definition: Operation.cpp:502
static void deleteNode(Operation *op)
Definition: Operation.cpp:490
simple_ilist< Operation >::iterator op_iterator
Definition: BlockSupport.h:228
This trait tags element-wise ops on vectors or tensors.
This trait tags Elementwise operatons that can be systematically scalarized.
This trait tags Elementwise operatons that can be systematically tensorized.
This trait tags Elementwise operatons that can be systematically vectorized.
This represents an operation in an abstracted form, suitable for use with the builder APIs.
This class provides the implementation for an operation result whose index can be represented "inline...
Definition: Value.h:387