MLIR  21.0.0git
Operation.cpp
Go to the documentation of this file.
1 //===- Operation.cpp - Operation support code -----------------------------===//
2 //
3 // Part of the LLVM Project, under the Apache License v2.0 with LLVM Exceptions.
4 // See https://llvm.org/LICENSE.txt for license information.
5 // SPDX-License-Identifier: Apache-2.0 WITH LLVM-exception
6 //
7 //===----------------------------------------------------------------------===//
8 
9 #include "mlir/IR/Operation.h"
10 #include "mlir/IR/Attributes.h"
12 #include "mlir/IR/BuiltinTypes.h"
13 #include "mlir/IR/Dialect.h"
14 #include "mlir/IR/IRMapping.h"
15 #include "mlir/IR/Matchers.h"
18 #include "mlir/IR/PatternMatch.h"
19 #include "mlir/IR/TypeUtilities.h"
21 #include "llvm/ADT/SmallVector.h"
22 #include "llvm/ADT/StringExtras.h"
23 #include "llvm/Support/ErrorHandling.h"
24 #include <numeric>
25 #include <optional>
26 
27 using namespace mlir;
28 
29 //===----------------------------------------------------------------------===//
30 // Operation
31 //===----------------------------------------------------------------------===//
32 
33 /// Create a new Operation from operation state.
35  Operation *op =
36  create(state.location, state.name, state.types, state.operands,
37  state.attributes.getDictionary(state.getContext()),
38  state.properties, state.successors, state.regions);
39  if (LLVM_UNLIKELY(state.propertiesAttr)) {
40  assert(!state.properties);
41  LogicalResult result =
42  op->setPropertiesFromAttribute(state.propertiesAttr,
43  /*diagnostic=*/nullptr);
44  assert(result.succeeded() && "invalid properties in op creation");
45  (void)result;
46  }
47  return op;
48 }
49 
50 /// Create a new Operation with the specific fields.
52  TypeRange resultTypes, ValueRange operands,
53  NamedAttrList &&attributes,
54  OpaqueProperties properties, BlockRange successors,
55  RegionRange regions) {
56  unsigned numRegions = regions.size();
57  Operation *op =
58  create(location, name, resultTypes, operands, std::move(attributes),
59  properties, successors, numRegions);
60  for (unsigned i = 0; i < numRegions; ++i)
61  if (regions[i])
62  op->getRegion(i).takeBody(*regions[i]);
63  return op;
64 }
65 
66 /// Create a new Operation with the specific fields.
68  TypeRange resultTypes, ValueRange operands,
69  NamedAttrList &&attributes,
70  OpaqueProperties properties, BlockRange successors,
71  unsigned numRegions) {
72  // Populate default attributes.
73  name.populateDefaultAttrs(attributes);
74 
75  return create(location, name, resultTypes, operands,
76  attributes.getDictionary(location.getContext()), properties,
77  successors, numRegions);
78 }
79 
80 /// Overload of create that takes an existing DictionaryAttr to avoid
81 /// unnecessarily uniquing a list of attributes.
83  TypeRange resultTypes, ValueRange operands,
84  DictionaryAttr attributes,
85  OpaqueProperties properties, BlockRange successors,
86  unsigned numRegions) {
87  assert(llvm::all_of(resultTypes, [](Type t) { return t; }) &&
88  "unexpected null result type");
89 
90  // We only need to allocate additional memory for a subset of results.
91  unsigned numTrailingResults = OpResult::getNumTrailing(resultTypes.size());
92  unsigned numInlineResults = OpResult::getNumInline(resultTypes.size());
93  unsigned numSuccessors = successors.size();
94  unsigned numOperands = operands.size();
95  unsigned numResults = resultTypes.size();
96  int opPropertiesAllocSize = llvm::alignTo<8>(name.getOpPropertyByteSize());
97 
98  // If the operation is known to have no operands, don't allocate an operand
99  // storage.
100  bool needsOperandStorage =
101  operands.empty() ? !name.hasTrait<OpTrait::ZeroOperands>() : true;
102 
103  // Compute the byte size for the operation and the operand storage. This takes
104  // into account the size of the operation, its trailing objects, and its
105  // prefixed objects.
106  size_t byteSize =
109  needsOperandStorage ? 1 : 0, opPropertiesAllocSize, numSuccessors,
110  numRegions, numOperands);
111  size_t prefixByteSize = llvm::alignTo(
112  Operation::prefixAllocSize(numTrailingResults, numInlineResults),
113  alignof(Operation));
114  char *mallocMem = reinterpret_cast<char *>(malloc(byteSize + prefixByteSize));
115  void *rawMem = mallocMem + prefixByteSize;
116 
117  // Create the new Operation.
118  Operation *op = ::new (rawMem) Operation(
119  location, name, numResults, numSuccessors, numRegions,
120  opPropertiesAllocSize, attributes, properties, needsOperandStorage);
121 
122  assert((numSuccessors == 0 || op->mightHaveTrait<OpTrait::IsTerminator>()) &&
123  "unexpected successors in a non-terminator operation");
124 
125  // Initialize the results.
126  auto resultTypeIt = resultTypes.begin();
127  for (unsigned i = 0; i < numInlineResults; ++i, ++resultTypeIt)
128  new (op->getInlineOpResult(i)) detail::InlineOpResult(*resultTypeIt, i);
129  for (unsigned i = 0; i < numTrailingResults; ++i, ++resultTypeIt) {
130  new (op->getOutOfLineOpResult(i))
131  detail::OutOfLineOpResult(*resultTypeIt, i);
132  }
133 
134  // Initialize the regions.
135  for (unsigned i = 0; i != numRegions; ++i)
136  new (&op->getRegion(i)) Region(op);
137 
138  // Initialize the operands.
139  if (needsOperandStorage) {
140  new (&op->getOperandStorage()) detail::OperandStorage(
141  op, op->getTrailingObjects<OpOperand>(), operands);
142  }
143 
144  // Initialize the successors.
145  auto blockOperands = op->getBlockOperands();
146  for (unsigned i = 0; i != numSuccessors; ++i)
147  new (&blockOperands[i]) BlockOperand(op, successors[i]);
148 
149  // This must be done after properties are initalized.
150  op->setAttrs(attributes);
151 
152  return op;
153 }
154 
155 Operation::Operation(Location location, OperationName name, unsigned numResults,
156  unsigned numSuccessors, unsigned numRegions,
157  int fullPropertiesStorageSize, DictionaryAttr attributes,
158  OpaqueProperties properties, bool hasOperandStorage)
159  : location(location), numResults(numResults), numSuccs(numSuccessors),
160  numRegions(numRegions), hasOperandStorage(hasOperandStorage),
161  propertiesStorageSize((fullPropertiesStorageSize + 7) / 8), name(name) {
162  assert(attributes && "unexpected null attribute dictionary");
163  assert(fullPropertiesStorageSize <= propertiesCapacity &&
164  "Properties size overflow");
165 #ifndef NDEBUG
166  if (!getDialect() && !getContext()->allowsUnregisteredDialects())
167  llvm::report_fatal_error(
168  name.getStringRef() +
169  " created with unregistered dialect. If this is intended, please call "
170  "allowUnregisteredDialects() on the MLIRContext, or use "
171  "-allow-unregistered-dialect with the MLIR tool used.");
172 #endif
173  if (fullPropertiesStorageSize)
174  name.initOpProperties(getPropertiesStorage(), properties);
175 }
176 
177 // Operations are deleted through the destroy() member because they are
178 // allocated via malloc.
179 Operation::~Operation() {
180  assert(block == nullptr && "operation destroyed but still in a block");
181 #ifndef NDEBUG
182  if (!use_empty()) {
183  {
185  emitOpError("operation destroyed but still has uses");
186  for (Operation *user : getUsers())
187  diag.attachNote(user->getLoc()) << "- use: " << *user << "\n";
188  }
189  llvm::report_fatal_error("operation destroyed but still has uses");
190  }
191 #endif
192  // Explicitly run the destructors for the operands.
193  if (hasOperandStorage)
194  getOperandStorage().~OperandStorage();
195 
196  // Explicitly run the destructors for the successors.
197  for (auto &successor : getBlockOperands())
198  successor.~BlockOperand();
199 
200  // Explicitly destroy the regions.
201  for (auto &region : getRegions())
202  region.~Region();
203  if (propertiesStorageSize)
205 }
206 
207 /// Destroy this operation or one of its subclasses.
209  // Operations may have additional prefixed allocation, which needs to be
210  // accounted for here when computing the address to free.
211  char *rawMem = reinterpret_cast<char *>(this) -
212  llvm::alignTo(prefixAllocSize(), alignof(Operation));
213  this->~Operation();
214  free(rawMem);
215 }
216 
217 /// Return true if this operation is a proper ancestor of the `other`
218 /// operation.
220  while ((other = other->getParentOp()))
221  if (this == other)
222  return true;
223  return false;
224 }
225 
226 /// Replace any uses of 'from' with 'to' within this operation.
228  if (from == to)
229  return;
230  for (auto &operand : getOpOperands())
231  if (operand.get() == from)
232  operand.set(to);
233 }
234 
235 /// Replace the current operands of this operation with the ones provided in
236 /// 'operands'.
238  if (LLVM_LIKELY(hasOperandStorage))
239  return getOperandStorage().setOperands(this, operands);
240  assert(operands.empty() && "setting operands without an operand storage");
241 }
242 
243 /// Replace the operands beginning at 'start' and ending at 'start' + 'length'
244 /// with the ones provided in 'operands'. 'operands' may be smaller or larger
245 /// than the range pointed to by 'start'+'length'.
246 void Operation::setOperands(unsigned start, unsigned length,
247  ValueRange operands) {
248  assert((start + length) <= getNumOperands() &&
249  "invalid operand range specified");
250  if (LLVM_LIKELY(hasOperandStorage))
251  return getOperandStorage().setOperands(this, start, length, operands);
252  assert(operands.empty() && "setting operands without an operand storage");
253 }
254 
255 /// Insert the given operands into the operand list at the given 'index'.
256 void Operation::insertOperands(unsigned index, ValueRange operands) {
257  if (LLVM_LIKELY(hasOperandStorage))
258  return setOperands(index, /*length=*/0, operands);
259  assert(operands.empty() && "inserting operands without an operand storage");
260 }
261 
262 //===----------------------------------------------------------------------===//
263 // Diagnostics
264 //===----------------------------------------------------------------------===//
265 
266 /// Emit an error about fatal conditions with this operation, reporting up to
267 /// any diagnostic handlers that may be listening.
270  if (getContext()->shouldPrintOpOnDiagnostic()) {
271  diag.attachNote(getLoc())
272  .append("see current operation: ")
273  .appendOp(*this, OpPrintingFlags().printGenericOpForm());
274  }
275  return diag;
276 }
277 
278 /// Emit a warning about this operation, reporting up to any diagnostic
279 /// handlers that may be listening.
282  if (getContext()->shouldPrintOpOnDiagnostic())
283  diag.attachNote(getLoc()) << "see current operation: " << *this;
284  return diag;
285 }
286 
287 /// Emit a remark about this operation, reporting up to any diagnostic
288 /// handlers that may be listening.
291  if (getContext()->shouldPrintOpOnDiagnostic())
292  diag.attachNote(getLoc()) << "see current operation: " << *this;
293  return diag;
294 }
295 
297  if (getPropertiesStorageSize()) {
298  NamedAttrList attrsList = attrs;
299  getName().populateInherentAttrs(this, attrsList);
300  return attrsList.getDictionary(getContext());
301  }
302  return attrs;
303 }
304 
305 void Operation::setAttrs(DictionaryAttr newAttrs) {
306  assert(newAttrs && "expected valid attribute dictionary");
307  if (getPropertiesStorageSize()) {
308  // We're spliting the providing DictionaryAttr by removing the inherentAttr
309  // which will be stored in the properties.
310  SmallVector<NamedAttribute> discardableAttrs;
311  discardableAttrs.reserve(newAttrs.size());
312  for (NamedAttribute attr : newAttrs) {
313  if (getInherentAttr(attr.getName()))
314  setInherentAttr(attr.getName(), attr.getValue());
315  else
316  discardableAttrs.push_back(attr);
317  }
318  if (discardableAttrs.size() != newAttrs.size())
319  newAttrs = DictionaryAttr::get(getContext(), discardableAttrs);
320  }
321  attrs = newAttrs;
322 }
324  if (getPropertiesStorageSize()) {
325  // We're spliting the providing array of attributes by removing the
326  // inherentAttr which will be stored in the properties.
327  SmallVector<NamedAttribute> discardableAttrs;
328  discardableAttrs.reserve(newAttrs.size());
329  for (NamedAttribute attr : newAttrs) {
330  if (getInherentAttr(attr.getName()))
331  setInherentAttr(attr.getName(), attr.getValue());
332  else
333  discardableAttrs.push_back(attr);
334  }
335  attrs = DictionaryAttr::get(getContext(), discardableAttrs);
336  return;
337  }
338  attrs = DictionaryAttr::get(getContext(), newAttrs);
339 }
340 
341 std::optional<Attribute> Operation::getInherentAttr(StringRef name) {
342  return getName().getInherentAttr(this, name);
343 }
344 
345 void Operation::setInherentAttr(StringAttr name, Attribute value) {
346  getName().setInherentAttr(this, name, value);
347 }
348 
350  std::optional<RegisteredOperationName> info = getRegisteredInfo();
351  if (LLVM_UNLIKELY(!info))
352  return *getPropertiesStorage().as<Attribute *>();
353  return info->getOpPropertiesAsAttribute(this);
354 }
357  std::optional<RegisteredOperationName> info = getRegisteredInfo();
358  if (LLVM_UNLIKELY(!info)) {
359  *getPropertiesStorage().as<Attribute *>() = attr;
360  return success();
361  }
362  return info->setOpPropertiesFromAttribute(
363  this->getName(), this->getPropertiesStorage(), attr, emitError);
364 }
365 
368 }
369 
370 llvm::hash_code Operation::hashProperties() {
371  return name.hashOpProperties(getPropertiesStorage());
372 }
373 
374 //===----------------------------------------------------------------------===//
375 // Operation Ordering
376 //===----------------------------------------------------------------------===//
377 
378 constexpr unsigned Operation::kInvalidOrderIdx;
379 constexpr unsigned Operation::kOrderStride;
380 
381 /// Given an operation 'other' that is within the same parent block, return
382 /// whether the current operation is before 'other' in the operation list
383 /// of the parent block.
384 /// Note: This function has an average complexity of O(1), but worst case may
385 /// take O(N) where N is the number of operations within the parent block.
387  assert(block && "Operations without parent blocks have no order.");
388  assert(other && other->block == block &&
389  "Expected other operation to have the same parent block.");
390  // If the order of the block is already invalid, directly recompute the
391  // parent.
392  if (!block->isOpOrderValid()) {
393  block->recomputeOpOrder();
394  } else {
395  // Update the order either operation if necessary.
396  updateOrderIfNecessary();
397  other->updateOrderIfNecessary();
398  }
399 
400  return orderIndex < other->orderIndex;
401 }
402 
403 /// Update the order index of this operation of this operation if necessary,
404 /// potentially recomputing the order of the parent block.
405 void Operation::updateOrderIfNecessary() {
406  assert(block && "expected valid parent");
407 
408  // If the order is valid for this operation there is nothing to do.
409  if (hasValidOrder() || llvm::hasSingleElement(*block))
410  return;
411  Operation *blockFront = &block->front();
412  Operation *blockBack = &block->back();
413 
414  // This method is expected to only be invoked on blocks with more than one
415  // operation.
416  assert(blockFront != blockBack && "expected more than one operation");
417 
418  // If the operation is at the end of the block.
419  if (this == blockBack) {
420  Operation *prevNode = getPrevNode();
421  if (!prevNode->hasValidOrder())
422  return block->recomputeOpOrder();
423 
424  // Add the stride to the previous operation.
425  orderIndex = prevNode->orderIndex + kOrderStride;
426  return;
427  }
428 
429  // If this is the first operation try to use the next operation to compute the
430  // ordering.
431  if (this == blockFront) {
432  Operation *nextNode = getNextNode();
433  if (!nextNode->hasValidOrder())
434  return block->recomputeOpOrder();
435  // There is no order to give this operation.
436  if (nextNode->orderIndex == 0)
437  return block->recomputeOpOrder();
438 
439  // If we can't use the stride, just take the middle value left. This is safe
440  // because we know there is at least one valid index to assign to.
441  if (nextNode->orderIndex <= kOrderStride)
442  orderIndex = (nextNode->orderIndex / 2);
443  else
444  orderIndex = kOrderStride;
445  return;
446  }
447 
448  // Otherwise, this operation is between two others. Place this operation in
449  // the middle of the previous and next if possible.
450  Operation *prevNode = getPrevNode(), *nextNode = getNextNode();
451  if (!prevNode->hasValidOrder() || !nextNode->hasValidOrder())
452  return block->recomputeOpOrder();
453  unsigned prevOrder = prevNode->orderIndex, nextOrder = nextNode->orderIndex;
454 
455  // Check to see if there is a valid order between the two.
456  if (prevOrder + 1 == nextOrder)
457  return block->recomputeOpOrder();
458  orderIndex = prevOrder + ((nextOrder - prevOrder) / 2);
459 }
460 
461 //===----------------------------------------------------------------------===//
462 // ilist_traits for Operation
463 //===----------------------------------------------------------------------===//
464 
465 auto llvm::ilist_detail::SpecificNodeAccess<
466  typename llvm::ilist_detail::compute_node_options<
467  ::mlir::Operation>::type>::getNodePtr(pointer n) -> node_type * {
468  return NodeAccess::getNodePtr<OptionsT>(n);
469 }
470 
471 auto llvm::ilist_detail::SpecificNodeAccess<
472  typename llvm::ilist_detail::compute_node_options<
473  ::mlir::Operation>::type>::getNodePtr(const_pointer n)
474  -> const node_type * {
475  return NodeAccess::getNodePtr<OptionsT>(n);
476 }
477 
478 auto llvm::ilist_detail::SpecificNodeAccess<
479  typename llvm::ilist_detail::compute_node_options<
480  ::mlir::Operation>::type>::getValuePtr(node_type *n) -> pointer {
481  return NodeAccess::getValuePtr<OptionsT>(n);
482 }
483 
484 auto llvm::ilist_detail::SpecificNodeAccess<
485  typename llvm::ilist_detail::compute_node_options<
486  ::mlir::Operation>::type>::getValuePtr(const node_type *n)
487  -> const_pointer {
488  return NodeAccess::getValuePtr<OptionsT>(n);
489 }
490 
492  op->destroy();
493 }
494 
495 Block *llvm::ilist_traits<::mlir::Operation>::getContainingBlock() {
496  size_t offset(size_t(&((Block *)nullptr->*Block::getSublistAccess(nullptr))));
497  iplist<Operation> *anchor(static_cast<iplist<Operation> *>(this));
498  return reinterpret_cast<Block *>(reinterpret_cast<char *>(anchor) - offset);
499 }
500 
501 /// This is a trait method invoked when an operation is added to a block. We
502 /// keep the block pointer up to date.
504  assert(!op->getBlock() && "already in an operation block!");
505  op->block = getContainingBlock();
506 
507  // Invalidate the order on the operation.
508  op->orderIndex = Operation::kInvalidOrderIdx;
509 }
510 
511 /// This is a trait method invoked when an operation is removed from a block.
512 /// We keep the block pointer up to date.
514  assert(op->block && "not already in an operation block!");
515  op->block = nullptr;
516 }
517 
518 /// This is a trait method invoked when an operation is moved from one block
519 /// to another. We keep the block pointer up to date.
521  ilist_traits<Operation> &otherList, op_iterator first, op_iterator last) {
522  Block *curParent = getContainingBlock();
523 
524  // Invalidate the ordering of the parent block.
525  curParent->invalidateOpOrder();
526 
527  // If we are transferring operations within the same block, the block
528  // pointer doesn't need to be updated.
529  if (curParent == otherList.getContainingBlock())
530  return;
531 
532  // Update the 'block' member of each operation.
533  for (; first != last; ++first)
534  first->block = curParent;
535 }
536 
537 /// Remove this operation (and its descendants) from its Block and delete
538 /// all of them.
540  if (auto *parent = getBlock())
541  parent->getOperations().erase(this);
542  else
543  destroy();
544 }
545 
546 /// Remove the operation from its parent block, but don't delete it.
548  if (Block *parent = getBlock())
549  parent->getOperations().remove(this);
550 }
551 
552 /// Unlink this operation from its current block and insert it right before
553 /// `existingOp` which may be in the same or another block in the same
554 /// function.
555 void Operation::moveBefore(Operation *existingOp) {
556  moveBefore(existingOp->getBlock(), existingOp->getIterator());
557 }
558 
559 /// Unlink this operation from its current basic block and insert it right
560 /// before `iterator` in the specified basic block.
562  llvm::iplist<Operation>::iterator iterator) {
563  assert(getBlock() &&
564  "cannot move an operation that isn't contained in a block");
565  block->getOperations().splice(iterator, getBlock()->getOperations(),
566  getIterator());
567 }
568 
569 /// Unlink this operation from its current block and insert it right after
570 /// `existingOp` which may be in the same or another block in the same function.
571 void Operation::moveAfter(Operation *existingOp) {
572  moveAfter(existingOp->getBlock(), existingOp->getIterator());
573 }
574 
575 /// Unlink this operation from its current block and insert it right after
576 /// `iterator` in the specified block.
578  llvm::iplist<Operation>::iterator iterator) {
579  assert(iterator != block->end() && "cannot move after end of block");
580  moveBefore(block, std::next(iterator));
581 }
582 
583 /// This drops all operand uses from this operation, which is an essential
584 /// step in breaking cyclic dependences between references when they are to
585 /// be deleted.
587  for (auto &op : getOpOperands())
588  op.drop();
589 
590  for (auto &region : getRegions())
591  region.dropAllReferences();
592 
593  for (auto &dest : getBlockOperands())
594  dest.drop();
595 }
596 
597 /// This drops all uses of any values defined by this operation or its nested
598 /// regions, wherever they are located.
600  dropAllUses();
601 
602  for (auto &region : getRegions())
603  for (auto &block : region)
604  block.dropAllDefinedValueUses();
605 }
606 
607 void Operation::setSuccessor(Block *block, unsigned index) {
608  assert(index < getNumSuccessors());
609  getBlockOperands()[index].set(block);
610 }
611 
612 #ifndef NDEBUG
613 /// Assert that the folded results (in case of values) have the same type as
614 /// the results of the given op.
617  if (results.empty())
618  return;
619 
620  for (auto [ofr, opResult] : llvm::zip_equal(results, op->getResults())) {
621  if (auto value = dyn_cast<Value>(ofr)) {
622  if (value.getType() != opResult.getType()) {
623  op->emitOpError() << "folder produced a value of incorrect type: "
624  << value.getType()
625  << ", expected: " << opResult.getType();
626  assert(false && "incorrect fold result type");
627  }
628  }
629  }
630 }
631 #endif // NDEBUG
632 
633 /// Attempt to fold this operation using the Op's registered foldHook.
634 LogicalResult Operation::fold(ArrayRef<Attribute> operands,
636  // If we have a registered operation definition matching this one, use it to
637  // try to constant fold the operation.
638  if (succeeded(name.foldHook(this, operands, results))) {
639 #ifndef NDEBUG
640  checkFoldResultTypes(this, results);
641 #endif // NDEBUG
642  return success();
643  }
644 
645  // Otherwise, fall back on the dialect hook to handle it.
646  Dialect *dialect = getDialect();
647  if (!dialect)
648  return failure();
649 
650  auto *interface = dyn_cast<DialectFoldInterface>(dialect);
651  if (!interface)
652  return failure();
653 
654  LogicalResult status = interface->fold(this, operands, results);
655 #ifndef NDEBUG
656  if (succeeded(status))
657  checkFoldResultTypes(this, results);
658 #endif // NDEBUG
659  return status;
660 }
661 
663  // Check if any operands are constants.
664  SmallVector<Attribute> constants;
665  constants.assign(getNumOperands(), Attribute());
666  for (unsigned i = 0, e = getNumOperands(); i != e; ++i)
667  matchPattern(getOperand(i), m_Constant(&constants[i]));
668  return fold(constants, results);
669 }
670 
671 /// Emit an error with the op name prefixed, like "'dim' op " which is
672 /// convenient for verifiers.
674  return emitError() << "'" << getName() << "' op " << message;
675 }
676 
677 //===----------------------------------------------------------------------===//
678 // Operation Cloning
679 //===----------------------------------------------------------------------===//
680 
682  : cloneRegionsFlag(false), cloneOperandsFlag(false) {}
683 
684 Operation::CloneOptions::CloneOptions(bool cloneRegions, bool cloneOperands)
685  : cloneRegionsFlag(cloneRegions), cloneOperandsFlag(cloneOperands) {}
686 
689 }
690 
692  cloneRegionsFlag = enable;
693  return *this;
694 }
695 
697  cloneOperandsFlag = enable;
698  return *this;
699 }
700 
701 /// Create a deep copy of this operation but keep the operation regions empty.
702 /// Operands are remapped using `mapper` (if present), and `mapper` is updated
703 /// to contain the results. The `mapResults` flag specifies whether the results
704 /// of the cloned operation should be added to the map.
706  return clone(mapper, CloneOptions::all().cloneRegions(false));
707 }
708 
710  IRMapping mapper;
711  return cloneWithoutRegions(mapper);
712 }
713 
714 /// Create a deep copy of this operation, remapping any operands that use
715 /// values outside of the operation using the map that is provided (leaving
716 /// them alone if no entry is present). Replaces references to cloned
717 /// sub-operations to the corresponding operation that is copied, and adds
718 /// those mappings to the map.
720  SmallVector<Value, 8> operands;
721  SmallVector<Block *, 2> successors;
722 
723  // Remap the operands.
724  if (options.shouldCloneOperands()) {
725  operands.reserve(getNumOperands());
726  for (auto opValue : getOperands())
727  operands.push_back(mapper.lookupOrDefault(opValue));
728  }
729 
730  // Remap the successors.
731  successors.reserve(getNumSuccessors());
732  for (Block *successor : getSuccessors())
733  successors.push_back(mapper.lookupOrDefault(successor));
734 
735  // Create the new operation.
736  auto *newOp = create(getLoc(), getName(), getResultTypes(), operands, attrs,
737  getPropertiesStorage(), successors, getNumRegions());
738  mapper.map(this, newOp);
739 
740  // Clone the regions.
741  if (options.shouldCloneRegions()) {
742  for (unsigned i = 0; i != numRegions; ++i)
743  getRegion(i).cloneInto(&newOp->getRegion(i), mapper);
744  }
745 
746  // Remember the mapping of any results.
747  for (unsigned i = 0, e = getNumResults(); i != e; ++i)
748  mapper.map(getResult(i), newOp->getResult(i));
749 
750  return newOp;
751 }
752 
754  IRMapping mapper;
755  return clone(mapper, options);
756 }
757 
758 //===----------------------------------------------------------------------===//
759 // OpState trait class.
760 //===----------------------------------------------------------------------===//
761 
762 // The fallback for the parser is to try for a dialect operation parser.
763 // Otherwise, reject the custom assembly form.
764 ParseResult OpState::parse(OpAsmParser &parser, OperationState &result) {
765  if (auto parseFn = result.name.getDialect()->getParseOperationHook(
766  result.name.getStringRef()))
767  return (*parseFn)(parser, result);
768  return parser.emitError(parser.getNameLoc(), "has no custom assembly form");
769 }
770 
771 // The fallback for the printer is to try for a dialect operation printer.
772 // Otherwise, it prints the generic form.
773 void OpState::print(Operation *op, OpAsmPrinter &p, StringRef defaultDialect) {
774  if (auto printFn = op->getDialect()->getOperationPrinter(op)) {
775  printOpName(op, p, defaultDialect);
776  printFn(op, p);
777  } else {
778  p.printGenericOp(op);
779  }
780 }
781 
782 /// Print an operation name, eliding the dialect prefix if necessary and doesn't
783 /// lead to ambiguities.
785  StringRef defaultDialect) {
786  StringRef name = op->getName().getStringRef();
787  if (name.starts_with((defaultDialect + ".").str()) && name.count('.') == 1)
788  name = name.drop_front(defaultDialect.size() + 1);
789  p.getStream() << name;
790 }
791 
792 /// Parse properties as a Attribute.
794  Attribute &result) {
795  if (succeeded(parser.parseOptionalLess())) { // The less is optional.
796  if (parser.parseAttribute(result) || parser.parseGreater())
797  return failure();
798  }
799  return success();
800 }
801 
802 /// Print the properties as a Attribute with names not included within
803 /// 'elidedProps'
805  ArrayRef<StringRef> elidedProps) {
806  if (!properties)
807  return;
808  auto dictAttr = dyn_cast_or_null<::mlir::DictionaryAttr>(properties);
809  if (dictAttr && !elidedProps.empty()) {
810  ArrayRef<NamedAttribute> attrs = dictAttr.getValue();
811  llvm::SmallDenseSet<StringRef> elidedAttrsSet(elidedProps.begin(),
812  elidedProps.end());
813  bool atLeastOneAttr = llvm::any_of(attrs, [&](NamedAttribute attr) {
814  return !elidedAttrsSet.contains(attr.getName().strref());
815  });
816  if (atLeastOneAttr) {
817  p << "<";
818  p.printOptionalAttrDict(dictAttr.getValue(), elidedProps);
819  p << ">";
820  }
821  } else {
822  p << "<" << properties << ">";
823  }
824 }
825 
826 /// Emit an error about fatal conditions with this operation, reporting up to
827 /// any diagnostic handlers that may be listening.
828 InFlightDiagnostic OpState::emitError(const Twine &message) {
829  return getOperation()->emitError(message);
830 }
831 
832 /// Emit an error with the op name prefixed, like "'dim' op " which is
833 /// convenient for verifiers.
835  return getOperation()->emitOpError(message);
836 }
837 
838 /// Emit a warning about this operation, reporting up to any diagnostic
839 /// handlers that may be listening.
841  return getOperation()->emitWarning(message);
842 }
843 
844 /// Emit a remark about this operation, reporting up to any diagnostic
845 /// handlers that may be listening.
847  return getOperation()->emitRemark(message);
848 }
849 
850 //===----------------------------------------------------------------------===//
851 // Op Trait implementations
852 //===----------------------------------------------------------------------===//
853 
854 LogicalResult
857  // Nothing to fold if there are not at least 2 operands.
858  if (op->getNumOperands() < 2)
859  return failure();
860  // Move all constant operands to the end.
861  OpOperand *operandsBegin = op->getOpOperands().begin();
862  auto isNonConstant = [&](OpOperand &o) {
863  return !static_cast<bool>(operands[std::distance(operandsBegin, &o)]);
864  };
865  auto *firstConstantIt = llvm::find_if_not(op->getOpOperands(), isNonConstant);
866  auto *newConstantIt = std::stable_partition(
867  firstConstantIt, op->getOpOperands().end(), isNonConstant);
868  // Return success if the op was modified.
869  return success(firstConstantIt != newConstantIt);
870 }
871 
873  if (op->getNumOperands() == 1) {
874  auto *argumentOp = op->getOperand(0).getDefiningOp();
875  if (argumentOp && op->getName() == argumentOp->getName()) {
876  // Replace the outer operation output with the inner operation.
877  return op->getOperand(0);
878  }
879  } else if (op->getOperand(0) == op->getOperand(1)) {
880  return op->getOperand(0);
881  }
882 
883  return {};
884 }
885 
887  auto *argumentOp = op->getOperand(0).getDefiningOp();
888  if (argumentOp && op->getName() == argumentOp->getName()) {
889  // Replace the outer involutions output with inner's input.
890  return argumentOp->getOperand(0);
891  }
892 
893  return {};
894 }
895 
897  if (op->getNumOperands() != 0)
898  return op->emitOpError() << "requires zero operands";
899  return success();
900 }
901 
903  if (op->getNumOperands() != 1)
904  return op->emitOpError() << "requires a single operand";
905  return success();
906 }
907 
909  unsigned numOperands) {
910  if (op->getNumOperands() != numOperands) {
911  return op->emitOpError() << "expected " << numOperands
912  << " operands, but found " << op->getNumOperands();
913  }
914  return success();
915 }
916 
918  unsigned numOperands) {
919  if (op->getNumOperands() < numOperands)
920  return op->emitOpError()
921  << "expected " << numOperands << " or more operands, but found "
922  << op->getNumOperands();
923  return success();
924 }
925 
926 /// If this is a vector type, or a tensor type, return the scalar element type
927 /// that it is built around, otherwise return the type unmodified.
929  if (auto vec = llvm::dyn_cast<VectorType>(type))
930  return vec.getElementType();
931 
932  // Look through tensor<vector<...>> to find the underlying element type.
933  if (auto tensor = llvm::dyn_cast<TensorType>(type))
934  return getTensorOrVectorElementType(tensor.getElementType());
935  return type;
936 }
937 
939  // FIXME: Add back check for no side effects on operation.
940  // Currently adding it would cause the shared library build
941  // to fail since there would be a dependency of IR on SideEffectInterfaces
942  // which is cyclical.
943  return success();
944 }
945 
947  // FIXME: Add back check for no side effects on operation.
948  // Currently adding it would cause the shared library build
949  // to fail since there would be a dependency of IR on SideEffectInterfaces
950  // which is cyclical.
951  return success();
952 }
953 
954 LogicalResult
956  for (auto opType : op->getOperandTypes()) {
957  auto type = getTensorOrVectorElementType(opType);
958  if (!type.isSignlessIntOrIndex())
959  return op->emitOpError() << "requires an integer or index type";
960  }
961  return success();
962 }
963 
965  for (auto opType : op->getOperandTypes()) {
966  auto type = getTensorOrVectorElementType(opType);
967  if (!llvm::isa<FloatType>(type))
968  return op->emitOpError("requires a float type");
969  }
970  return success();
971 }
972 
974  // Zero or one operand always have the "same" type.
975  unsigned nOperands = op->getNumOperands();
976  if (nOperands < 2)
977  return success();
978 
979  auto type = op->getOperand(0).getType();
980  for (auto opType : llvm::drop_begin(op->getOperandTypes(), 1))
981  if (opType != type)
982  return op->emitOpError() << "requires all operands to have the same type";
983  return success();
984 }
985 
987  if (op->getNumRegions() != 0)
988  return op->emitOpError() << "requires zero regions";
989  return success();
990 }
991 
993  if (op->getNumRegions() != 1)
994  return op->emitOpError() << "requires one region";
995  return success();
996 }
997 
999  unsigned numRegions) {
1000  if (op->getNumRegions() != numRegions)
1001  return op->emitOpError() << "expected " << numRegions << " regions";
1002  return success();
1003 }
1004 
1006  unsigned numRegions) {
1007  if (op->getNumRegions() < numRegions)
1008  return op->emitOpError() << "expected " << numRegions << " or more regions";
1009  return success();
1010 }
1011 
1013  if (op->getNumResults() != 0)
1014  return op->emitOpError() << "requires zero results";
1015  return success();
1016 }
1017 
1019  if (op->getNumResults() != 1)
1020  return op->emitOpError() << "requires one result";
1021  return success();
1022 }
1023 
1025  unsigned numOperands) {
1026  if (op->getNumResults() != numOperands)
1027  return op->emitOpError() << "expected " << numOperands << " results";
1028  return success();
1029 }
1030 
1032  unsigned numOperands) {
1033  if (op->getNumResults() < numOperands)
1034  return op->emitOpError()
1035  << "expected " << numOperands << " or more results";
1036  return success();
1037 }
1038 
1040  if (failed(verifyAtLeastNOperands(op, 1)))
1041  return failure();
1042 
1043  if (failed(verifyCompatibleShapes(op->getOperandTypes())))
1044  return op->emitOpError() << "requires the same shape for all operands";
1045 
1046  return success();
1047 }
1048 
1050  if (failed(verifyAtLeastNOperands(op, 1)) ||
1051  failed(verifyAtLeastNResults(op, 1)))
1052  return failure();
1053 
1055  types.append(llvm::to_vector<4>(op->getResultTypes()));
1056 
1057  if (failed(verifyCompatibleShapes(types)))
1058  return op->emitOpError()
1059  << "requires the same shape for all operands and results";
1060 
1061  return success();
1062 }
1063 
1065  if (failed(verifyAtLeastNOperands(op, 1)))
1066  return failure();
1067  auto elementType = getElementTypeOrSelf(op->getOperand(0));
1068 
1069  for (auto operand : llvm::drop_begin(op->getOperands(), 1)) {
1070  if (getElementTypeOrSelf(operand) != elementType)
1071  return op->emitOpError("requires the same element type for all operands");
1072  }
1073 
1074  return success();
1075 }
1076 
1077 LogicalResult
1079  if (failed(verifyAtLeastNOperands(op, 1)) ||
1080  failed(verifyAtLeastNResults(op, 1)))
1081  return failure();
1082 
1083  auto elementType = getElementTypeOrSelf(op->getResult(0));
1084 
1085  // Verify result element type matches first result's element type.
1086  for (auto result : llvm::drop_begin(op->getResults(), 1)) {
1087  if (getElementTypeOrSelf(result) != elementType)
1088  return op->emitOpError(
1089  "requires the same element type for all operands and results");
1090  }
1091 
1092  // Verify operand's element type matches first result's element type.
1093  for (auto operand : op->getOperands()) {
1094  if (getElementTypeOrSelf(operand) != elementType)
1095  return op->emitOpError(
1096  "requires the same element type for all operands and results");
1097  }
1098 
1099  return success();
1100 }
1101 
1103  if (failed(verifyAtLeastNOperands(op, 1)) ||
1104  failed(verifyAtLeastNResults(op, 1)))
1105  return failure();
1106 
1107  auto type = op->getResult(0).getType();
1108  auto elementType = getElementTypeOrSelf(type);
1109  Attribute encoding = nullptr;
1110  if (auto rankedType = dyn_cast<RankedTensorType>(type))
1111  encoding = rankedType.getEncoding();
1112  for (auto resultType : llvm::drop_begin(op->getResultTypes())) {
1113  if (getElementTypeOrSelf(resultType) != elementType ||
1114  failed(verifyCompatibleShape(resultType, type)))
1115  return op->emitOpError()
1116  << "requires the same type for all operands and results";
1117  if (encoding)
1118  if (auto rankedType = dyn_cast<RankedTensorType>(resultType);
1119  encoding != rankedType.getEncoding())
1120  return op->emitOpError()
1121  << "requires the same encoding for all operands and results";
1122  }
1123  for (auto opType : op->getOperandTypes()) {
1124  if (getElementTypeOrSelf(opType) != elementType ||
1125  failed(verifyCompatibleShape(opType, type)))
1126  return op->emitOpError()
1127  << "requires the same type for all operands and results";
1128  if (encoding)
1129  if (auto rankedType = dyn_cast<RankedTensorType>(opType);
1130  encoding != rankedType.getEncoding())
1131  return op->emitOpError()
1132  << "requires the same encoding for all operands and results";
1133  }
1134  return success();
1135 }
1136 
1138  if (failed(verifyAtLeastNOperands(op, 1)))
1139  return failure();
1140 
1141  // delegate function that returns true if type is a shaped type with known
1142  // rank
1143  auto hasRank = [](const Type type) {
1144  if (auto shapedType = dyn_cast<ShapedType>(type))
1145  return shapedType.hasRank();
1146 
1147  return false;
1148  };
1149 
1150  auto rankedOperandTypes =
1151  llvm::make_filter_range(op->getOperandTypes(), hasRank);
1152  auto rankedResultTypes =
1153  llvm::make_filter_range(op->getResultTypes(), hasRank);
1154 
1155  // If all operands and results are unranked, then no further verification.
1156  if (rankedOperandTypes.empty() && rankedResultTypes.empty())
1157  return success();
1158 
1159  // delegate function that returns rank of shaped type with known rank
1160  auto getRank = [](const Type type) {
1161  return cast<ShapedType>(type).getRank();
1162  };
1163 
1164  auto rank = !rankedOperandTypes.empty() ? getRank(*rankedOperandTypes.begin())
1165  : getRank(*rankedResultTypes.begin());
1166 
1167  for (const auto type : rankedOperandTypes) {
1168  if (rank != getRank(type)) {
1169  return op->emitOpError("operands don't have matching ranks");
1170  }
1171  }
1172 
1173  for (const auto type : rankedResultTypes) {
1174  if (rank != getRank(type)) {
1175  return op->emitOpError("result type has different rank than operands");
1176  }
1177  }
1178 
1179  return success();
1180 }
1181 
1183  Block *block = op->getBlock();
1184  // Verify that the operation is at the end of the respective parent block.
1185  if (!block || &block->back() != op)
1186  return op->emitOpError("must be the last operation in the parent block");
1187  return success();
1188 }
1189 
1190 static LogicalResult verifyTerminatorSuccessors(Operation *op) {
1191  auto *parent = op->getParentRegion();
1192 
1193  // Verify that the operands lines up with the BB arguments in the successor.
1194  for (Block *succ : op->getSuccessors())
1195  if (succ->getParent() != parent)
1196  return op->emitError("reference to block defined in another region");
1197  return success();
1198 }
1199 
1201  if (op->getNumSuccessors() != 0) {
1202  return op->emitOpError("requires 0 successors but found ")
1203  << op->getNumSuccessors();
1204  }
1205  return success();
1206 }
1207 
1209  if (op->getNumSuccessors() != 1) {
1210  return op->emitOpError("requires 1 successor but found ")
1211  << op->getNumSuccessors();
1212  }
1213  return verifyTerminatorSuccessors(op);
1214 }
1216  unsigned numSuccessors) {
1217  if (op->getNumSuccessors() != numSuccessors) {
1218  return op->emitOpError("requires ")
1219  << numSuccessors << " successors but found "
1220  << op->getNumSuccessors();
1221  }
1222  return verifyTerminatorSuccessors(op);
1223 }
1225  unsigned numSuccessors) {
1226  if (op->getNumSuccessors() < numSuccessors) {
1227  return op->emitOpError("requires at least ")
1228  << numSuccessors << " successors but found "
1229  << op->getNumSuccessors();
1230  }
1231  return verifyTerminatorSuccessors(op);
1232 }
1233 
1235  for (auto resultType : op->getResultTypes()) {
1236  auto elementType = getTensorOrVectorElementType(resultType);
1237  bool isBoolType = elementType.isInteger(1);
1238  if (!isBoolType)
1239  return op->emitOpError() << "requires a bool result type";
1240  }
1241 
1242  return success();
1243 }
1244 
1246  for (auto resultType : op->getResultTypes())
1247  if (!llvm::isa<FloatType>(getTensorOrVectorElementType(resultType)))
1248  return op->emitOpError() << "requires a floating point type";
1249 
1250  return success();
1251 }
1252 
1253 LogicalResult
1255  for (auto resultType : op->getResultTypes())
1256  if (!getTensorOrVectorElementType(resultType).isSignlessIntOrIndex())
1257  return op->emitOpError() << "requires an integer or index type";
1258  return success();
1259 }
1260 
1262  StringRef attrName,
1263  StringRef valueGroupName,
1264  size_t expectedCount) {
1265  auto sizeAttr = op->getAttrOfType<DenseI32ArrayAttr>(attrName);
1266  if (!sizeAttr)
1267  return op->emitOpError("requires dense i32 array attribute '")
1268  << attrName << "'";
1269 
1270  ArrayRef<int32_t> sizes = sizeAttr.asArrayRef();
1271  if (llvm::any_of(sizes, [](int32_t element) { return element < 0; }))
1272  return op->emitOpError("'")
1273  << attrName << "' attribute cannot have negative elements";
1274 
1275  size_t totalCount =
1276  std::accumulate(sizes.begin(), sizes.end(), 0,
1277  [](unsigned all, int32_t one) { return all + one; });
1278 
1279  if (totalCount != expectedCount)
1280  return op->emitOpError()
1281  << valueGroupName << " count (" << expectedCount
1282  << ") does not match with the total size (" << totalCount
1283  << ") specified in attribute '" << attrName << "'";
1284  return success();
1285 }
1286 
1288  StringRef attrName) {
1289  return verifyValueSizeAttr(op, attrName, "operand", op->getNumOperands());
1290 }
1291 
1293  StringRef attrName) {
1294  return verifyValueSizeAttr(op, attrName, "result", op->getNumResults());
1295 }
1296 
1298  for (Region &region : op->getRegions()) {
1299  if (region.empty())
1300  continue;
1301 
1302  if (region.getNumArguments() != 0) {
1303  if (op->getNumRegions() > 1)
1304  return op->emitOpError("region #")
1305  << region.getRegionNumber() << " should have no arguments";
1306  return op->emitOpError("region should have no arguments");
1307  }
1308  }
1309  return success();
1310 }
1311 
1313  auto isMappableType = llvm::IsaPred<VectorType, TensorType>;
1314  auto resultMappableTypes =
1315  llvm::filter_to_vector<1>(op->getResultTypes(), isMappableType);
1316  auto operandMappableTypes =
1317  llvm::filter_to_vector<2>(op->getOperandTypes(), isMappableType);
1318 
1319  // If the op only has scalar operand/result types, then we have nothing to
1320  // check.
1321  if (resultMappableTypes.empty() && operandMappableTypes.empty())
1322  return success();
1323 
1324  if (!resultMappableTypes.empty() && operandMappableTypes.empty())
1325  return op->emitOpError("if a result is non-scalar, then at least one "
1326  "operand must be non-scalar");
1327 
1328  assert(!operandMappableTypes.empty());
1329 
1330  if (resultMappableTypes.empty())
1331  return op->emitOpError("if an operand is non-scalar, then there must be at "
1332  "least one non-scalar result");
1333 
1334  if (resultMappableTypes.size() != op->getNumResults())
1335  return op->emitOpError(
1336  "if an operand is non-scalar, then all results must be non-scalar");
1337 
1338  SmallVector<Type, 4> types = llvm::to_vector<2>(
1339  llvm::concat<Type>(operandMappableTypes, resultMappableTypes));
1340  TypeID expectedBaseTy = types.front().getTypeID();
1341  if (!llvm::all_of(types,
1342  [&](Type t) { return t.getTypeID() == expectedBaseTy; }) ||
1343  failed(verifyCompatibleShapes(types))) {
1344  return op->emitOpError() << "all non-scalar operands/results must have the "
1345  "same shape and base type";
1346  }
1347 
1348  return success();
1349 }
1350 
1351 /// Check for any values used by operations regions attached to the
1352 /// specified "IsIsolatedFromAbove" operation defined outside of it.
1354  assert(isolatedOp->hasTrait<OpTrait::IsIsolatedFromAbove>() &&
1355  "Intended to check IsolatedFromAbove ops");
1356 
1357  // List of regions to analyze. Each region is processed independently, with
1358  // respect to the common `limit` region, so we can look at them in any order.
1359  // Therefore, use a simple vector and push/pop back the current region.
1360  SmallVector<Region *, 8> pendingRegions;
1361  for (auto &region : isolatedOp->getRegions()) {
1362  pendingRegions.push_back(&region);
1363 
1364  // Traverse all operations in the region.
1365  while (!pendingRegions.empty()) {
1366  for (Operation &op : pendingRegions.pop_back_val()->getOps()) {
1367  for (Value operand : op.getOperands()) {
1368  // Check that any value that is used by an operation is defined in the
1369  // same region as either an operation result.
1370  auto *operandRegion = operand.getParentRegion();
1371  if (!operandRegion)
1372  return op.emitError("operation's operand is unlinked");
1373  if (!region.isAncestor(operandRegion)) {
1374  return op.emitOpError("using value defined outside the region")
1375  .attachNote(isolatedOp->getLoc())
1376  << "required by region isolation constraints";
1377  }
1378  }
1379 
1380  // Schedule any regions in the operation for further checking. Don't
1381  // recurse into other IsolatedFromAbove ops, because they will check
1382  // themselves.
1383  if (op.getNumRegions() &&
1384  !op.hasTrait<OpTrait::IsIsolatedFromAbove>()) {
1385  for (Region &subRegion : op.getRegions())
1386  pendingRegions.push_back(&subRegion);
1387  }
1388  }
1389  }
1390  }
1391 
1392  return success();
1393 }
1394 
1396  return op->hasTrait<Elementwise>() && op->hasTrait<Scalarizable>() &&
1397  op->hasTrait<Vectorizable>() && op->hasTrait<Tensorizable>();
1398 }
1399 
1400 //===----------------------------------------------------------------------===//
1401 // Misc. utils
1402 //===----------------------------------------------------------------------===//
1403 
1404 /// Insert an operation, generated by `buildTerminatorOp`, at the end of the
1405 /// region's only block if it does not have a terminator already. If the region
1406 /// is empty, insert a new block first. `buildTerminatorOp` should return the
1407 /// terminator operation to insert.
1409  Region &region, OpBuilder &builder, Location loc,
1410  function_ref<Operation *(OpBuilder &, Location)> buildTerminatorOp) {
1411  OpBuilder::InsertionGuard guard(builder);
1412  if (region.empty())
1413  builder.createBlock(&region);
1414 
1415  Block &block = region.back();
1416  if (!block.empty() && block.back().hasTrait<OpTrait::IsTerminator>())
1417  return;
1418 
1419  builder.setInsertionPointToEnd(&block);
1420  builder.insert(buildTerminatorOp(builder, loc));
1421 }
1422 
1423 /// Create a simple OpBuilder and forward to the OpBuilder version of this
1424 /// function.
1426  Region &region, Builder &builder, Location loc,
1427  function_ref<Operation *(OpBuilder &, Location)> buildTerminatorOp) {
1428  OpBuilder opBuilder(builder.getContext());
1429  ensureRegionTerminator(region, opBuilder, loc, buildTerminatorOp);
1430 }
static LogicalResult verifyTerminatorSuccessors(Operation *op)
Definition: Operation.cpp:1190
static Type getTensorOrVectorElementType(Type type)
If this is a vector type, or a tensor type, return the scalar element type that it is built around,...
Definition: Operation.cpp:928
static void checkFoldResultTypes(Operation *op, SmallVectorImpl< OpFoldResult > &results)
Assert that the folded results (in case of values) have the same type as the results of the given op.
Definition: Operation.cpp:615
static std::string diag(const llvm::Value &value)
static llvm::ManagedStatic< PassManagerOptions > options
virtual InFlightDiagnostic emitError(SMLoc loc, const Twine &message={})=0
Emit a diagnostic at the specified location and return failure.
virtual SMLoc getNameLoc() const =0
Return the location of the original name token.
virtual ParseResult parseOptionalLess()=0
Parse a '<' token if present.
virtual ParseResult parseGreater()=0
Parse a '>' token.
virtual ParseResult parseAttribute(Attribute &result, Type type={})=0
Parse an arbitrary attribute of a given type and return it in result.
virtual raw_ostream & getStream() const
Return the raw output stream used by this printer.
Attributes are known-constant values of operations.
Definition: Attributes.h:25
A block operand represents an operand that holds a reference to a Block, e.g.
Definition: BlockSupport.h:30
This class provides an abstraction over the different types of ranges over Blocks.
Definition: BlockSupport.h:106
Block represents an ordered list of Operations.
Definition: Block.h:33
void recomputeOpOrder()
Recomputes the ordering of child operations within the block.
Definition: Block.cpp:138
bool empty()
Definition: Block.h:148
Operation & back()
Definition: Block.h:152
bool isOpOrderValid()
Returns true if the ordering of the child operations is valid, false otherwise.
Definition: Block.cpp:106
void dropAllDefinedValueUses()
This drops all uses of values defined in this block or in the blocks of nested regions wherever the u...
Definition: Block.cpp:96
void invalidateOpOrder()
Invalidates the current ordering of operations.
Definition: Block.cpp:109
OpListType & getOperations()
Definition: Block.h:137
Operation & front()
Definition: Block.h:153
iterator end()
Definition: Block.h:144
static OpListType Block::* getSublistAccess(Operation *)
Returns pointer to member of operation list.
Definition: Block.h:390
This class is a general helper class for creating context-global objects like types,...
Definition: Builders.h:51
MLIRContext * getContext() const
Definition: Builders.h:56
Define a fold interface to allow for dialects to control specific aspects of the folding behavior for...
Dialects are groups of MLIR operations, types and attributes, as well as behavior associated with the...
Definition: Dialect.h:38
virtual std::optional< ParseOpHook > getParseOperationHook(StringRef opName) const
Return the hook to parse an operation registered to this dialect, if any.
Definition: Dialect.cpp:85
virtual llvm::unique_function< void(Operation *, OpAsmPrinter &printer)> getOperationPrinter(Operation *op) const
Print an operation registered to this dialect.
Definition: Dialect.cpp:90
This is a utility class for mapping one set of IR entities to another.
Definition: IRMapping.h:26
auto lookupOrDefault(T from) const
Lookup a mapped value within the map.
Definition: IRMapping.h:65
void map(Value from, Value to)
Inserts a new mapping for 'from' to 'to'.
Definition: IRMapping.h:30
This class represents a diagnostic that is inflight and set to be reported.
Definition: Diagnostics.h:314
This class defines the main interface for locations in MLIR and acts as a non-nullable wrapper around...
Definition: Location.h:66
MLIRContext * getContext() const
Return the context this location is uniqued in.
Definition: Location.h:76
NamedAttrList is array of NamedAttributes that tracks whether it is sorted and does some basic work t...
DictionaryAttr getDictionary(MLIRContext *context) const
Return a dictionary attribute for the underlying dictionary.
NamedAttribute represents a combination of a name and an Attribute value.
Definition: Attributes.h:164
StringAttr getName() const
Return the name of the attribute.
Definition: Attributes.cpp:55
The OpAsmParser has methods for interacting with the asm parser: parsing things from it,...
This is a pure-virtual base class that exposes the asmprinter hooks necessary to implement a custom p...
virtual void printOptionalAttrDict(ArrayRef< NamedAttribute > attrs, ArrayRef< StringRef > elidedAttrs={})=0
If the specified operation has attributes, print out an attribute dictionary with their values.
virtual void printGenericOp(Operation *op, bool printOpName=true)=0
Print the entire operation with the default generic assembly form.
RAII guard to reset the insertion point of the builder when destroyed.
Definition: Builders.h:346
This class helps build Operations.
Definition: Builders.h:205
void setInsertionPointToEnd(Block *block)
Sets the insertion point to the end of the specified block.
Definition: Builders.h:434
Block * createBlock(Region *parent, Region::iterator insertPt={}, TypeRange argTypes=std::nullopt, ArrayRef< Location > locs=std::nullopt)
Add new block with 'argTypes' arguments and set the insertion point to the end of it.
Definition: Builders.cpp:426
Operation * insert(Operation *op)
Insert the given operation at the current insertion point and return it.
Definition: Builders.cpp:417
This class represents a single result from folding an operation.
Definition: OpDefinition.h:271
This class represents an operand of an operation.
Definition: Value.h:243
Set of flags used to control the behavior of the various IR print methods (e.g.
static void genericPrintProperties(OpAsmPrinter &p, Attribute properties, ArrayRef< StringRef > elidedProps={})
Print the properties as a Attribute with names not included within 'elidedProps'.
Definition: Operation.cpp:804
static void printOpName(Operation *op, OpAsmPrinter &p, StringRef defaultDialect)
Print an operation name, eliding the dialect prefix if necessary.
Definition: Operation.cpp:784
InFlightDiagnostic emitError(const Twine &message={})
Emit an error about fatal conditions with this operation, reporting up to any diagnostic handlers tha...
Definition: Operation.cpp:828
InFlightDiagnostic emitOpError(const Twine &message={})
Emit an error with the op name prefixed, like "'dim' op " which is convenient for verifiers.
Definition: Operation.cpp:834
InFlightDiagnostic emitWarning(const Twine &message={})
Emit a warning about this operation, reporting up to any diagnostic handlers that may be listening.
Definition: Operation.cpp:840
static ParseResult genericParseProperties(OpAsmParser &parser, Attribute &result)
Parse properties as a Attribute.
Definition: Operation.cpp:793
static ParseResult parse(OpAsmParser &parser, OperationState &result)
Parse the custom form of an operation.
Definition: Operation.cpp:764
InFlightDiagnostic emitRemark(const Twine &message={})
Emit a remark about this operation, reporting up to any diagnostic handlers that may be listening.
Definition: Operation.cpp:846
void print(raw_ostream &os, OpPrintingFlags flags=std::nullopt)
Print the operation to the given stream.
Definition: OpDefinition.h:117
This class provides the API for ops that are known to be isolated from above.
This class provides the API for ops that are known to be terminators.
Definition: OpDefinition.h:772
This class provides the API for ops that are known to have no SSA operand.
Definition: OpDefinition.h:446
Simple wrapper around a void* in order to express generically how to pass in op properties through AP...
void populateInherentAttrs(Operation *op, NamedAttrList &attrs) const
StringRef getStringRef() const
Return the name of this operation. This always succeeds.
void setInherentAttr(Operation *op, StringAttr name, Attribute value) const
bool hasTrait() const
Returns true if the operation was registered with a particular trait, e.g.
std::optional< Attribute > getInherentAttr(Operation *op, StringRef name) const
Lookup an inherent attribute by name, this method isn't recommended and may be removed in the future.
Dialect * getDialect() const
Return the dialect this operation is registered to if the dialect is loaded in the context,...
void initOpProperties(OpaqueProperties storage, OpaqueProperties init) const
Initialize the op properties.
llvm::hash_code hashOpProperties(OpaqueProperties properties) const
LogicalResult foldHook(Operation *op, ArrayRef< Attribute > operands, SmallVectorImpl< OpFoldResult > &results) const
This hook implements a generalized folder for this operation.
void populateDefaultAttrs(NamedAttrList &attrs) const
This hook implements the method to populate defaults attributes that are unset.
void destroyOpProperties(OpaqueProperties properties) const
This hooks destroy the op properties.
int getOpPropertyByteSize() const
This hooks return the number of bytes to allocate for the op properties.
void copyOpProperties(OpaqueProperties lhs, OpaqueProperties rhs) const
Class encompassing various options related to cloning an operation.
Definition: Operation.h:143
CloneOptions()
Default constructs an option with all flags set to false.
Definition: Operation.cpp:681
static CloneOptions all()
Returns an instance with all flags set to true.
Definition: Operation.cpp:687
CloneOptions & cloneRegions(bool enable=true)
Configures whether cloning should traverse into any of the regions of the operation.
Definition: Operation.cpp:691
CloneOptions & cloneOperands(bool enable=true)
Configures whether operation' operands should be cloned.
Definition: Operation.cpp:696
Operation is the basic unit of execution within MLIR.
Definition: Operation.h:88
void setInherentAttr(StringAttr name, Attribute value)
Set an inherent attribute by name.
Definition: Operation.cpp:345
void replaceUsesOfWith(Value from, Value to)
Replace any uses of 'from' with 'to' within this operation.
Definition: Operation.cpp:227
DictionaryAttr getAttrDictionary()
Return all of the attributes on this operation as a DictionaryAttr.
Definition: Operation.cpp:296
LogicalResult fold(ArrayRef< Attribute > operands, SmallVectorImpl< OpFoldResult > &results)
Attempt to fold this operation with the specified constant operand values.
Definition: Operation.cpp:634
bool use_empty()
Returns true if this operation has no uses.
Definition: Operation.h:853
Value getOperand(unsigned idx)
Definition: Operation.h:350
bool hasTrait()
Returns true if the operation was registered with a particular trait, e.g.
Definition: Operation.h:750
Operation * cloneWithoutRegions()
Create a partial copy of this operation without traversing into attached regions.
Definition: Operation.cpp:709
Dialect * getDialect()
Return the dialect this operation is associated with, or nullptr if the associated dialect is not loa...
Definition: Operation.h:220
void insertOperands(unsigned index, ValueRange operands)
Insert the given operands into the operand list at the given 'index'.
Definition: Operation.cpp:256
void dropAllUses()
Drop all uses of results of this operation.
Definition: Operation.h:835
AttrClass getAttrOfType(StringAttr name)
Definition: Operation.h:550
void setAttrs(DictionaryAttr newAttrs)
Set the attributes from a dictionary on this operation.
Definition: Operation.cpp:305
unsigned getNumSuccessors()
Definition: Operation.h:707
bool isBeforeInBlock(Operation *other)
Given an operation 'other' that is within the same parent block, return whether the current operation...
Definition: Operation.cpp:386
void dropAllReferences()
This drops all operand uses from this operation, which is an essential step in breaking cyclic depend...
Definition: Operation.cpp:586
InFlightDiagnostic emitWarning(const Twine &message={})
Emit a warning about this operation, reporting up to any diagnostic handlers that may be listening.
Definition: Operation.cpp:280
Operation * clone(IRMapping &mapper, CloneOptions options=CloneOptions::all())
Create a deep copy of this operation, remapping any operands that use values outside of the operation...
Definition: Operation.cpp:719
bool mightHaveTrait()
Returns true if the operation might have the provided trait.
Definition: Operation.h:758
OpResult getResult(unsigned idx)
Get the 'idx'th result of this operation.
Definition: Operation.h:407
std::optional< Attribute > getInherentAttr(StringRef name)
Access an inherent attribute by name: returns an empty optional if there is no inherent attribute wit...
Definition: Operation.cpp:341
MLIRContext * getContext()
Return the context this operation is associated with.
Definition: Operation.h:216
unsigned getNumRegions()
Returns the number of regions held by this operation.
Definition: Operation.h:674
std::optional< RegisteredOperationName > getRegisteredInfo()
If this operation has a registered operation description, return it.
Definition: Operation.h:123
Location getLoc()
The source location the operation was defined or derived from.
Definition: Operation.h:223
void dropAllDefinedValueUses()
Drop uses of all values defined by this operation or its nested regions.
Definition: Operation.cpp:599
unsigned getNumOperands()
Definition: Operation.h:346
static Operation * create(Location location, OperationName name, TypeRange resultTypes, ValueRange operands, NamedAttrList &&attributes, OpaqueProperties properties, BlockRange successors, unsigned numRegions)
Create a new Operation with the specific fields.
Definition: Operation.cpp:67
Attribute getPropertiesAsAttribute()
Return the properties converted to an attribute.
Definition: Operation.cpp:349
Operation * getParentOp()
Returns the closest surrounding operation that contains this operation or nullptr if this is a top-le...
Definition: Operation.h:234
InFlightDiagnostic emitError(const Twine &message={})
Emit an error about fatal conditions with this operation, reporting up to any diagnostic handlers tha...
Definition: Operation.cpp:268
Block * getBlock()
Returns the operation block that contains this operation.
Definition: Operation.h:213
Region & getRegion(unsigned index)
Returns the region held by this operation at position 'index'.
Definition: Operation.h:687
MutableArrayRef< Region > getRegions()
Returns the regions held by this operation.
Definition: Operation.h:677
void destroy()
Destroys this operation and its subclass data.
Definition: Operation.cpp:208
OperationName getName()
The name of an operation is the key identifier for it.
Definition: Operation.h:119
void remove()
Remove the operation from its parent block, but don't delete it.
Definition: Operation.cpp:547
LogicalResult setPropertiesFromAttribute(Attribute attr, function_ref< InFlightDiagnostic()> emitError)
Set the properties from the provided attribute.
Definition: Operation.cpp:355
MutableArrayRef< BlockOperand > getBlockOperands()
Definition: Operation.h:696
operand_type_range getOperandTypes()
Definition: Operation.h:397
MutableArrayRef< OpOperand > getOpOperands()
Definition: Operation.h:383
result_type_range getResultTypes()
Definition: Operation.h:428
operand_range getOperands()
Returns an iterator on the underlying Value's.
Definition: Operation.h:378
void setSuccessor(Block *block, unsigned index)
Definition: Operation.cpp:607
void moveBefore(Operation *existingOp)
Unlink this operation from its current block and insert it right before existingOp which may be in th...
Definition: Operation.cpp:555
void setOperands(ValueRange operands)
Replace the current operands of this operation with the ones provided in 'operands'.
Definition: Operation.cpp:237
user_range getUsers()
Returns a range of all users.
Definition: Operation.h:874
SuccessorRange getSuccessors()
Definition: Operation.h:704
Region * getParentRegion()
Returns the region to which the instruction belongs.
Definition: Operation.h:230
result_range getResults()
Definition: Operation.h:415
int getPropertiesStorageSize() const
Returns the properties storage size.
Definition: Operation.h:897
bool isProperAncestor(Operation *other)
Return true if this operation is a proper ancestor of the other operation.
Definition: Operation.cpp:219
InFlightDiagnostic emitRemark(const Twine &message={})
Emit a remark about this operation, reporting up to any diagnostic handlers that may be listening.
Definition: Operation.cpp:289
void moveAfter(Operation *existingOp)
Unlink this operation from its current block and insert it right after existingOp which may be in the...
Definition: Operation.cpp:571
llvm::hash_code hashProperties()
Compute a hash for the op properties (if any).
Definition: Operation.cpp:370
InFlightDiagnostic emitOpError(const Twine &message={})
Emit an error with the op name prefixed, like "'dim' op " which is convenient for verifiers.
Definition: Operation.cpp:673
OpaqueProperties getPropertiesStorage()
Returns the properties storage.
Definition: Operation.h:901
void erase()
Remove this operation from its parent block and delete it.
Definition: Operation.cpp:539
void copyProperties(OpaqueProperties rhs)
Copy properties from an existing other properties object.
Definition: Operation.cpp:366
unsigned getNumResults()
Return the number of results held by this operation.
Definition: Operation.h:404
This class provides an abstraction over the different types of ranges over Regions.
Definition: Region.h:346
This class contains a list of basic blocks and a link to the parent operation it is attached to.
Definition: Region.h:26
bool empty()
Definition: Region.h:60
void cloneInto(Region *dest, IRMapping &mapper)
Clone the internal blocks from this region into dest.
Definition: Region.cpp:70
Block & back()
Definition: Region.h:64
void takeBody(Region &other)
Takes body of another region (that region will have no body after this operation completes).
Definition: Region.h:241
This class provides an efficient unique identifier for a specific C++ type.
Definition: TypeID.h:107
This class provides an abstraction over the various different ranges of value types.
Definition: TypeRange.h:37
Instances of the Type class are uniqued, have an immutable identifier and an optional mutable compone...
Definition: Types.h:74
TypeID getTypeID()
Return a unique identifier for the concrete type.
Definition: Types.h:101
This class provides an abstraction over the different types of ranges over Values.
Definition: ValueRange.h:387
This class represents an instance of an SSA value in the MLIR system, representing a computable value...
Definition: Value.h:96
Type getType() const
Return the type of this value.
Definition: Value.h:105
Operation * getDefiningOp() const
If this value is the result of an operation, return the operation that defines it.
Definition: Value.cpp:20
This class handles the management of operation operands.
void setOperands(Operation *owner, ValueRange values)
Replace the operands contained in the storage with the ones provided in 'values'.
This class provides the implementation for an operation result whose index cannot be represented "inl...
Definition: Value.h:390
OpFoldResult foldIdempotent(Operation *op)
Definition: Operation.cpp:872
LogicalResult verifyResultsAreFloatLike(Operation *op)
Definition: Operation.cpp:1245
LogicalResult verifyAtLeastNResults(Operation *op, unsigned numOperands)
Definition: Operation.cpp:1031
LogicalResult verifyIsIdempotent(Operation *op)
Definition: Operation.cpp:938
LogicalResult verifyOperandsAreSignlessIntegerLike(Operation *op)
Definition: Operation.cpp:955
LogicalResult verifyNOperands(Operation *op, unsigned numOperands)
Definition: Operation.cpp:908
LogicalResult verifyNoRegionArguments(Operation *op)
Definition: Operation.cpp:1297
LogicalResult verifyResultsAreSignlessIntegerLike(Operation *op)
Definition: Operation.cpp:1254
LogicalResult verifyIsInvolution(Operation *op)
Definition: Operation.cpp:946
LogicalResult verifyOperandsAreFloatLike(Operation *op)
Definition: Operation.cpp:964
LogicalResult foldCommutative(Operation *op, ArrayRef< Attribute > operands, SmallVectorImpl< OpFoldResult > &results)
Definition: Operation.cpp:855
LogicalResult verifyZeroRegions(Operation *op)
Definition: Operation.cpp:986
LogicalResult verifyNSuccessors(Operation *op, unsigned numSuccessors)
Definition: Operation.cpp:1215
LogicalResult verifyOperandSizeAttr(Operation *op, StringRef sizeAttrName)
Definition: Operation.cpp:1287
LogicalResult verifyAtLeastNRegions(Operation *op, unsigned numRegions)
Definition: Operation.cpp:1005
LogicalResult verifyValueSizeAttr(Operation *op, StringRef attrName, StringRef valueGroupName, size_t expectedCount)
Definition: Operation.cpp:1261
LogicalResult verifyZeroResults(Operation *op)
Definition: Operation.cpp:1012
LogicalResult verifySameOperandsAndResultType(Operation *op)
Definition: Operation.cpp:1102
LogicalResult verifySameOperandsShape(Operation *op)
Definition: Operation.cpp:1039
LogicalResult verifyAtLeastNSuccessors(Operation *op, unsigned numSuccessors)
Definition: Operation.cpp:1224
LogicalResult verifyIsTerminator(Operation *op)
Definition: Operation.cpp:1182
LogicalResult verifyAtLeastNOperands(Operation *op, unsigned numOperands)
Definition: Operation.cpp:917
LogicalResult verifyZeroOperands(Operation *op)
Definition: Operation.cpp:896
LogicalResult verifyElementwise(Operation *op)
Definition: Operation.cpp:1312
LogicalResult verifyOneRegion(Operation *op)
Definition: Operation.cpp:992
LogicalResult verifySameOperandsAndResultRank(Operation *op)
Definition: Operation.cpp:1137
LogicalResult verifyOneOperand(Operation *op)
Definition: Operation.cpp:902
LogicalResult verifyIsIsolatedFromAbove(Operation *op)
Check for any values used by operations regions attached to the specified "IsIsolatedFromAbove" opera...
Definition: Operation.cpp:1353
LogicalResult verifyZeroSuccessors(Operation *op)
Definition: Operation.cpp:1200
LogicalResult verifySameOperandsElementType(Operation *op)
Definition: Operation.cpp:1064
LogicalResult verifyOneSuccessor(Operation *op)
Definition: Operation.cpp:1208
LogicalResult verifySameOperandsAndResultElementType(Operation *op)
Definition: Operation.cpp:1078
OpFoldResult foldInvolution(Operation *op)
Definition: Operation.cpp:886
LogicalResult verifyResultsAreBoolLike(Operation *op)
Definition: Operation.cpp:1234
LogicalResult verifyNResults(Operation *op, unsigned numOperands)
Definition: Operation.cpp:1024
LogicalResult verifyResultSizeAttr(Operation *op, StringRef sizeAttrName)
Definition: Operation.cpp:1292
LogicalResult verifyNRegions(Operation *op, unsigned numRegions)
Definition: Operation.cpp:998
LogicalResult verifyOneResult(Operation *op)
Definition: Operation.cpp:1018
LogicalResult verifySameTypeOperands(Operation *op)
Definition: Operation.cpp:973
LogicalResult verifySameOperandsAndResultShape(Operation *op)
Definition: Operation.cpp:1049
bool hasElementwiseMappableTraits(Operation *op)
Together, Elementwise, Scalarizable, Vectorizable, and Tensorizable provide an easy way for scalar op...
Definition: Operation.cpp:1395
bool isMappableType(mlir::Type type)
Used to check whether the provided type implements the MappableType interface.
Definition: OpenACC.h:162
OpProperties
This is a "tag" used for mapping the properties storage in llvm::TrailingObjects.
Definition: Operation.h:28
void ensureRegionTerminator(Region &region, OpBuilder &builder, Location loc, function_ref< Operation *(OpBuilder &, Location)> buildTerminatorOp)
Insert an operation, generated by buildTerminatorOp, at the end of the region's only block if it does...
Definition: Operation.cpp:1408
Include the generated interface declarations.
bool matchPattern(Value value, const Pattern &pattern)
Entry point for matching a pattern over a Value.
Definition: Matchers.h:490
InFlightDiagnostic emitWarning(Location loc)
Utility method to emit a warning message using this location.
LogicalResult verifyCompatibleShapes(TypeRange types1, TypeRange types2)
Returns success if the given two arrays have the same number of elements and each pair wise entries h...
InFlightDiagnostic emitError(Location loc)
Utility method to emit an error message using this location.
Type getElementTypeOrSelf(Type type)
Return the element type or return the type itself.
InFlightDiagnostic emitRemark(Location loc)
Utility method to emit a remark message using this location.
auto get(MLIRContext *context, Ts &&...params)
Helper method that injects context only if needed, this helps unify some of the attribute constructio...
LogicalResult verifyCompatibleShape(ArrayRef< int64_t > shape1, ArrayRef< int64_t > shape2)
Returns success if the given two shapes are compatible.
detail::constant_op_matcher m_Constant()
Matches a constant foldable operation.
Definition: Matchers.h:369
void removeNodeFromList(Operation *op)
This is a trait method invoked when an operation is removed from a block.
Definition: Operation.cpp:513
void transferNodesFromList(ilist_traits< Operation > &otherList, op_iterator first, op_iterator last)
This is a trait method invoked when an operation is moved from one block to another.
Definition: Operation.cpp:520
void addNodeToList(Operation *op)
This is a trait method invoked when an operation is added to a block.
Definition: Operation.cpp:503
static void deleteNode(Operation *op)
Definition: Operation.cpp:491
simple_ilist< Operation >::iterator op_iterator
Definition: BlockSupport.h:228
This trait tags element-wise ops on vectors or tensors.
This trait tags Elementwise operatons that can be systematically scalarized.
This trait tags Elementwise operatons that can be systematically tensorized.
This trait tags Elementwise operatons that can be systematically vectorized.
This represents an operation in an abstracted form, suitable for use with the builder APIs.
This class provides the implementation for an operation result whose index can be represented "inline...
Definition: Value.h:373