MLIR  17.0.0git
PatternMatch.h
Go to the documentation of this file.
1 //===- PatternMatch.h - PatternMatcher classes -------==---------*- C++ -*-===//
2 //
3 // Part of the LLVM Project, under the Apache License v2.0 with LLVM Exceptions.
4 // See https://llvm.org/LICENSE.txt for license information.
5 // SPDX-License-Identifier: Apache-2.0 WITH LLVM-exception
6 //
7 //===----------------------------------------------------------------------===//
8 
9 #ifndef MLIR_IR_PATTERNMATCH_H
10 #define MLIR_IR_PATTERNMATCH_H
11 
12 #include "mlir/IR/Builders.h"
13 #include "mlir/IR/BuiltinOps.h"
14 #include "llvm/ADT/FunctionExtras.h"
15 #include "llvm/Support/TypeName.h"
16 #include <optional>
17 
18 namespace mlir {
19 
20 class PatternRewriter;
21 
22 //===----------------------------------------------------------------------===//
23 // PatternBenefit class
24 //===----------------------------------------------------------------------===//
25 
26 /// This class represents the benefit of a pattern match in a unitless scheme
27 /// that ranges from 0 (very little benefit) to 65K. The most common unit to
28 /// use here is the "number of operations matched" by the pattern.
29 ///
30 /// This also has a sentinel representation that can be used for patterns that
31 /// fail to match.
32 ///
34  enum { ImpossibleToMatchSentinel = 65535 };
35 
36 public:
37  PatternBenefit() = default;
38  PatternBenefit(unsigned benefit);
39  PatternBenefit(const PatternBenefit &) = default;
41 
43  bool isImpossibleToMatch() const { return *this == impossibleToMatch(); }
44 
45  /// If the corresponding pattern can match, return its benefit. If the
46  // corresponding pattern isImpossibleToMatch() then this aborts.
47  unsigned short getBenefit() const;
48 
49  bool operator==(const PatternBenefit &rhs) const {
50  return representation == rhs.representation;
51  }
52  bool operator!=(const PatternBenefit &rhs) const { return !(*this == rhs); }
53  bool operator<(const PatternBenefit &rhs) const {
54  return representation < rhs.representation;
55  }
56  bool operator>(const PatternBenefit &rhs) const { return rhs < *this; }
57  bool operator<=(const PatternBenefit &rhs) const { return !(*this > rhs); }
58  bool operator>=(const PatternBenefit &rhs) const { return !(*this < rhs); }
59 
60 private:
61  unsigned short representation{ImpossibleToMatchSentinel};
62 };
63 
64 //===----------------------------------------------------------------------===//
65 // Pattern
66 //===----------------------------------------------------------------------===//
67 
68 /// This class contains all of the data related to a pattern, but does not
69 /// contain any methods or logic for the actual matching. This class is solely
70 /// used to interface with the metadata of a pattern, such as the benefit or
71 /// root operation.
72 class Pattern {
73  /// This enum represents the kind of value used to select the root operations
74  /// that match this pattern.
75  enum class RootKind {
76  /// The pattern root matches "any" operation.
77  Any,
78  /// The pattern root is matched using a concrete operation name.
80  /// The pattern root is matched using an interface ID.
81  InterfaceID,
82  /// The patter root is matched using a trait ID.
83  TraitID
84  };
85 
86 public:
87  /// Return a list of operations that may be generated when rewriting an
88  /// operation instance with this pattern.
89  ArrayRef<OperationName> getGeneratedOps() const { return generatedOps; }
90 
91  /// Return the root node that this pattern matches. Patterns that can match
92  /// multiple root types return std::nullopt.
93  std::optional<OperationName> getRootKind() const {
94  if (rootKind == RootKind::OperationName)
95  return OperationName::getFromOpaquePointer(rootValue);
96  return std::nullopt;
97  }
98 
99  /// Return the interface ID used to match the root operation of this pattern.
100  /// If the pattern does not use an interface ID for deciding the root match,
101  /// this returns std::nullopt.
102  std::optional<TypeID> getRootInterfaceID() const {
103  if (rootKind == RootKind::InterfaceID)
104  return TypeID::getFromOpaquePointer(rootValue);
105  return std::nullopt;
106  }
107 
108  /// Return the trait ID used to match the root operation of this pattern.
109  /// If the pattern does not use a trait ID for deciding the root match, this
110  /// returns std::nullopt.
111  std::optional<TypeID> getRootTraitID() const {
112  if (rootKind == RootKind::TraitID)
113  return TypeID::getFromOpaquePointer(rootValue);
114  return std::nullopt;
115  }
116 
117  /// Return the benefit (the inverse of "cost") of matching this pattern. The
118  /// benefit of a Pattern is always static - rewrites that may have dynamic
119  /// benefit can be instantiated multiple times (different Pattern instances)
120  /// for each benefit that they may return, and be guarded by different match
121  /// condition predicates.
122  PatternBenefit getBenefit() const { return benefit; }
123 
124  /// Returns true if this pattern is known to result in recursive application,
125  /// i.e. this pattern may generate IR that also matches this pattern, but is
126  /// known to bound the recursion. This signals to a rewrite driver that it is
127  /// safe to apply this pattern recursively to generated IR.
129  return contextAndHasBoundedRecursion.getInt();
130  }
131 
132  /// Return the MLIRContext used to create this pattern.
134  return contextAndHasBoundedRecursion.getPointer();
135  }
136 
137  /// Return a readable name for this pattern. This name should only be used for
138  /// debugging purposes, and may be empty.
139  StringRef getDebugName() const { return debugName; }
140 
141  /// Set the human readable debug name used for this pattern. This name will
142  /// only be used for debugging purposes.
143  void setDebugName(StringRef name) { debugName = name; }
144 
145  /// Return the set of debug labels attached to this pattern.
146  ArrayRef<StringRef> getDebugLabels() const { return debugLabels; }
147 
148  /// Add the provided debug labels to this pattern.
150  debugLabels.append(labels.begin(), labels.end());
151  }
152  void addDebugLabels(StringRef label) { debugLabels.push_back(label); }
153 
154 protected:
155  /// This class acts as a special tag that makes the desire to match "any"
156  /// operation type explicit. This helps to avoid unnecessary usages of this
157  /// feature, and ensures that the user is making a conscious decision.
158  struct MatchAnyOpTypeTag {};
159  /// This class acts as a special tag that makes the desire to match any
160  /// operation that implements a given interface explicit. This helps to avoid
161  /// unnecessary usages of this feature, and ensures that the user is making a
162  /// conscious decision.
164  /// This class acts as a special tag that makes the desire to match any
165  /// operation that implements a given trait explicit. This helps to avoid
166  /// unnecessary usages of this feature, and ensures that the user is making a
167  /// conscious decision.
169 
170  /// Construct a pattern with a certain benefit that matches the operation
171  /// with the given root name.
172  Pattern(StringRef rootName, PatternBenefit benefit, MLIRContext *context,
173  ArrayRef<StringRef> generatedNames = {});
174  /// Construct a pattern that may match any operation type. `generatedNames`
175  /// contains the names of operations that may be generated during a successful
176  /// rewrite. `MatchAnyOpTypeTag` is just a tag to ensure that the "match any"
177  /// behavior is what the user actually desired, `MatchAnyOpTypeTag()` should
178  /// always be supplied here.
179  Pattern(MatchAnyOpTypeTag tag, PatternBenefit benefit, MLIRContext *context,
180  ArrayRef<StringRef> generatedNames = {});
181  /// Construct a pattern that may match any operation that implements the
182  /// interface defined by the provided `interfaceID`. `generatedNames` contains
183  /// the names of operations that may be generated during a successful rewrite.
184  /// `MatchInterfaceOpTypeTag` is just a tag to ensure that the "match
185  /// interface" behavior is what the user actually desired,
186  /// `MatchInterfaceOpTypeTag()` should always be supplied here.
187  Pattern(MatchInterfaceOpTypeTag tag, TypeID interfaceID,
188  PatternBenefit benefit, MLIRContext *context,
189  ArrayRef<StringRef> generatedNames = {});
190  /// Construct a pattern that may match any operation that implements the
191  /// trait defined by the provided `traitID`. `generatedNames` contains the
192  /// names of operations that may be generated during a successful rewrite.
193  /// `MatchTraitOpTypeTag` is just a tag to ensure that the "match trait"
194  /// behavior is what the user actually desired, `MatchTraitOpTypeTag()` should
195  /// always be supplied here.
196  Pattern(MatchTraitOpTypeTag tag, TypeID traitID, PatternBenefit benefit,
197  MLIRContext *context, ArrayRef<StringRef> generatedNames = {});
198 
199  /// Set the flag detailing if this pattern has bounded rewrite recursion or
200  /// not.
201  void setHasBoundedRewriteRecursion(bool hasBoundedRecursionArg = true) {
202  contextAndHasBoundedRecursion.setInt(hasBoundedRecursionArg);
203  }
204 
205 private:
206  Pattern(const void *rootValue, RootKind rootKind,
207  ArrayRef<StringRef> generatedNames, PatternBenefit benefit,
208  MLIRContext *context);
209 
210  /// The value used to match the root operation of the pattern.
211  const void *rootValue;
212  RootKind rootKind;
213 
214  /// The expected benefit of matching this pattern.
215  const PatternBenefit benefit;
216 
217  /// The context this pattern was created from, and a boolean flag indicating
218  /// whether this pattern has bounded recursion or not.
219  llvm::PointerIntPair<MLIRContext *, 1, bool> contextAndHasBoundedRecursion;
220 
221  /// A list of the potential operations that may be generated when rewriting
222  /// an op with this pattern.
223  SmallVector<OperationName, 2> generatedOps;
224 
225  /// A readable name for this pattern. May be empty.
226  StringRef debugName;
227 
228  /// The set of debug labels attached to this pattern.
229  SmallVector<StringRef, 0> debugLabels;
230 };
231 
232 //===----------------------------------------------------------------------===//
233 // RewritePattern
234 //===----------------------------------------------------------------------===//
235 
236 /// RewritePattern is the common base class for all DAG to DAG replacements.
237 /// There are two possible usages of this class:
238 /// * Multi-step RewritePattern with "match" and "rewrite"
239 /// - By overloading the "match" and "rewrite" functions, the user can
240 /// separate the concerns of matching and rewriting.
241 /// * Single-step RewritePattern with "matchAndRewrite"
242 /// - By overloading the "matchAndRewrite" function, the user can perform
243 /// the rewrite in the same call as the match.
244 ///
245 class RewritePattern : public Pattern {
246 public:
247  virtual ~RewritePattern() = default;
248 
249  /// Rewrite the IR rooted at the specified operation with the result of
250  /// this pattern, generating any new operations with the specified
251  /// builder. If an unexpected error is encountered (an internal
252  /// compiler error), it is emitted through the normal MLIR diagnostic
253  /// hooks and the IR is left in a valid state.
254  virtual void rewrite(Operation *op, PatternRewriter &rewriter) const;
255 
256  /// Attempt to match against code rooted at the specified operation,
257  /// which is the same operation code as getRootKind().
258  virtual LogicalResult match(Operation *op) const;
259 
260  /// Attempt to match against code rooted at the specified operation,
261  /// which is the same operation code as getRootKind(). If successful, this
262  /// function will automatically perform the rewrite.
264  PatternRewriter &rewriter) const {
265  if (succeeded(match(op))) {
266  rewrite(op, rewriter);
267  return success();
268  }
269  return failure();
270  }
271 
272  /// This method provides a convenient interface for creating and initializing
273  /// derived rewrite patterns of the given type `T`.
274  template <typename T, typename... Args>
275  static std::unique_ptr<T> create(Args &&...args) {
276  std::unique_ptr<T> pattern =
277  std::make_unique<T>(std::forward<Args>(args)...);
278  initializePattern<T>(*pattern);
279 
280  // Set a default debug name if one wasn't provided.
281  if (pattern->getDebugName().empty())
282  pattern->setDebugName(llvm::getTypeName<T>());
283  return pattern;
284  }
285 
286 protected:
287  /// Inherit the base constructors from `Pattern`.
288  using Pattern::Pattern;
289 
290 private:
291  /// Trait to check if T provides a `getOperationName` method.
292  template <typename T, typename... Args>
293  using has_initialize = decltype(std::declval<T>().initialize());
294  template <typename T>
295  using detect_has_initialize = llvm::is_detected<has_initialize, T>;
296 
297  /// Initialize the derived pattern by calling its `initialize` method.
298  template <typename T>
299  static std::enable_if_t<detect_has_initialize<T>::value>
300  initializePattern(T &pattern) {
301  pattern.initialize();
302  }
303  /// Empty derived pattern initializer for patterns that do not have an
304  /// initialize method.
305  template <typename T>
306  static std::enable_if_t<!detect_has_initialize<T>::value>
307  initializePattern(T &) {}
308 
309  /// An anchor for the virtual table.
310  virtual void anchor();
311 };
312 
313 namespace detail {
314 /// OpOrInterfaceRewritePatternBase is a wrapper around RewritePattern that
315 /// allows for matching and rewriting against an instance of a derived operation
316 /// class or Interface.
317 template <typename SourceOp>
319  using RewritePattern::RewritePattern;
320 
321  /// Wrappers around the RewritePattern methods that pass the derived op type.
322  void rewrite(Operation *op, PatternRewriter &rewriter) const final {
323  rewrite(cast<SourceOp>(op), rewriter);
324  }
325  LogicalResult match(Operation *op) const final {
326  return match(cast<SourceOp>(op));
327  }
329  PatternRewriter &rewriter) const final {
330  return matchAndRewrite(cast<SourceOp>(op), rewriter);
331  }
332 
333  /// Rewrite and Match methods that operate on the SourceOp type. These must be
334  /// overridden by the derived pattern class.
335  virtual void rewrite(SourceOp op, PatternRewriter &rewriter) const {
336  llvm_unreachable("must override rewrite or matchAndRewrite");
337  }
338  virtual LogicalResult match(SourceOp op) const {
339  llvm_unreachable("must override match or matchAndRewrite");
340  }
341  virtual LogicalResult matchAndRewrite(SourceOp op,
342  PatternRewriter &rewriter) const {
343  if (succeeded(match(op))) {
344  rewrite(op, rewriter);
345  return success();
346  }
347  return failure();
348  }
349 };
350 } // namespace detail
351 
352 /// OpRewritePattern is a wrapper around RewritePattern that allows for
353 /// matching and rewriting against an instance of a derived operation class as
354 /// opposed to a raw Operation.
355 template <typename SourceOp>
357  : public detail::OpOrInterfaceRewritePatternBase<SourceOp> {
358  /// Patterns must specify the root operation name they match against, and can
359  /// also specify the benefit of the pattern matching and a list of generated
360  /// ops.
362  ArrayRef<StringRef> generatedNames = {})
364  SourceOp::getOperationName(), benefit, context, generatedNames) {}
365 };
366 
367 /// OpInterfaceRewritePattern is a wrapper around RewritePattern that allows for
368 /// matching and rewriting against an instance of an operation interface instead
369 /// of a raw Operation.
370 template <typename SourceOp>
372  : public detail::OpOrInterfaceRewritePatternBase<SourceOp> {
374  : detail::OpOrInterfaceRewritePatternBase<SourceOp>(
375  Pattern::MatchInterfaceOpTypeTag(), SourceOp::getInterfaceID(),
376  benefit, context) {}
377 };
378 
379 /// OpTraitRewritePattern is a wrapper around RewritePattern that allows for
380 /// matching and rewriting against instances of an operation that possess a
381 /// given trait.
382 template <template <typename> class TraitType>
384 public:
386  : RewritePattern(Pattern::MatchTraitOpTypeTag(), TypeID::get<TraitType>(),
387  benefit, context) {}
388 };
389 
390 //===----------------------------------------------------------------------===//
391 // RewriterBase
392 //===----------------------------------------------------------------------===//
393 
394 /// This class coordinates the application of a rewrite on a set of IR,
395 /// providing a way for clients to track mutations and create new operations.
396 /// This class serves as a common API for IR mutation between pattern rewrites
397 /// and non-pattern rewrites, and facilitates the development of shared
398 /// IR transformation utilities.
399 class RewriterBase : public OpBuilder {
400 public:
401  struct Listener : public OpBuilder::Listener {
403  : OpBuilder::Listener(ListenerBase::Kind::RewriterBaseListener) {}
404 
405  /// Notify the listener that the specified operation was modified in-place.
406  virtual void notifyOperationModified(Operation *op) {}
407 
408  /// Notify the listener that the specified operation is about to be replaced
409  /// with the set of values potentially produced by new operations. This is
410  /// called before the uses of the operation have been changed.
412  ValueRange replacement) {}
413 
414  /// This is called on an operation that a rewrite is removing, right before
415  /// the operation is deleted. At this point, the operation has zero uses.
416  virtual void notifyOperationRemoved(Operation *op) {}
417 
418  /// Notify the listener that the pattern failed to match the given
419  /// operation, and provide a callback to populate a diagnostic with the
420  /// reason why the failure occurred. This method allows for derived
421  /// listeners to optionally hook into the reason why a rewrite failed, and
422  /// display it to users.
423  virtual LogicalResult
425  function_ref<void(Diagnostic &)> reasonCallback) {
426  return failure();
427  }
428 
429  static bool classof(const OpBuilder::Listener *base);
430  };
431 
432  /// Move the blocks that belong to "region" before the given position in
433  /// another region "parent". The two regions must be different. The caller
434  /// is responsible for creating or updating the operation transferring flow
435  /// of control to the region and passing it the correct block arguments.
436  virtual void inlineRegionBefore(Region &region, Region &parent,
437  Region::iterator before);
438  void inlineRegionBefore(Region &region, Block *before);
439 
440  /// Clone the blocks that belong to "region" before the given position in
441  /// another region "parent". The two regions must be different. The caller is
442  /// responsible for creating or updating the operation transferring flow of
443  /// control to the region and passing it the correct block arguments.
444  virtual void cloneRegionBefore(Region &region, Region &parent,
445  Region::iterator before, IRMapping &mapping);
446  void cloneRegionBefore(Region &region, Region &parent,
447  Region::iterator before);
448  void cloneRegionBefore(Region &region, Block *before);
449 
450  /// This method replaces the uses of the results of `op` with the values in
451  /// `newValues` when the provided `functor` returns true for a specific use.
452  /// The number of values in `newValues` is required to match the number of
453  /// results of `op`. `allUsesReplaced`, if non-null, is set to true if all of
454  /// the uses of `op` were replaced. Note that in some rewriters, the given
455  /// 'functor' may be stored beyond the lifetime of the rewrite being applied.
456  /// As such, the function should not capture by reference and instead use
457  /// value capture as necessary.
458  virtual void
459  replaceOpWithIf(Operation *op, ValueRange newValues, bool *allUsesReplaced,
460  llvm::unique_function<bool(OpOperand &) const> functor);
461  void replaceOpWithIf(Operation *op, ValueRange newValues,
462  llvm::unique_function<bool(OpOperand &) const> functor) {
463  replaceOpWithIf(op, newValues, /*allUsesReplaced=*/nullptr,
464  std::move(functor));
465  }
466 
467  /// This method replaces the uses of the results of `op` with the values in
468  /// `newValues` when a use is nested within the given `block`. The number of
469  /// values in `newValues` is required to match the number of results of `op`.
470  /// If all uses of this operation are replaced, the operation is erased.
471  void replaceOpWithinBlock(Operation *op, ValueRange newValues, Block *block,
472  bool *allUsesReplaced = nullptr);
473 
474  /// This method replaces the results of the operation with the specified list
475  /// of values. The number of provided values must match the number of results
476  /// of the operation.
477  virtual void replaceOp(Operation *op, ValueRange newValues);
478 
479  /// Replaces the result op with a new op that is created without verification.
480  /// The result values of the two ops must be the same types.
481  template <typename OpTy, typename... Args>
482  OpTy replaceOpWithNewOp(Operation *op, Args &&...args) {
483  auto newOp = create<OpTy>(op->getLoc(), std::forward<Args>(args)...);
484  replaceOpWithResultsOfAnotherOp(op, newOp.getOperation());
485  return newOp;
486  }
487 
488  /// This method erases an operation that is known to have no uses.
489  virtual void eraseOp(Operation *op);
490 
491  /// This method erases all operations in a block.
492  virtual void eraseBlock(Block *block);
493 
494  /// Inline the operations of block 'source' into block 'dest' before the given
495  /// position. The source block will be deleted and must have no uses.
496  /// 'argValues' is used to replace the block arguments of 'source'.
497  ///
498  /// If the source block is inserted at the end of the dest block, the dest
499  /// block must have no successors. Similarly, if the source block is inserted
500  /// somewhere in the middle (or beginning) of the dest block, the source block
501  /// must have no successors. Otherwise, the resulting IR would have
502  /// unreachable operations.
503  virtual void inlineBlockBefore(Block *source, Block *dest,
504  Block::iterator before,
505  ValueRange argValues = std::nullopt);
506 
507  /// Inline the operations of block 'source' before the operation 'op'. The
508  /// source block will be deleted and must have no uses. 'argValues' is used to
509  /// replace the block arguments of 'source'
510  ///
511  /// The source block must have no successors. Otherwise, the resulting IR
512  /// would have unreachable operations.
513  void inlineBlockBefore(Block *source, Operation *op,
514  ValueRange argValues = std::nullopt);
515 
516  /// Inline the operations of block 'source' into the end of block 'dest'. The
517  /// source block will be deleted and must have no uses. 'argValues' is used to
518  /// replace the block arguments of 'source'
519  ///
520  /// The dest block must have no successors. Otherwise, the resulting IR would
521  /// have unreachable operation.
522  void mergeBlocks(Block *source, Block *dest,
523  ValueRange argValues = std::nullopt);
524 
525  /// Split the operations starting at "before" (inclusive) out of the given
526  /// block into a new block, and return it.
527  virtual Block *splitBlock(Block *block, Block::iterator before);
528 
529  /// This method is used to notify the rewriter that an in-place operation
530  /// modification is about to happen. A call to this function *must* be
531  /// followed by a call to either `finalizeRootUpdate` or `cancelRootUpdate`.
532  /// This is a minor efficiency win (it avoids creating a new operation and
533  /// removing the old one) but also often allows simpler code in the client.
534  virtual void startRootUpdate(Operation *op) {}
535 
536  /// This method is used to signal the end of a root update on the given
537  /// operation. This can only be called on operations that were provided to a
538  /// call to `startRootUpdate`.
539  virtual void finalizeRootUpdate(Operation *op);
540 
541  /// This method cancels a pending root update. This can only be called on
542  /// operations that were provided to a call to `startRootUpdate`.
543  virtual void cancelRootUpdate(Operation *op) {}
544 
545  /// This method is a utility wrapper around a root update of an operation. It
546  /// wraps calls to `startRootUpdate` and `finalizeRootUpdate` around the given
547  /// callable.
548  template <typename CallableT>
549  void updateRootInPlace(Operation *root, CallableT &&callable) {
550  startRootUpdate(root);
551  callable();
552  finalizeRootUpdate(root);
553  }
554 
555  /// Find uses of `from` and replace them with `to`. It also marks every
556  /// modified uses and notifies the rewriter that an in-place operation
557  /// modification is about to happen.
558  void replaceAllUsesWith(Value from, Value to) {
559  return replaceAllUsesWith(from.getImpl(), to);
560  }
561  template <typename OperandType, typename ValueT>
563  for (OperandType &operand : llvm::make_early_inc_range(from->getUses())) {
564  Operation *op = operand.getOwner();
565  updateRootInPlace(op, [&]() { operand.set(to); });
566  }
567  }
569  assert(from.size() == to.size() && "incorrect number of replacements");
570  for (auto it : llvm::zip(from, to))
571  replaceAllUsesWith(std::get<0>(it), std::get<1>(it));
572  }
573 
574  /// Find uses of `from` and replace them with `to` if the `functor` returns
575  /// true. It also marks every modified uses and notifies the rewriter that an
576  /// in-place operation modification is about to happen.
577  void replaceUsesWithIf(Value from, Value to,
578  function_ref<bool(OpOperand &)> functor);
579 
580  /// Find uses of `from` and replace them with `to` except if the user is
581  /// `exceptedUser`. It also marks every modified uses and notifies the
582  /// rewriter that an in-place operation modification is about to happen.
583  void replaceAllUsesExcept(Value from, Value to, Operation *exceptedUser) {
584  return replaceUsesWithIf(from, to, [&](OpOperand &use) {
585  Operation *user = use.getOwner();
586  return user != exceptedUser;
587  });
588  }
589 
590  /// Used to notify the rewriter that the IR failed to be rewritten because of
591  /// a match failure, and provide a callback to populate a diagnostic with the
592  /// reason why the failure occurred. This method allows for derived rewriters
593  /// to optionally hook into the reason why a rewrite failed, and display it to
594  /// users.
595  template <typename CallbackT>
596  std::enable_if_t<!std::is_convertible<CallbackT, Twine>::value, LogicalResult>
597  notifyMatchFailure(Location loc, CallbackT &&reasonCallback) {
598 #ifndef NDEBUG
599  if (auto *rewriteListener = dyn_cast_if_present<Listener>(listener))
600  return rewriteListener->notifyMatchFailure(
601  loc, function_ref<void(Diagnostic &)>(reasonCallback));
602  return failure();
603 #else
604  return failure();
605 #endif
606  }
607  template <typename CallbackT>
608  std::enable_if_t<!std::is_convertible<CallbackT, Twine>::value, LogicalResult>
609  notifyMatchFailure(Operation *op, CallbackT &&reasonCallback) {
610  if (auto *rewriteListener = dyn_cast_if_present<Listener>(listener))
611  return rewriteListener->notifyMatchFailure(
612  op->getLoc(), function_ref<void(Diagnostic &)>(reasonCallback));
613  return failure();
614  }
615  template <typename ArgT>
616  LogicalResult notifyMatchFailure(ArgT &&arg, const Twine &msg) {
617  return notifyMatchFailure(std::forward<ArgT>(arg),
618  [&](Diagnostic &diag) { diag << msg; });
619  }
620  template <typename ArgT>
621  LogicalResult notifyMatchFailure(ArgT &&arg, const char *msg) {
622  return notifyMatchFailure(std::forward<ArgT>(arg), Twine(msg));
623  }
624 
625 protected:
626  /// Initialize the builder.
627  explicit RewriterBase(MLIRContext *ctx,
628  OpBuilder::Listener *listener = nullptr)
629  : OpBuilder(ctx, listener) {}
630  explicit RewriterBase(const OpBuilder &otherBuilder)
631  : OpBuilder(otherBuilder) {}
632  virtual ~RewriterBase();
633 
634 private:
635  void operator=(const RewriterBase &) = delete;
636  RewriterBase(const RewriterBase &) = delete;
637 
638  /// 'op' and 'newOp' are known to have the same number of results, replace the
639  /// uses of op with uses of newOp.
640  void replaceOpWithResultsOfAnotherOp(Operation *op, Operation *newOp);
641 };
642 
643 //===----------------------------------------------------------------------===//
644 // IRRewriter
645 //===----------------------------------------------------------------------===//
646 
647 /// This class coordinates rewriting a piece of IR outside of a pattern rewrite,
648 /// providing a way to keep track of the mutations made to the IR. This class
649 /// should only be used in situations where another `RewriterBase` instance,
650 /// such as a `PatternRewriter`, is not available.
651 class IRRewriter : public RewriterBase {
652 public:
654  : RewriterBase(ctx, listener) {}
655  explicit IRRewriter(const OpBuilder &builder) : RewriterBase(builder) {}
656 };
657 
658 //===----------------------------------------------------------------------===//
659 // PatternRewriter
660 //===----------------------------------------------------------------------===//
661 
662 /// A special type of `RewriterBase` that coordinates the application of a
663 /// rewrite pattern on the current IR being matched, providing a way to keep
664 /// track of any mutations made. This class should be used to perform all
665 /// necessary IR mutations within a rewrite pattern, as the pattern driver may
666 /// be tracking various state that would be invalidated when a mutation takes
667 /// place.
669 public:
671 
672  /// A hook used to indicate if the pattern rewriter can recover from failure
673  /// during the rewrite stage of a pattern. For example, if the pattern
674  /// rewriter supports rollback, it may progress smoothly even if IR was
675  /// changed during the rewrite.
676  virtual bool canRecoverFromRewriteFailure() const { return false; }
677 };
678 
679 //===----------------------------------------------------------------------===//
680 // PDL Patterns
681 //===----------------------------------------------------------------------===//
682 
683 //===----------------------------------------------------------------------===//
684 // PDLValue
685 
686 /// Storage type of byte-code interpreter values. These are passed to constraint
687 /// functions as arguments.
688 class PDLValue {
689 public:
690  /// The underlying kind of a PDL value.
692 
693  /// Construct a new PDL value.
694  PDLValue(const PDLValue &other) = default;
695  PDLValue(std::nullptr_t = nullptr) {}
697  : value(value.getAsOpaquePointer()), kind(Kind::Attribute) {}
698  PDLValue(Operation *value) : value(value), kind(Kind::Operation) {}
699  PDLValue(Type value) : value(value.getAsOpaquePointer()), kind(Kind::Type) {}
700  PDLValue(TypeRange *value) : value(value), kind(Kind::TypeRange) {}
702  : value(value.getAsOpaquePointer()), kind(Kind::Value) {}
703  PDLValue(ValueRange *value) : value(value), kind(Kind::ValueRange) {}
704 
705  /// Returns true if the type of the held value is `T`.
706  template <typename T>
707  bool isa() const {
708  assert(value && "isa<> used on a null value");
709  return kind == getKindOf<T>();
710  }
711 
712  /// Attempt to dynamically cast this value to type `T`, returns null if this
713  /// value is not an instance of `T`.
714  template <typename T,
715  typename ResultT = std::conditional_t<
716  std::is_convertible<T, bool>::value, T, std::optional<T>>>
717  ResultT dyn_cast() const {
718  return isa<T>() ? castImpl<T>() : ResultT();
719  }
720 
721  /// Cast this value to type `T`, asserts if this value is not an instance of
722  /// `T`.
723  template <typename T>
724  T cast() const {
725  assert(isa<T>() && "expected value to be of type `T`");
726  return castImpl<T>();
727  }
728 
729  /// Get an opaque pointer to the value.
730  const void *getAsOpaquePointer() const { return value; }
731 
732  /// Return if this value is null or not.
733  explicit operator bool() const { return value; }
734 
735  /// Return the kind of this value.
736  Kind getKind() const { return kind; }
737 
738  /// Print this value to the provided output stream.
739  void print(raw_ostream &os) const;
740 
741  /// Print the specified value kind to an output stream.
742  static void print(raw_ostream &os, Kind kind);
743 
744 private:
745  /// Find the index of a given type in a range of other types.
746  template <typename...>
747  struct index_of_t;
748  template <typename T, typename... R>
749  struct index_of_t<T, T, R...> : std::integral_constant<size_t, 0> {};
750  template <typename T, typename F, typename... R>
751  struct index_of_t<T, F, R...>
752  : std::integral_constant<size_t, 1 + index_of_t<T, R...>::value> {};
753 
754  /// Return the kind used for the given T.
755  template <typename T>
756  static Kind getKindOf() {
757  return static_cast<Kind>(index_of_t<T, Attribute, Operation *, Type,
758  TypeRange, Value, ValueRange>::value);
759  }
760 
761  /// The internal implementation of `cast`, that returns the underlying value
762  /// as the given type `T`.
763  template <typename T>
764  std::enable_if_t<llvm::is_one_of<T, Attribute, Type, Value>::value, T>
765  castImpl() const {
766  return T::getFromOpaquePointer(value);
767  }
768  template <typename T>
769  std::enable_if_t<llvm::is_one_of<T, TypeRange, ValueRange>::value, T>
770  castImpl() const {
771  return *reinterpret_cast<T *>(const_cast<void *>(value));
772  }
773  template <typename T>
774  std::enable_if_t<std::is_pointer<T>::value, T> castImpl() const {
775  return reinterpret_cast<T>(const_cast<void *>(value));
776  }
777 
778  /// The internal opaque representation of a PDLValue.
779  const void *value{nullptr};
780  /// The kind of the opaque value.
781  Kind kind{Kind::Attribute};
782 };
783 
784 inline raw_ostream &operator<<(raw_ostream &os, PDLValue value) {
785  value.print(os);
786  return os;
787 }
788 
789 inline raw_ostream &operator<<(raw_ostream &os, PDLValue::Kind kind) {
790  PDLValue::print(os, kind);
791  return os;
792 }
793 
794 //===----------------------------------------------------------------------===//
795 // PDLResultList
796 
797 /// The class represents a list of PDL results, returned by a native rewrite
798 /// method. It provides the mechanism with which to pass PDLValues back to the
799 /// PDL bytecode.
801 public:
802  /// Push a new Attribute value onto the result list.
803  void push_back(Attribute value) { results.push_back(value); }
804 
805  /// Push a new Operation onto the result list.
806  void push_back(Operation *value) { results.push_back(value); }
807 
808  /// Push a new Type onto the result list.
809  void push_back(Type value) { results.push_back(value); }
810 
811  /// Push a new TypeRange onto the result list.
812  void push_back(TypeRange value) {
813  // The lifetime of a TypeRange can't be guaranteed, so we'll need to
814  // allocate a storage for it.
815  llvm::OwningArrayRef<Type> storage(value.size());
816  llvm::copy(value, storage.begin());
817  allocatedTypeRanges.emplace_back(std::move(storage));
818  typeRanges.push_back(allocatedTypeRanges.back());
819  results.push_back(&typeRanges.back());
820  }
822  typeRanges.push_back(value);
823  results.push_back(&typeRanges.back());
824  }
826  typeRanges.push_back(value);
827  results.push_back(&typeRanges.back());
828  }
829 
830  /// Push a new Value onto the result list.
831  void push_back(Value value) { results.push_back(value); }
832 
833  /// Push a new ValueRange onto the result list.
834  void push_back(ValueRange value) {
835  // The lifetime of a ValueRange can't be guaranteed, so we'll need to
836  // allocate a storage for it.
837  llvm::OwningArrayRef<Value> storage(value.size());
838  llvm::copy(value, storage.begin());
839  allocatedValueRanges.emplace_back(std::move(storage));
840  valueRanges.push_back(allocatedValueRanges.back());
841  results.push_back(&valueRanges.back());
842  }
843  void push_back(OperandRange value) {
844  valueRanges.push_back(value);
845  results.push_back(&valueRanges.back());
846  }
847  void push_back(ResultRange value) {
848  valueRanges.push_back(value);
849  results.push_back(&valueRanges.back());
850  }
851 
852 protected:
853  /// Create a new result list with the expected number of results.
854  PDLResultList(unsigned maxNumResults) {
855  // For now just reserve enough space for all of the results. We could do
856  // separate counts per range type, but it isn't really worth it unless there
857  // are a "large" number of results.
858  typeRanges.reserve(maxNumResults);
859  valueRanges.reserve(maxNumResults);
860  }
861 
862  /// The PDL results held by this list.
864  /// Memory used to store ranges held by the list.
867  /// Memory allocated to store ranges in the result list whose lifetime was
868  /// generated in the native function.
871 };
872 
873 //===----------------------------------------------------------------------===//
874 // PDLPatternConfig
875 
876 /// An individual configuration for a pattern, which can be accessed by native
877 /// functions via the PDLPatternConfigSet. This allows for injecting additional
878 /// configuration into PDL patterns that is specific to certain compilation
879 /// flows.
881 public:
882  virtual ~PDLPatternConfig() = default;
883 
884  /// Hooks that are invoked at the beginning and end of a rewrite of a matched
885  /// pattern. These can be used to setup any specific state necessary for the
886  /// rewrite.
887  virtual void notifyRewriteBegin(PatternRewriter &rewriter) {}
888  virtual void notifyRewriteEnd(PatternRewriter &rewriter) {}
889 
890  /// Return the TypeID that represents this configuration.
891  TypeID getTypeID() const { return id; }
892 
893 protected:
894  PDLPatternConfig(TypeID id) : id(id) {}
895 
896 private:
897  TypeID id;
898 };
899 
900 /// This class provides a base class for users implementing a type of pattern
901 /// configuration.
902 template <typename T>
904 public:
905  /// Support LLVM style casting.
906  static bool classof(const PDLPatternConfig *config) {
907  return config->getTypeID() == getConfigID();
908  }
909 
910  /// Return the type id used for this configuration.
911  static TypeID getConfigID() { return TypeID::get<T>(); }
912 
913 protected:
915 };
916 
917 /// This class contains a set of configurations for a specific pattern.
918 /// Configurations are uniqued by TypeID, meaning that only one configuration of
919 /// each type is allowed.
921 public:
922  PDLPatternConfigSet() = default;
923 
924  /// Construct a set with the given configurations.
925  template <typename... ConfigsT>
926  PDLPatternConfigSet(ConfigsT &&...configs) {
927  (addConfig(std::forward<ConfigsT>(configs)), ...);
928  }
929 
930  /// Get the configuration defined by the given type. Asserts that the
931  /// configuration of the provided type exists.
932  template <typename T>
933  const T &get() const {
934  const T *config = tryGet<T>();
935  assert(config && "configuration not found");
936  return *config;
937  }
938 
939  /// Get the configuration defined by the given type, returns nullptr if the
940  /// configuration does not exist.
941  template <typename T>
942  const T *tryGet() const {
943  for (const auto &configIt : configs)
944  if (const T *config = dyn_cast<T>(configIt.get()))
945  return config;
946  return nullptr;
947  }
948 
949  /// Notify the configurations within this set at the beginning or end of a
950  /// rewrite of a matched pattern.
952  for (const auto &config : configs)
953  config->notifyRewriteBegin(rewriter);
954  }
956  for (const auto &config : configs)
957  config->notifyRewriteEnd(rewriter);
958  }
959 
960 protected:
961  /// Add a configuration to the set.
962  template <typename T>
963  void addConfig(T &&config) {
964  assert(!tryGet<std::decay_t<T>>() && "configuration already exists");
965  configs.emplace_back(
966  std::make_unique<std::decay_t<T>>(std::forward<T>(config)));
967  }
968 
969  /// The set of configurations for this pattern. This uses a vector instead of
970  /// a map with the expectation that the number of configurations per set is
971  /// small (<= 1).
973 };
974 
975 //===----------------------------------------------------------------------===//
976 // PDLPatternModule
977 
978 /// A generic PDL pattern constraint function. This function applies a
979 /// constraint to a given set of opaque PDLValue entities. Returns success if
980 /// the constraint successfully held, failure otherwise.
983 /// A native PDL rewrite function. This function performs a rewrite on the
984 /// given set of values. Any results from this rewrite that should be passed
985 /// back to PDL should be added to the provided result list. This method is only
986 /// invoked when the corresponding match was successful. Returns failure if an
987 /// invariant of the rewrite was broken (certain rewriters may recover from
988 /// partial pattern application).
989 using PDLRewriteFunction = std::function<LogicalResult(
991 
992 namespace detail {
993 namespace pdl_function_builder {
994 /// A utility variable that always resolves to false. This is useful for static
995 /// asserts that are always false, but only should fire in certain templated
996 /// constructs. For example, if a templated function should never be called, the
997 /// function could be defined as:
998 ///
999 /// template <typename T>
1000 /// void foo() {
1001 /// static_assert(always_false<T>, "This function should never be called");
1002 /// }
1003 ///
1004 template <class... T>
1005 constexpr bool always_false = false;
1006 
1007 //===----------------------------------------------------------------------===//
1008 // PDL Function Builder: Type Processing
1009 //===----------------------------------------------------------------------===//
1010 
1011 /// This struct provides a convenient way to determine how to process a given
1012 /// type as either a PDL parameter, or a result value. This allows for
1013 /// supporting complex types in constraint and rewrite functions, without
1014 /// requiring the user to hand-write the necessary glue code themselves.
1015 /// Specializations of this class should implement the following methods to
1016 /// enable support as a PDL argument or result type:
1017 ///
1018 /// static LogicalResult verifyAsArg(
1019 /// function_ref<LogicalResult(const Twine &)> errorFn, PDLValue pdlValue,
1020 /// size_t argIdx);
1021 ///
1022 /// * This method verifies that the given PDLValue is valid for use as a
1023 /// value of `T`.
1024 ///
1025 /// static T processAsArg(PDLValue pdlValue);
1026 ///
1027 /// * This method processes the given PDLValue as a value of `T`.
1028 ///
1029 /// static void processAsResult(PatternRewriter &, PDLResultList &results,
1030 /// const T &value);
1031 ///
1032 /// * This method processes the given value of `T` as the result of a
1033 /// function invocation. The method should package the value into an
1034 /// appropriate form and append it to the given result list.
1035 ///
1036 /// If the type `T` is based on a higher order value, consider using
1037 /// `ProcessPDLValueBasedOn` as a base class of the specialization to simplify
1038 /// the implementation.
1039 ///
1040 template <typename T, typename Enable = void>
1042 
1043 /// This struct provides a simplified model for processing types that are based
1044 /// on another type, e.g. APInt is based on the handling for IntegerAttr. This
1045 /// allows for building the necessary processing functions on top of the base
1046 /// value instead of a PDLValue. Derived users should implement the following
1047 /// (which subsume the ProcessPDLValue variants):
1048 ///
1049 /// static LogicalResult verifyAsArg(
1050 /// function_ref<LogicalResult(const Twine &)> errorFn,
1051 /// const BaseT &baseValue, size_t argIdx);
1052 ///
1053 /// * This method verifies that the given PDLValue is valid for use as a
1054 /// value of `T`.
1055 ///
1056 /// static T processAsArg(BaseT baseValue);
1057 ///
1058 /// * This method processes the given base value as a value of `T`.
1059 ///
1060 template <typename T, typename BaseT>
1062  static LogicalResult
1063  verifyAsArg(function_ref<LogicalResult(const Twine &)> errorFn,
1064  PDLValue pdlValue, size_t argIdx) {
1065  // Verify the base class before continuing.
1066  if (failed(ProcessPDLValue<BaseT>::verifyAsArg(errorFn, pdlValue, argIdx)))
1067  return failure();
1069  errorFn, ProcessPDLValue<BaseT>::processAsArg(pdlValue), argIdx);
1070  }
1071  static T processAsArg(PDLValue pdlValue) {
1074  }
1075 
1076  /// Explicitly add the expected parent API to ensure the parent class
1077  /// implements the necessary API (and doesn't implicitly inherit it from
1078  /// somewhere else).
1079  static LogicalResult
1080  verifyAsArg(function_ref<LogicalResult(const Twine &)> errorFn, BaseT value,
1081  size_t argIdx) {
1082  return success();
1083  }
1084  static T processAsArg(BaseT baseValue);
1085 };
1086 
1087 /// This struct provides a simplified model for processing types that have
1088 /// "builtin" PDLValue support:
1089 /// * Attribute, Operation *, Type, TypeRange, ValueRange
1090 template <typename T>
1092  static LogicalResult
1093  verifyAsArg(function_ref<LogicalResult(const Twine &)> errorFn,
1094  PDLValue pdlValue, size_t argIdx) {
1095  if (pdlValue)
1096  return success();
1097  return errorFn("expected a non-null value for argument " + Twine(argIdx) +
1098  " of type: " + llvm::getTypeName<T>());
1099  }
1100 
1101  static T processAsArg(PDLValue pdlValue) { return pdlValue.cast<T>(); }
1103  T value) {
1104  results.push_back(value);
1105  }
1106 };
1107 
1108 /// This struct provides a simplified model for processing types that inherit
1109 /// from builtin PDLValue types. For example, derived attributes like
1110 /// IntegerAttr, derived types like IntegerType, derived operations like
1111 /// ModuleOp, Interfaces, etc.
1112 template <typename T, typename BaseT>
1114  static LogicalResult
1115  verifyAsArg(function_ref<LogicalResult(const Twine &)> errorFn,
1116  BaseT baseValue, size_t argIdx) {
1117  return TypeSwitch<BaseT, LogicalResult>(baseValue)
1118  .Case([&](T) { return success(); })
1119  .Default([&](BaseT) {
1120  return errorFn("expected argument " + Twine(argIdx) +
1121  " to be of type: " + llvm::getTypeName<T>());
1122  });
1123  }
1125 
1126  static T processAsArg(BaseT baseValue) {
1127  return baseValue.template cast<T>();
1128  }
1130 
1132  T value) {
1133  results.push_back(value);
1134  }
1135 };
1136 
1137 //===----------------------------------------------------------------------===//
1138 // Attribute
1139 
1140 template <>
1141 struct ProcessPDLValue<Attribute> : public ProcessBuiltinPDLValue<Attribute> {};
1142 template <typename T>
1144  std::enable_if_t<std::is_base_of<Attribute, T>::value>>
1145  : public ProcessDerivedPDLValue<T, Attribute> {};
1146 
1147 /// Handling for various Attribute value types.
1148 template <>
1149 struct ProcessPDLValue<StringRef>
1150  : public ProcessPDLValueBasedOn<StringRef, StringAttr> {
1151  static StringRef processAsArg(StringAttr value) { return value.getValue(); }
1153 
1154  static void processAsResult(PatternRewriter &rewriter, PDLResultList &results,
1155  StringRef value) {
1156  results.push_back(rewriter.getStringAttr(value));
1157  }
1158 };
1159 template <>
1160 struct ProcessPDLValue<std::string>
1161  : public ProcessPDLValueBasedOn<std::string, StringAttr> {
1162  template <typename T>
1163  static std::string processAsArg(T value) {
1164  static_assert(always_false<T>,
1165  "`std::string` arguments require a string copy, use "
1166  "`StringRef` for string-like arguments instead");
1167  return {};
1168  }
1169  static void processAsResult(PatternRewriter &rewriter, PDLResultList &results,
1170  StringRef value) {
1171  results.push_back(rewriter.getStringAttr(value));
1172  }
1173 };
1174 
1175 //===----------------------------------------------------------------------===//
1176 // Operation
1177 
1178 template <>
1181 template <typename T>
1182 struct ProcessPDLValue<T, std::enable_if_t<std::is_base_of<OpState, T>::value>>
1183  : public ProcessDerivedPDLValue<T, Operation *> {
1184  static T processAsArg(Operation *value) { return cast<T>(value); }
1185 };
1186 
1187 //===----------------------------------------------------------------------===//
1188 // Type
1189 
1190 template <>
1191 struct ProcessPDLValue<Type> : public ProcessBuiltinPDLValue<Type> {};
1192 template <typename T>
1193 struct ProcessPDLValue<T, std::enable_if_t<std::is_base_of<Type, T>::value>>
1194  : public ProcessDerivedPDLValue<T, Type> {};
1195 
1196 //===----------------------------------------------------------------------===//
1197 // TypeRange
1198 
1199 template <>
1200 struct ProcessPDLValue<TypeRange> : public ProcessBuiltinPDLValue<TypeRange> {};
1201 template <>
1205  results.push_back(types);
1206  }
1207 };
1208 template <>
1212  results.push_back(types);
1213  }
1214 };
1215 template <unsigned N>
1218  SmallVector<Type, N> values) {
1219  results.push_back(TypeRange(values));
1220  }
1221 };
1222 
1223 //===----------------------------------------------------------------------===//
1224 // Value
1225 
1226 template <>
1227 struct ProcessPDLValue<Value> : public ProcessBuiltinPDLValue<Value> {};
1228 
1229 //===----------------------------------------------------------------------===//
1230 // ValueRange
1231 
1232 template <>
1233 struct ProcessPDLValue<ValueRange> : public ProcessBuiltinPDLValue<ValueRange> {
1234 };
1235 template <>
1238  OperandRange values) {
1239  results.push_back(values);
1240  }
1241 };
1242 template <>
1245  ResultRange values) {
1246  results.push_back(values);
1247  }
1248 };
1249 template <unsigned N>
1252  SmallVector<Value, N> values) {
1253  results.push_back(ValueRange(values));
1254  }
1255 };
1256 
1257 //===----------------------------------------------------------------------===//
1258 // PDL Function Builder: Argument Handling
1259 //===----------------------------------------------------------------------===//
1260 
1261 /// Validate the given PDLValues match the constraints defined by the argument
1262 /// types of the given function. In the case of failure, a match failure
1263 /// diagnostic is emitted.
1264 /// FIXME: This should be completely removed in favor of `assertArgs`, but PDL
1265 /// does not currently preserve Constraint application ordering.
1266 template <typename PDLFnT, std::size_t... I>
1268  std::index_sequence<I...>) {
1269  using FnTraitsT = llvm::function_traits<PDLFnT>;
1270 
1271  auto errorFn = [&](const Twine &msg) {
1272  return rewriter.notifyMatchFailure(rewriter.getUnknownLoc(), msg);
1273  };
1274  return success(
1275  (succeeded(ProcessPDLValue<typename FnTraitsT::template arg_t<I + 1>>::
1276  verifyAsArg(errorFn, values[I], I)) &&
1277  ...));
1278 }
1279 
1280 /// Assert that the given PDLValues match the constraints defined by the
1281 /// arguments of the given function. In the case of failure, a fatal error
1282 /// is emitted.
1283 template <typename PDLFnT, std::size_t... I>
1285  std::index_sequence<I...>) {
1286  // We only want to do verification in debug builds, same as with `assert`.
1287 #if LLVM_ENABLE_ABI_BREAKING_CHECKS
1288  using FnTraitsT = llvm::function_traits<PDLFnT>;
1289  auto errorFn = [&](const Twine &msg) -> LogicalResult {
1290  llvm::report_fatal_error(msg);
1291  };
1292  (void)errorFn;
1293  assert((succeeded(ProcessPDLValue<typename FnTraitsT::template arg_t<I + 1>>::
1294  verifyAsArg(errorFn, values[I], I)) &&
1295  ...));
1296 #endif
1297  (void)values;
1298 }
1299 
1300 //===----------------------------------------------------------------------===//
1301 // PDL Function Builder: Results Handling
1302 //===----------------------------------------------------------------------===//
1303 
1304 /// Store a single result within the result list.
1305 template <typename T>
1307  PDLResultList &results, T &&value) {
1308  ProcessPDLValue<T>::processAsResult(rewriter, results,
1309  std::forward<T>(value));
1310  return success();
1311 }
1312 
1313 /// Store a std::pair<> as individual results within the result list.
1314 template <typename T1, typename T2>
1316  PDLResultList &results,
1317  std::pair<T1, T2> &&pair) {
1318  if (failed(processResults(rewriter, results, std::move(pair.first))) ||
1319  failed(processResults(rewriter, results, std::move(pair.second))))
1320  return failure();
1321  return success();
1322 }
1323 
1324 /// Store a std::tuple<> as individual results within the result list.
1325 template <typename... Ts>
1327  PDLResultList &results,
1328  std::tuple<Ts...> &&tuple) {
1329  auto applyFn = [&](auto &&...args) {
1330  return (succeeded(processResults(rewriter, results, std::move(args))) &&
1331  ...);
1332  };
1333  return success(std::apply(applyFn, std::move(tuple)));
1334 }
1335 
1336 /// Handle LogicalResult propagation.
1338  PDLResultList &results,
1339  LogicalResult &&result) {
1340  return result;
1341 }
1342 template <typename T>
1344  PDLResultList &results,
1345  FailureOr<T> &&result) {
1346  if (failed(result))
1347  return failure();
1348  return processResults(rewriter, results, std::move(*result));
1349 }
1350 
1351 //===----------------------------------------------------------------------===//
1352 // PDL Constraint Builder
1353 //===----------------------------------------------------------------------===//
1354 
1355 /// Process the arguments of a native constraint and invoke it.
1356 template <typename PDLFnT, std::size_t... I,
1357  typename FnTraitsT = llvm::function_traits<PDLFnT>>
1358 typename FnTraitsT::result_t
1360  ArrayRef<PDLValue> values,
1361  std::index_sequence<I...>) {
1362  return fn(
1363  rewriter,
1364  (ProcessPDLValue<typename FnTraitsT::template arg_t<I + 1>>::processAsArg(
1365  values[I]))...);
1366 }
1367 
1368 /// Build a constraint function from the given function `ConstraintFnT`. This
1369 /// allows for enabling the user to define simpler, more direct constraint
1370 /// functions without needing to handle the low-level PDL goop.
1371 ///
1372 /// If the constraint function is already in the correct form, we just forward
1373 /// it directly.
1374 template <typename ConstraintFnT>
1375 std::enable_if_t<
1376  std::is_convertible<ConstraintFnT, PDLConstraintFunction>::value,
1378 buildConstraintFn(ConstraintFnT &&constraintFn) {
1379  return std::forward<ConstraintFnT>(constraintFn);
1380 }
1381 /// Otherwise, we generate a wrapper that will unpack the PDLValues in the form
1382 /// we desire.
1383 template <typename ConstraintFnT>
1384 std::enable_if_t<
1385  !std::is_convertible<ConstraintFnT, PDLConstraintFunction>::value,
1387 buildConstraintFn(ConstraintFnT &&constraintFn) {
1388  return [constraintFn = std::forward<ConstraintFnT>(constraintFn)](
1389  PatternRewriter &rewriter,
1390  ArrayRef<PDLValue> values) -> LogicalResult {
1391  auto argIndices = std::make_index_sequence<
1392  llvm::function_traits<ConstraintFnT>::num_args - 1>();
1393  if (failed(verifyAsArgs<ConstraintFnT>(rewriter, values, argIndices)))
1394  return failure();
1395  return processArgsAndInvokeConstraint(constraintFn, rewriter, values,
1396  argIndices);
1397  };
1398 }
1399 
1400 //===----------------------------------------------------------------------===//
1401 // PDL Rewrite Builder
1402 //===----------------------------------------------------------------------===//
1403 
1404 /// Process the arguments of a native rewrite and invoke it.
1405 /// This overload handles the case of no return values.
1406 template <typename PDLFnT, std::size_t... I,
1407  typename FnTraitsT = llvm::function_traits<PDLFnT>>
1408 std::enable_if_t<std::is_same<typename FnTraitsT::result_t, void>::value,
1409  LogicalResult>
1412  std::index_sequence<I...>) {
1413  fn(rewriter,
1414  (ProcessPDLValue<typename FnTraitsT::template arg_t<I + 1>>::processAsArg(
1415  values[I]))...);
1416  return success();
1417 }
1418 /// This overload handles the case of return values, which need to be packaged
1419 /// into the result list.
1420 template <typename PDLFnT, std::size_t... I,
1421  typename FnTraitsT = llvm::function_traits<PDLFnT>>
1422 std::enable_if_t<!std::is_same<typename FnTraitsT::result_t, void>::value,
1423  LogicalResult>
1425  PDLResultList &results, ArrayRef<PDLValue> values,
1426  std::index_sequence<I...>) {
1427  return processResults(
1428  rewriter, results,
1429  fn(rewriter, (ProcessPDLValue<typename FnTraitsT::template arg_t<I + 1>>::
1430  processAsArg(values[I]))...));
1431  (void)values;
1432 }
1433 
1434 /// Build a rewrite function from the given function `RewriteFnT`. This
1435 /// allows for enabling the user to define simpler, more direct rewrite
1436 /// functions without needing to handle the low-level PDL goop.
1437 ///
1438 /// If the rewrite function is already in the correct form, we just forward
1439 /// it directly.
1440 template <typename RewriteFnT>
1441 std::enable_if_t<std::is_convertible<RewriteFnT, PDLRewriteFunction>::value,
1443 buildRewriteFn(RewriteFnT &&rewriteFn) {
1444  return std::forward<RewriteFnT>(rewriteFn);
1445 }
1446 /// Otherwise, we generate a wrapper that will unpack the PDLValues in the form
1447 /// we desire.
1448 template <typename RewriteFnT>
1449 std::enable_if_t<!std::is_convertible<RewriteFnT, PDLRewriteFunction>::value,
1451 buildRewriteFn(RewriteFnT &&rewriteFn) {
1452  return [rewriteFn = std::forward<RewriteFnT>(rewriteFn)](
1453  PatternRewriter &rewriter, PDLResultList &results,
1454  ArrayRef<PDLValue> values) {
1455  auto argIndices =
1456  std::make_index_sequence<llvm::function_traits<RewriteFnT>::num_args -
1457  1>();
1458  assertArgs<RewriteFnT>(rewriter, values, argIndices);
1459  return processArgsAndInvokeRewrite(rewriteFn, rewriter, results, values,
1460  argIndices);
1461  };
1462 }
1463 
1464 } // namespace pdl_function_builder
1465 } // namespace detail
1466 
1467 //===----------------------------------------------------------------------===//
1468 // PDLPatternModule
1469 
1470 /// This class contains all of the necessary data for a set of PDL patterns, or
1471 /// pattern rewrites specified in the form of the PDL dialect. This PDL module
1472 /// contained by this pattern may contain any number of `pdl.pattern`
1473 /// operations.
1475 public:
1476  PDLPatternModule() = default;
1477 
1478  /// Construct a PDL pattern with the given module and configurations.
1480  : pdlModule(std::move(module)) {}
1481  template <typename... ConfigsT>
1482  PDLPatternModule(OwningOpRef<ModuleOp> module, ConfigsT &&...patternConfigs)
1483  : PDLPatternModule(std::move(module)) {
1484  auto configSet = std::make_unique<PDLPatternConfigSet>(
1485  std::forward<ConfigsT>(patternConfigs)...);
1486  attachConfigToPatterns(*pdlModule, *configSet);
1487  configs.emplace_back(std::move(configSet));
1488  }
1489 
1490  /// Merge the state in `other` into this pattern module.
1491  void mergeIn(PDLPatternModule &&other);
1492 
1493  /// Return the internal PDL module of this pattern.
1494  ModuleOp getModule() { return pdlModule.get(); }
1495 
1496  //===--------------------------------------------------------------------===//
1497  // Function Registry
1498 
1499  /// Register a constraint function with PDL. A constraint function may be
1500  /// specified in one of two ways:
1501  ///
1502  /// * `LogicalResult (PatternRewriter &, ArrayRef<PDLValue>)`
1503  ///
1504  /// In this overload the arguments of the constraint function are passed via
1505  /// the low-level PDLValue form.
1506  ///
1507  /// * `LogicalResult (PatternRewriter &, ValueTs... values)`
1508  ///
1509  /// In this form the arguments of the constraint function are passed via the
1510  /// expected high level C++ type. In this form, the framework will
1511  /// automatically unwrap PDLValues and convert them to the expected ValueTs.
1512  /// For example, if the constraint function accepts a `Operation *`, the
1513  /// framework will automatically cast the input PDLValue. In the case of a
1514  /// `StringRef`, the framework will automatically unwrap the argument as a
1515  /// StringAttr and pass the underlying string value. To see the full list of
1516  /// supported types, or to see how to add handling for custom types, view
1517  /// the definition of `ProcessPDLValue` above.
1518  void registerConstraintFunction(StringRef name,
1519  PDLConstraintFunction constraintFn);
1520  template <typename ConstraintFnT>
1521  void registerConstraintFunction(StringRef name,
1522  ConstraintFnT &&constraintFn) {
1525  std::forward<ConstraintFnT>(constraintFn)));
1526  }
1527 
1528  /// Register a rewrite function with PDL. A rewrite function may be specified
1529  /// in one of two ways:
1530  ///
1531  /// * `void (PatternRewriter &, PDLResultList &, ArrayRef<PDLValue>)`
1532  ///
1533  /// In this overload the arguments of the constraint function are passed via
1534  /// the low-level PDLValue form, and the results are manually appended to
1535  /// the given result list.
1536  ///
1537  /// * `ResultT (PatternRewriter &, ValueTs... values)`
1538  ///
1539  /// In this form the arguments and result of the rewrite function are passed
1540  /// via the expected high level C++ type. In this form, the framework will
1541  /// automatically unwrap the PDLValues arguments and convert them to the
1542  /// expected ValueTs. It will also automatically handle the processing and
1543  /// packaging of the result value to the result list. For example, if the
1544  /// rewrite function takes a `Operation *`, the framework will automatically
1545  /// cast the input PDLValue. In the case of a `StringRef`, the framework
1546  /// will automatically unwrap the argument as a StringAttr and pass the
1547  /// underlying string value. In the reverse case, if the rewrite returns a
1548  /// StringRef or std::string, it will automatically package this as a
1549  /// StringAttr and append it to the result list. To see the full list of
1550  /// supported types, or to see how to add handling for custom types, view
1551  /// the definition of `ProcessPDLValue` above.
1552  void registerRewriteFunction(StringRef name, PDLRewriteFunction rewriteFn);
1553  template <typename RewriteFnT>
1554  void registerRewriteFunction(StringRef name, RewriteFnT &&rewriteFn) {
1556  std::forward<RewriteFnT>(rewriteFn)));
1557  }
1558 
1559  /// Return the set of the registered constraint functions.
1560  const llvm::StringMap<PDLConstraintFunction> &getConstraintFunctions() const {
1561  return constraintFunctions;
1562  }
1563  llvm::StringMap<PDLConstraintFunction> takeConstraintFunctions() {
1564  return constraintFunctions;
1565  }
1566  /// Return the set of the registered rewrite functions.
1567  const llvm::StringMap<PDLRewriteFunction> &getRewriteFunctions() const {
1568  return rewriteFunctions;
1569  }
1570  llvm::StringMap<PDLRewriteFunction> takeRewriteFunctions() {
1571  return rewriteFunctions;
1572  }
1573 
1574  /// Return the set of the registered pattern configs.
1576  return std::move(configs);
1577  }
1579  return std::move(configMap);
1580  }
1581 
1582  /// Clear out the patterns and functions within this module.
1583  void clear() {
1584  pdlModule = nullptr;
1585  constraintFunctions.clear();
1586  rewriteFunctions.clear();
1587  }
1588 
1589 private:
1590  /// Attach the given pattern config set to the patterns defined within the
1591  /// given module.
1592  void attachConfigToPatterns(ModuleOp module, PDLPatternConfigSet &configSet);
1593 
1594  /// The module containing the `pdl.pattern` operations.
1595  OwningOpRef<ModuleOp> pdlModule;
1596 
1597  /// The set of configuration sets referenced by patterns within `pdlModule`.
1600 
1601  /// The external functions referenced from within the PDL module.
1602  llvm::StringMap<PDLConstraintFunction> constraintFunctions;
1603  llvm::StringMap<PDLRewriteFunction> rewriteFunctions;
1604 };
1605 
1606 //===----------------------------------------------------------------------===//
1607 // RewritePatternSet
1608 //===----------------------------------------------------------------------===//
1609 
1611  using NativePatternListT = std::vector<std::unique_ptr<RewritePattern>>;
1612 
1613 public:
1614  RewritePatternSet(MLIRContext *context) : context(context) {}
1615 
1616  /// Construct a RewritePatternSet populated with the given pattern.
1618  std::unique_ptr<RewritePattern> pattern)
1619  : context(context) {
1620  nativePatterns.emplace_back(std::move(pattern));
1621  }
1623  : context(pattern.getModule()->getContext()),
1624  pdlPatterns(std::move(pattern)) {}
1625 
1626  MLIRContext *getContext() const { return context; }
1627 
1628  /// Return the native patterns held in this list.
1629  NativePatternListT &getNativePatterns() { return nativePatterns; }
1630 
1631  /// Return the PDL patterns held in this list.
1632  PDLPatternModule &getPDLPatterns() { return pdlPatterns; }
1633 
1634  /// Clear out all of the held patterns in this list.
1635  void clear() {
1636  nativePatterns.clear();
1637  pdlPatterns.clear();
1638  }
1639 
1640  //===--------------------------------------------------------------------===//
1641  // 'add' methods for adding patterns to the set.
1642  //===--------------------------------------------------------------------===//
1643 
1644  /// Add an instance of each of the pattern types 'Ts' to the pattern list with
1645  /// the given arguments. Return a reference to `this` for chaining insertions.
1646  /// Note: ConstructorArg is necessary here to separate the two variadic lists.
1647  template <typename... Ts, typename ConstructorArg,
1648  typename... ConstructorArgs,
1649  typename = std::enable_if_t<sizeof...(Ts) != 0>>
1650  RewritePatternSet &add(ConstructorArg &&arg, ConstructorArgs &&...args) {
1651  // The following expands a call to emplace_back for each of the pattern
1652  // types 'Ts'.
1653  (addImpl<Ts>(/*debugLabels=*/std::nullopt,
1654  std::forward<ConstructorArg>(arg),
1655  std::forward<ConstructorArgs>(args)...),
1656  ...);
1657  return *this;
1658  }
1659  /// An overload of the above `add` method that allows for attaching a set
1660  /// of debug labels to the attached patterns. This is useful for labeling
1661  /// groups of patterns that may be shared between multiple different
1662  /// passes/users.
1663  template <typename... Ts, typename ConstructorArg,
1664  typename... ConstructorArgs,
1665  typename = std::enable_if_t<sizeof...(Ts) != 0>>
1667  ConstructorArg &&arg,
1668  ConstructorArgs &&...args) {
1669  // The following expands a call to emplace_back for each of the pattern
1670  // types 'Ts'.
1671  (addImpl<Ts>(debugLabels, arg, args...), ...);
1672  return *this;
1673  }
1674 
1675  /// Add an instance of each of the pattern types 'Ts'. Return a reference to
1676  /// `this` for chaining insertions.
1677  template <typename... Ts>
1679  (addImpl<Ts>(), ...);
1680  return *this;
1681  }
1682 
1683  /// Add the given native pattern to the pattern list. Return a reference to
1684  /// `this` for chaining insertions.
1685  RewritePatternSet &add(std::unique_ptr<RewritePattern> pattern) {
1686  nativePatterns.emplace_back(std::move(pattern));
1687  return *this;
1688  }
1689 
1690  /// Add the given PDL pattern to the pattern list. Return a reference to
1691  /// `this` for chaining insertions.
1693  pdlPatterns.mergeIn(std::move(pattern));
1694  return *this;
1695  }
1696 
1697  // Add a matchAndRewrite style pattern represented as a C function pointer.
1698  template <typename OpType>
1700  add(LogicalResult (*implFn)(OpType, PatternRewriter &rewriter),
1701  PatternBenefit benefit = 1, ArrayRef<StringRef> generatedNames = {}) {
1702  struct FnPattern final : public OpRewritePattern<OpType> {
1703  FnPattern(LogicalResult (*implFn)(OpType, PatternRewriter &rewriter),
1704  MLIRContext *context, PatternBenefit benefit,
1705  ArrayRef<StringRef> generatedNames)
1706  : OpRewritePattern<OpType>(context, benefit, generatedNames),
1707  implFn(implFn) {}
1708 
1709  LogicalResult matchAndRewrite(OpType op,
1710  PatternRewriter &rewriter) const override {
1711  return implFn(op, rewriter);
1712  }
1713 
1714  private:
1715  LogicalResult (*implFn)(OpType, PatternRewriter &rewriter);
1716  };
1717  add(std::make_unique<FnPattern>(std::move(implFn), getContext(), benefit,
1718  generatedNames));
1719  return *this;
1720  }
1721 
1722  //===--------------------------------------------------------------------===//
1723  // Pattern Insertion
1724  //===--------------------------------------------------------------------===//
1725 
1726  // TODO: These are soft deprecated in favor of the 'add' methods above.
1727 
1728  /// Add an instance of each of the pattern types 'Ts' to the pattern list with
1729  /// the given arguments. Return a reference to `this` for chaining insertions.
1730  /// Note: ConstructorArg is necessary here to separate the two variadic lists.
1731  template <typename... Ts, typename ConstructorArg,
1732  typename... ConstructorArgs,
1733  typename = std::enable_if_t<sizeof...(Ts) != 0>>
1734  RewritePatternSet &insert(ConstructorArg &&arg, ConstructorArgs &&...args) {
1735  // The following expands a call to emplace_back for each of the pattern
1736  // types 'Ts'.
1737  (addImpl<Ts>(/*debugLabels=*/std::nullopt, arg, args...), ...);
1738  return *this;
1739  }
1740 
1741  /// Add an instance of each of the pattern types 'Ts'. Return a reference to
1742  /// `this` for chaining insertions.
1743  template <typename... Ts>
1745  (addImpl<Ts>(), ...);
1746  return *this;
1747  }
1748 
1749  /// Add the given native pattern to the pattern list. Return a reference to
1750  /// `this` for chaining insertions.
1751  RewritePatternSet &insert(std::unique_ptr<RewritePattern> pattern) {
1752  nativePatterns.emplace_back(std::move(pattern));
1753  return *this;
1754  }
1755 
1756  /// Add the given PDL pattern to the pattern list. Return a reference to
1757  /// `this` for chaining insertions.
1759  pdlPatterns.mergeIn(std::move(pattern));
1760  return *this;
1761  }
1762 
1763  // Add a matchAndRewrite style pattern represented as a C function pointer.
1764  template <typename OpType>
1766  insert(LogicalResult (*implFn)(OpType, PatternRewriter &rewriter)) {
1767  struct FnPattern final : public OpRewritePattern<OpType> {
1768  FnPattern(LogicalResult (*implFn)(OpType, PatternRewriter &rewriter),
1769  MLIRContext *context)
1770  : OpRewritePattern<OpType>(context), implFn(implFn) {
1771  this->setDebugName(llvm::getTypeName<FnPattern>());
1772  }
1773 
1774  LogicalResult matchAndRewrite(OpType op,
1775  PatternRewriter &rewriter) const override {
1776  return implFn(op, rewriter);
1777  }
1778 
1779  private:
1780  LogicalResult (*implFn)(OpType, PatternRewriter &rewriter);
1781  };
1782  add(std::make_unique<FnPattern>(std::move(implFn), getContext()));
1783  return *this;
1784  }
1785 
1786 private:
1787  /// Add an instance of the pattern type 'T'. Return a reference to `this` for
1788  /// chaining insertions.
1789  template <typename T, typename... Args>
1790  std::enable_if_t<std::is_base_of<RewritePattern, T>::value>
1791  addImpl(ArrayRef<StringRef> debugLabels, Args &&...args) {
1792  std::unique_ptr<T> pattern =
1793  RewritePattern::create<T>(std::forward<Args>(args)...);
1794  pattern->addDebugLabels(debugLabels);
1795  nativePatterns.emplace_back(std::move(pattern));
1796  }
1797  template <typename T, typename... Args>
1798  std::enable_if_t<std::is_base_of<PDLPatternModule, T>::value>
1799  addImpl(ArrayRef<StringRef> debugLabels, Args &&...args) {
1800  // TODO: Add the provided labels to the PDL pattern when PDL supports
1801  // labels.
1802  pdlPatterns.mergeIn(T(std::forward<Args>(args)...));
1803  }
1804 
1805  MLIRContext *const context;
1806  NativePatternListT nativePatterns;
1807  PDLPatternModule pdlPatterns;
1808 };
1809 
1810 } // namespace mlir
1811 
1812 #endif // MLIR_IR_PATTERNMATCH_H
static void copy(Location loc, Value dst, Value src, Value size, OpBuilder &builder)
Copies the given number of bytes from src to dst pointers.
static std::string diag(const llvm::Value &value)
Attributes are known-constant values of operations.
Definition: Attributes.h:25
Block represents an ordered list of Operations.
Definition: Block.h:30
OpListType::iterator iterator
Definition: Block.h:129
StringAttr getStringAttr(const Twine &bytes)
Definition: Builders.cpp:255
Location getUnknownLoc()
Definition: Builders.cpp:26
This class contains all of the information necessary to report a diagnostic to the DiagnosticEngine.
Definition: Diagnostics.h:156
This class provides support for representing a failure result, or a valid value of type T.
Definition: LogicalResult.h:78
This is a utility class for mapping one set of IR entities to another.
Definition: IRMapping.h:26
This class represents a single IR object that contains a use list.
Definition: UseDefLists.h:172
use_range getUses() const
Returns a range of all uses, which is useful for iterating over all uses.
Definition: UseDefLists.h:206
This class coordinates rewriting a piece of IR outside of a pattern rewrite, providing a way to keep ...
Definition: PatternMatch.h:651
IRRewriter(MLIRContext *ctx, OpBuilder::Listener *listener=nullptr)
Definition: PatternMatch.h:653
IRRewriter(const OpBuilder &builder)
Definition: PatternMatch.h:655
This class defines the main interface for locations in MLIR and acts as a non-nullable wrapper around...
Definition: Location.h:63
MLIRContext is the top-level object for a collection of MLIR operations.
Definition: MLIRContext.h:60
This class helps build Operations.
Definition: Builders.h:202
Listener * listener
The optional listener for events of this builder.
Definition: Builders.h:568
This class represents an operand of an operation.
Definition: Value.h:255
OpTraitRewritePattern is a wrapper around RewritePattern that allows for matching and rewriting again...
Definition: PatternMatch.h:383
OpTraitRewritePattern(MLIRContext *context, PatternBenefit benefit=1)
Definition: PatternMatch.h:385
This class implements the operand iterators for the Operation class.
Definition: ValueRange.h:42
static OperationName getFromOpaquePointer(const void *pointer)
Operation is the basic unit of execution within MLIR.
Definition: Operation.h:75
Location getLoc()
The source location the operation was defined or derived from.
Definition: Operation.h:207
OpTy get() const
Allow accessing the internal op.
Definition: OwningOpRef.h:50
This class provides a base class for users implementing a type of pattern configuration.
Definition: PatternMatch.h:903
static TypeID getConfigID()
Return the type id used for this configuration.
Definition: PatternMatch.h:911
static bool classof(const PDLPatternConfig *config)
Support LLVM style casting.
Definition: PatternMatch.h:906
This class contains a set of configurations for a specific pattern.
Definition: PatternMatch.h:920
const T & get() const
Get the configuration defined by the given type.
Definition: PatternMatch.h:933
PDLPatternConfigSet(ConfigsT &&...configs)
Construct a set with the given configurations.
Definition: PatternMatch.h:926
const T * tryGet() const
Get the configuration defined by the given type, returns nullptr if the configuration does not exist.
Definition: PatternMatch.h:942
SmallVector< std::unique_ptr< PDLPatternConfig > > configs
The set of configurations for this pattern.
Definition: PatternMatch.h:972
void addConfig(T &&config)
Add a configuration to the set.
Definition: PatternMatch.h:963
void notifyRewriteBegin(PatternRewriter &rewriter)
Notify the configurations within this set at the beginning or end of a rewrite of a matched pattern.
Definition: PatternMatch.h:951
void notifyRewriteEnd(PatternRewriter &rewriter)
Definition: PatternMatch.h:955
An individual configuration for a pattern, which can be accessed by native functions via the PDLPatte...
Definition: PatternMatch.h:880
virtual ~PDLPatternConfig()=default
PDLPatternConfig(TypeID id)
Definition: PatternMatch.h:894
virtual void notifyRewriteEnd(PatternRewriter &rewriter)
Definition: PatternMatch.h:888
virtual void notifyRewriteBegin(PatternRewriter &rewriter)
Hooks that are invoked at the beginning and end of a rewrite of a matched pattern.
Definition: PatternMatch.h:887
TypeID getTypeID() const
Return the TypeID that represents this configuration.
Definition: PatternMatch.h:891
This class contains all of the necessary data for a set of PDL patterns, or pattern rewrites specifie...
void clear()
Clear out the patterns and functions within this module.
const llvm::StringMap< PDLRewriteFunction > & getRewriteFunctions() const
Return the set of the registered rewrite functions.
llvm::StringMap< PDLConstraintFunction > takeConstraintFunctions()
PDLPatternModule(OwningOpRef< ModuleOp > module, ConfigsT &&...patternConfigs)
void registerConstraintFunction(StringRef name, ConstraintFnT &&constraintFn)
PDLPatternModule(OwningOpRef< ModuleOp > module)
Construct a PDL pattern with the given module and configurations.
void registerRewriteFunction(StringRef name, RewriteFnT &&rewriteFn)
ModuleOp getModule()
Return the internal PDL module of this pattern.
const llvm::StringMap< PDLConstraintFunction > & getConstraintFunctions() const
Return the set of the registered constraint functions.
void registerRewriteFunction(StringRef name, PDLRewriteFunction rewriteFn)
Register a rewrite function with PDL.
SmallVector< std::unique_ptr< PDLPatternConfigSet > > takeConfigs()
Return the set of the registered pattern configs.
void mergeIn(PDLPatternModule &&other)
Merge the state in other into this pattern module.
void registerConstraintFunction(StringRef name, PDLConstraintFunction constraintFn)
Register a constraint function with PDL.
llvm::StringMap< PDLRewriteFunction > takeRewriteFunctions()
DenseMap< Operation *, PDLPatternConfigSet * > takeConfigMap()
The class represents a list of PDL results, returned by a native rewrite method.
Definition: PatternMatch.h:800
void push_back(ValueTypeRange< OperandRange > value)
Definition: PatternMatch.h:821
void push_back(ResultRange value)
Definition: PatternMatch.h:847
void push_back(ValueRange value)
Push a new ValueRange onto the result list.
Definition: PatternMatch.h:834
PDLResultList(unsigned maxNumResults)
Create a new result list with the expected number of results.
Definition: PatternMatch.h:854
SmallVector< llvm::OwningArrayRef< Type > > allocatedTypeRanges
Memory allocated to store ranges in the result list whose lifetime was generated in the native functi...
Definition: PatternMatch.h:869
void push_back(ValueTypeRange< ResultRange > value)
Definition: PatternMatch.h:825
SmallVector< llvm::OwningArrayRef< Value > > allocatedValueRanges
Definition: PatternMatch.h:870
void push_back(Attribute value)
Push a new Attribute value onto the result list.
Definition: PatternMatch.h:803
SmallVector< TypeRange > typeRanges
Memory used to store ranges held by the list.
Definition: PatternMatch.h:865
SmallVector< PDLValue > results
The PDL results held by this list.
Definition: PatternMatch.h:863
void push_back(Type value)
Push a new Type onto the result list.
Definition: PatternMatch.h:809
void push_back(Operation *value)
Push a new Operation onto the result list.
Definition: PatternMatch.h:806
void push_back(OperandRange value)
Definition: PatternMatch.h:843
void push_back(Value value)
Push a new Value onto the result list.
Definition: PatternMatch.h:831
SmallVector< ValueRange > valueRanges
Definition: PatternMatch.h:866
void push_back(TypeRange value)
Push a new TypeRange onto the result list.
Definition: PatternMatch.h:812
Storage type of byte-code interpreter values.
Definition: PatternMatch.h:688
PDLValue(std::nullptr_t=nullptr)
Definition: PatternMatch.h:695
PDLValue(Type value)
Definition: PatternMatch.h:699
const void * getAsOpaquePointer() const
Get an opaque pointer to the value.
Definition: PatternMatch.h:730
PDLValue(Attribute value)
Definition: PatternMatch.h:696
PDLValue(Operation *value)
Definition: PatternMatch.h:698
Kind getKind() const
Return the kind of this value.
Definition: PatternMatch.h:736
ResultT dyn_cast() const
Attempt to dynamically cast this value to type T, returns null if this value is not an instance of T.
Definition: PatternMatch.h:717
bool isa() const
Returns true if the type of the held value is T.
Definition: PatternMatch.h:707
void print(raw_ostream &os) const
Print this value to the provided output stream.
PDLValue(TypeRange *value)
Definition: PatternMatch.h:700
Kind
The underlying kind of a PDL value.
Definition: PatternMatch.h:691
T cast() const
Cast this value to type T, asserts if this value is not an instance of T.
Definition: PatternMatch.h:724
PDLValue(ValueRange *value)
Definition: PatternMatch.h:703
PDLValue(const PDLValue &other)=default
Construct a new PDL value.
PDLValue(Value value)
Definition: PatternMatch.h:701
This class represents the benefit of a pattern match in a unitless scheme that ranges from 0 (very li...
Definition: PatternMatch.h:33
PatternBenefit & operator=(const PatternBenefit &)=default
bool operator<(const PatternBenefit &rhs) const
Definition: PatternMatch.h:53
bool operator==(const PatternBenefit &rhs) const
Definition: PatternMatch.h:49
static PatternBenefit impossibleToMatch()
Definition: PatternMatch.h:42
bool operator>=(const PatternBenefit &rhs) const
Definition: PatternMatch.h:58
bool operator<=(const PatternBenefit &rhs) const
Definition: PatternMatch.h:57
PatternBenefit(const PatternBenefit &)=default
bool isImpossibleToMatch() const
Definition: PatternMatch.h:43
bool operator!=(const PatternBenefit &rhs) const
Definition: PatternMatch.h:52
PatternBenefit()=default
bool operator>(const PatternBenefit &rhs) const
Definition: PatternMatch.h:56
unsigned short getBenefit() const
If the corresponding pattern can match, return its benefit. If the.
A special type of RewriterBase that coordinates the application of a rewrite pattern on the current I...
Definition: PatternMatch.h:668
virtual bool canRecoverFromRewriteFailure() const
A hook used to indicate if the pattern rewriter can recover from failure during the rewrite stage of ...
Definition: PatternMatch.h:676
This class contains all of the data related to a pattern, but does not contain any methods or logic f...
Definition: PatternMatch.h:72
std::optional< TypeID > getRootInterfaceID() const
Return the interface ID used to match the root operation of this pattern.
Definition: PatternMatch.h:102
Pattern(StringRef rootName, PatternBenefit benefit, MLIRContext *context, ArrayRef< StringRef > generatedNames={})
Construct a pattern with a certain benefit that matches the operation with the given root name.
bool hasBoundedRewriteRecursion() const
Returns true if this pattern is known to result in recursive application, i.e.
Definition: PatternMatch.h:128
std::optional< OperationName > getRootKind() const
Return the root node that this pattern matches.
Definition: PatternMatch.h:93
MLIRContext * getContext() const
Return the MLIRContext used to create this pattern.
Definition: PatternMatch.h:133
ArrayRef< StringRef > getDebugLabels() const
Return the set of debug labels attached to this pattern.
Definition: PatternMatch.h:146
void setHasBoundedRewriteRecursion(bool hasBoundedRecursionArg=true)
Set the flag detailing if this pattern has bounded rewrite recursion or not.
Definition: PatternMatch.h:201
ArrayRef< OperationName > getGeneratedOps() const
Return a list of operations that may be generated when rewriting an operation instance with this patt...
Definition: PatternMatch.h:89
PatternBenefit getBenefit() const
Return the benefit (the inverse of "cost") of matching this pattern.
Definition: PatternMatch.h:122
std::optional< TypeID > getRootTraitID() const
Return the trait ID used to match the root operation of this pattern.
Definition: PatternMatch.h:111
void setDebugName(StringRef name)
Set the human readable debug name used for this pattern.
Definition: PatternMatch.h:143
void addDebugLabels(StringRef label)
Definition: PatternMatch.h:152
void addDebugLabels(ArrayRef< StringRef > labels)
Add the provided debug labels to this pattern.
Definition: PatternMatch.h:149
StringRef getDebugName() const
Return a readable name for this pattern.
Definition: PatternMatch.h:139
This class contains a list of basic blocks and a link to the parent operation it is attached to.
Definition: Region.h:26
BlockListType::iterator iterator
Definition: Region.h:52
This class implements the result iterators for the Operation class.
Definition: ValueRange.h:231
RewritePatternSet & add(PDLPatternModule &&pattern)
Add the given PDL pattern to the pattern list.
RewritePatternSet & add(LogicalResult(*implFn)(OpType, PatternRewriter &rewriter), PatternBenefit benefit=1, ArrayRef< StringRef > generatedNames={})
NativePatternListT & getNativePatterns()
Return the native patterns held in this list.
RewritePatternSet & add(std::unique_ptr< RewritePattern > pattern)
Add the given native pattern to the pattern list.
RewritePatternSet(PDLPatternModule &&pattern)
RewritePatternSet & insert(ConstructorArg &&arg, ConstructorArgs &&...args)
Add an instance of each of the pattern types 'Ts' to the pattern list with the given arguments.
MLIRContext * getContext() const
void clear()
Clear out all of the held patterns in this list.
RewritePatternSet(MLIRContext *context)
RewritePatternSet & add()
Add an instance of each of the pattern types 'Ts'.
RewritePatternSet(MLIRContext *context, std::unique_ptr< RewritePattern > pattern)
Construct a RewritePatternSet populated with the given pattern.
RewritePatternSet & insert(PDLPatternModule &&pattern)
Add the given PDL pattern to the pattern list.
RewritePatternSet & insert(std::unique_ptr< RewritePattern > pattern)
Add the given native pattern to the pattern list.
PDLPatternModule & getPDLPatterns()
Return the PDL patterns held in this list.
RewritePatternSet & add(ConstructorArg &&arg, ConstructorArgs &&...args)
Add an instance of each of the pattern types 'Ts' to the pattern list with the given arguments.
RewritePatternSet & insert()
Add an instance of each of the pattern types 'Ts'.
RewritePatternSet & addWithLabel(ArrayRef< StringRef > debugLabels, ConstructorArg &&arg, ConstructorArgs &&...args)
An overload of the above add method that allows for attaching a set of debug labels to the attached p...
RewritePatternSet & insert(LogicalResult(*implFn)(OpType, PatternRewriter &rewriter))
RewritePattern is the common base class for all DAG to DAG replacements.
Definition: PatternMatch.h:245
virtual LogicalResult match(Operation *op) const
Attempt to match against code rooted at the specified operation, which is the same operation code as ...
static std::unique_ptr< T > create(Args &&...args)
This method provides a convenient interface for creating and initializing derived rewrite patterns of...
Definition: PatternMatch.h:275
virtual ~RewritePattern()=default
virtual LogicalResult matchAndRewrite(Operation *op, PatternRewriter &rewriter) const
Attempt to match against code rooted at the specified operation, which is the same operation code as ...
Definition: PatternMatch.h:263
virtual void rewrite(Operation *op, PatternRewriter &rewriter) const
Rewrite the IR rooted at the specified operation with the result of this pattern, generating any new ...
This class coordinates the application of a rewrite on a set of IR, providing a way for clients to tr...
Definition: PatternMatch.h:399
virtual void replaceOpWithIf(Operation *op, ValueRange newValues, bool *allUsesReplaced, llvm::unique_function< bool(OpOperand &) const > functor)
This method replaces the uses of the results of op with the values in newValues when the provided fun...
std::enable_if_t<!std::is_convertible< CallbackT, Twine >::value, LogicalResult > notifyMatchFailure(Location loc, CallbackT &&reasonCallback)
Used to notify the rewriter that the IR failed to be rewritten because of a match failure,...
Definition: PatternMatch.h:597
std::enable_if_t<!std::is_convertible< CallbackT, Twine >::value, LogicalResult > notifyMatchFailure(Operation *op, CallbackT &&reasonCallback)
Definition: PatternMatch.h:609
void replaceOpWithIf(Operation *op, ValueRange newValues, llvm::unique_function< bool(OpOperand &) const > functor)
Definition: PatternMatch.h:461
virtual void cloneRegionBefore(Region &region, Region &parent, Region::iterator before, IRMapping &mapping)
Clone the blocks that belong to "region" before the given position in another region "parent".
virtual void eraseBlock(Block *block)
This method erases all operations in a block.
LogicalResult notifyMatchFailure(ArgT &&arg, const Twine &msg)
Definition: PatternMatch.h:616
virtual Block * splitBlock(Block *block, Block::iterator before)
Split the operations starting at "before" (inclusive) out of the given block into a new block,...
virtual void replaceOp(Operation *op, ValueRange newValues)
This method replaces the results of the operation with the specified list of values.
RewriterBase(const OpBuilder &otherBuilder)
Definition: PatternMatch.h:630
void updateRootInPlace(Operation *root, CallableT &&callable)
This method is a utility wrapper around a root update of an operation.
Definition: PatternMatch.h:549
void replaceAllUsesWith(ValueRange from, ValueRange to)
Definition: PatternMatch.h:568
void replaceAllUsesWith(Value from, Value to)
Find uses of from and replace them with to.
Definition: PatternMatch.h:558
void mergeBlocks(Block *source, Block *dest, ValueRange argValues=std::nullopt)
Inline the operations of block 'source' into the end of block 'dest'.
virtual void eraseOp(Operation *op)
This method erases an operation that is known to have no uses.
void replaceOpWithinBlock(Operation *op, ValueRange newValues, Block *block, bool *allUsesReplaced=nullptr)
This method replaces the uses of the results of op with the values in newValues when a use is nested ...
virtual void finalizeRootUpdate(Operation *op)
This method is used to signal the end of a root update on the given operation.
void replaceAllUsesExcept(Value from, Value to, Operation *exceptedUser)
Find uses of from and replace them with to except if the user is exceptedUser.
Definition: PatternMatch.h:583
RewriterBase(MLIRContext *ctx, OpBuilder::Listener *listener=nullptr)
Initialize the builder.
Definition: PatternMatch.h:627
virtual void cancelRootUpdate(Operation *op)
This method cancels a pending root update.
Definition: PatternMatch.h:543
LogicalResult notifyMatchFailure(ArgT &&arg, const char *msg)
Definition: PatternMatch.h:621
void replaceUsesWithIf(Value from, Value to, function_ref< bool(OpOperand &)> functor)
Find uses of from and replace them with to if the functor returns true.
virtual void inlineRegionBefore(Region &region, Region &parent, Region::iterator before)
Move the blocks that belong to "region" before the given position in another region "parent".
virtual void startRootUpdate(Operation *op)
This method is used to notify the rewriter that an in-place operation modification is about to happen...
Definition: PatternMatch.h:534
void replaceAllUsesWith(IRObjectWithUseList< OperandType > *from, ValueT &&to)
Definition: PatternMatch.h:562
virtual void inlineBlockBefore(Block *source, Block *dest, Block::iterator before, ValueRange argValues=std::nullopt)
Inline the operations of block 'source' into block 'dest' before the given position.
OpTy replaceOpWithNewOp(Operation *op, Args &&...args)
Replaces the result op with a new op that is created without verification.
Definition: PatternMatch.h:482
This class provides an efficient unique identifier for a specific C++ type.
Definition: TypeID.h:104
static TypeID getFromOpaquePointer(const void *pointer)
Definition: TypeID.h:132
This class provides an abstraction over the various different ranges of value types.
Definition: TypeRange.h:36
Instances of the Type class are uniqued, have an immutable identifier and an optional mutable compone...
Definition: Types.h:74
This class provides an abstraction over the different types of ranges over Values.
Definition: ValueRange.h:370
This class implements iteration on the types of a given range of values.
Definition: TypeRange.h:131
This class represents an instance of an SSA value in the MLIR system, representing a computable value...
Definition: Value.h:93
detail::ValueImpl * getImpl() const
Definition: Value.h:235
Operation * getOwner() const
Return the owner of this operand.
Definition: UseDefLists.h:40
FnTraitsT::result_t processArgsAndInvokeConstraint(PDLFnT &fn, PatternRewriter &rewriter, ArrayRef< PDLValue > values, std::index_sequence< I... >)
Process the arguments of a native constraint and invoke it.
void assertArgs(PatternRewriter &rewriter, ArrayRef< PDLValue > values, std::index_sequence< I... >)
Assert that the given PDLValues match the constraints defined by the arguments of the given function.
LogicalResult verifyAsArgs(PatternRewriter &rewriter, ArrayRef< PDLValue > values, std::index_sequence< I... >)
Validate the given PDLValues match the constraints defined by the argument types of the given functio...
std::enable_if_t< std::is_same< typename FnTraitsT::result_t, void >::value, LogicalResult > processArgsAndInvokeRewrite(PDLFnT &fn, PatternRewriter &rewriter, PDLResultList &, ArrayRef< PDLValue > values, std::index_sequence< I... >)
Process the arguments of a native rewrite and invoke it.
static LogicalResult processResults(PatternRewriter &rewriter, PDLResultList &results, T &&value)
Store a single result within the result list.
std::enable_if_t< std::is_convertible< ConstraintFnT, PDLConstraintFunction >::value, PDLConstraintFunction > buildConstraintFn(ConstraintFnT &&constraintFn)
Build a constraint function from the given function ConstraintFnT.
constexpr bool always_false
A utility variable that always resolves to false.
std::enable_if_t< std::is_convertible< RewriteFnT, PDLRewriteFunction >::value, PDLRewriteFunction > buildRewriteFn(RewriteFnT &&rewriteFn)
Build a rewrite function from the given function RewriteFnT.
@ Type
An inlay hint that for a type annotation.
This header declares functions that assit transformations in the MemRef dialect.
LogicalResult failure(bool isFailure=true)
Utility function to generate a LogicalResult.
Definition: LogicalResult.h:62
bool succeeded(LogicalResult result)
Utility function that returns true if the provided LogicalResult corresponds to a success value.
Definition: LogicalResult.h:68
LogicalResult success(bool isSuccess=true)
Utility function to generate a LogicalResult.
Definition: LogicalResult.h:56
std::function< LogicalResult(PatternRewriter &, ArrayRef< PDLValue >)> PDLConstraintFunction
A generic PDL pattern constraint function.
Definition: PatternMatch.h:982
bool failed(LogicalResult result)
Utility function that returns true if the provided LogicalResult corresponds to a failure value.
Definition: LogicalResult.h:72
std::function< LogicalResult(PatternRewriter &, PDLResultList &, ArrayRef< PDLValue >)> PDLRewriteFunction
A native PDL rewrite function.
Definition: PatternMatch.h:990
raw_ostream & operator<<(raw_ostream &os, const AliasResult &result)
Definition: AliasAnalysis.h:78
This class represents an efficient way to signal success or failure.
Definition: LogicalResult.h:26
Base class for listeners.
Definition: Builders.h:258
Kind
The kind of listener.
Definition: Builders.h:260
This class represents a listener that may be used to hook into various actions within an OpBuilder.
Definition: Builders.h:279
OpInterfaceRewritePattern is a wrapper around RewritePattern that allows for matching and rewriting a...
Definition: PatternMatch.h:372
OpInterfaceRewritePattern(MLIRContext *context, PatternBenefit benefit=1)
Definition: PatternMatch.h:373
OpRewritePattern is a wrapper around RewritePattern that allows for matching and rewriting against an...
Definition: PatternMatch.h:357
OpRewritePattern(MLIRContext *context, PatternBenefit benefit=1, ArrayRef< StringRef > generatedNames={})
Patterns must specify the root operation name they match against, and can also specify the benefit of...
Definition: PatternMatch.h:361
This class acts as a special tag that makes the desire to match "any" operation type explicit.
Definition: PatternMatch.h:158
This class acts as a special tag that makes the desire to match any operation that implements a given...
Definition: PatternMatch.h:163
This class acts as a special tag that makes the desire to match any operation that implements a given...
Definition: PatternMatch.h:168
virtual void notifyOperationModified(Operation *op)
Notify the listener that the specified operation was modified in-place.
Definition: PatternMatch.h:406
virtual LogicalResult notifyMatchFailure(Location loc, function_ref< void(Diagnostic &)> reasonCallback)
Notify the listener that the pattern failed to match the given operation, and provide a callback to p...
Definition: PatternMatch.h:424
virtual void notifyOperationRemoved(Operation *op)
This is called on an operation that a rewrite is removing, right before the operation is deleted.
Definition: PatternMatch.h:416
virtual void notifyOperationReplaced(Operation *op, ValueRange replacement)
Notify the listener that the specified operation is about to be replaced with the set of values poten...
Definition: PatternMatch.h:411
static bool classof(const OpBuilder::Listener *base)
OpOrInterfaceRewritePatternBase is a wrapper around RewritePattern that allows for matching and rewri...
Definition: PatternMatch.h:318
virtual void rewrite(SourceOp op, PatternRewriter &rewriter) const
Rewrite and Match methods that operate on the SourceOp type.
Definition: PatternMatch.h:335
void rewrite(Operation *op, PatternRewriter &rewriter) const final
Wrappers around the RewritePattern methods that pass the derived op type.
Definition: PatternMatch.h:322
virtual LogicalResult match(SourceOp op) const
Definition: PatternMatch.h:338
LogicalResult match(Operation *op) const final
Attempt to match against code rooted at the specified operation, which is the same operation code as ...
Definition: PatternMatch.h:325
virtual LogicalResult matchAndRewrite(SourceOp op, PatternRewriter &rewriter) const
Definition: PatternMatch.h:341
LogicalResult matchAndRewrite(Operation *op, PatternRewriter &rewriter) const final
Attempt to match against code rooted at the specified operation, which is the same operation code as ...
Definition: PatternMatch.h:328
This struct provides a simplified model for processing types that have "builtin" PDLValue support:
static void processAsResult(PatternRewriter &, PDLResultList &results, T value)
static LogicalResult verifyAsArg(function_ref< LogicalResult(const Twine &)> errorFn, PDLValue pdlValue, size_t argIdx)
This struct provides a simplified model for processing types that inherit from builtin PDLValue types...
static void processAsResult(PatternRewriter &, PDLResultList &results, T value)
static LogicalResult verifyAsArg(function_ref< LogicalResult(const Twine &)> errorFn, BaseT baseValue, size_t argIdx)
This struct provides a simplified model for processing types that are based on another type,...
static LogicalResult verifyAsArg(function_ref< LogicalResult(const Twine &)> errorFn, BaseT value, size_t argIdx)
Explicitly add the expected parent API to ensure the parent class implements the necessary API (and d...
static LogicalResult verifyAsArg(function_ref< LogicalResult(const Twine &)> errorFn, PDLValue pdlValue, size_t argIdx)
static void processAsResult(PatternRewriter &, PDLResultList &results, OperandRange values)
static void processAsResult(PatternRewriter &, PDLResultList &results, ResultRange values)
static void processAsResult(PatternRewriter &, PDLResultList &results, SmallVector< Type, N > values)
static void processAsResult(PatternRewriter &, PDLResultList &results, SmallVector< Value, N > values)
static void processAsResult(PatternRewriter &rewriter, PDLResultList &results, StringRef value)
static void processAsResult(PatternRewriter &, PDLResultList &results, ValueTypeRange< OperandRange > types)
static void processAsResult(PatternRewriter &, PDLResultList &results, ValueTypeRange< ResultRange > types)
static void processAsResult(PatternRewriter &rewriter, PDLResultList &results, StringRef value)
This struct provides a convenient way to determine how to process a given type as either a PDL parame...