MLIR  19.0.0git
Transforms.h
Go to the documentation of this file.
1 //===- Transforms.h - Linalg transformations as patterns --------*- C++ -*-===//
2 //
3 // Part of the LLVM Project, under the Apache License v2.0 with LLVM Exceptions.
4 // See https://llvm.org/LICENSE.txt for license information.
5 // SPDX-License-Identifier: Apache-2.0 WITH LLVM-exception
6 //
7 //===----------------------------------------------------------------------===//
8 
9 #ifndef MLIR_DIALECT_LINALG_TRANSFORMS_TRANSFORMS_H
10 #define MLIR_DIALECT_LINALG_TRANSFORMS_TRANSFORMS_H
11 
12 #include <utility>
13 
23 #include "mlir/IR/PatternMatch.h"
27 #include "llvm/ADT/SmallBitVector.h"
28 #include "llvm/ADT/SmallSet.h"
29 
30 namespace mlir {
31 namespace bufferization {
32 class AllocTensorOp;
33 class OneShotAnalysisState;
34 } // namespace bufferization
35 
36 namespace linalg {
37 
38 class LinalgOp;
39 
40 //===----------------------------------------------------------------------===//
41 // Utils.
42 //===----------------------------------------------------------------------===//
43 
44 /// Return vector::CombiningKind for the given op.
45 std::optional<vector::CombiningKind> getCombinerOpKind(Operation *combinerOp);
46 
47 //===----------------------------------------------------------------------===//
48 // Bufferization-related transforms.
49 //===----------------------------------------------------------------------===//
50 
52  enum class AllocOp { MemrefAlloc = 0, MemrefAlloca = 1 };
54 
55  enum class MemcpyOp {
57  MemrefCopy = 1,
58  LinalgCopy = 2
59  };
61 
62  /// If set to "true", only the destination tensor operands are bufferized to
63  /// a new allocation (and wrapped in "bufferization.to_tensor"), but not the
64  /// targeted op itself.
66 
67  /// If set to "true", a `memref.dealloc` operation will be emitted for each
68  /// allocated buffer. Otherwise, the memory is leaked, which is useful if
69  /// the buffer deallocation pipeline should be run after bufferization is
70  /// done.
71  bool emitDealloc = false;
72 };
73 
74 /// Materialize a buffer allocation for the given tensor.pad op and lower the
75 /// op to linalg.fill/linalg.generic + bufferization.materialize_in_destination.
76 /// E.g.:
77 ///
78 /// %0 = tensor.pad low[%l] high[%h] %t ...
79 ///
80 /// is lowered to:
81 ///
82 /// %alloc = memref.alloc
83 /// linalg.fill ... outs(%alloc)
84 /// %subview = memref.subview %alloc [%l] [...] [1]
85 /// bufferization.materialize_in_destination %t in %subview
86 /// %0 = bufferization.to_tensor %alloc restrict writable
87 ///
88 /// In addition to rewriting the IR as shown above, this function returns the
89 /// newly allocated buffer. The `insertionPoint` parameter can be used to
90 /// specify a custom insertion point for the buffer allocation.
93  tensor::PadOp padOp, Attribute memorySpace = {},
94  Operation *insertionPoint = nullptr);
95 
96 /// Materialize a buffer allocation for the given vector.mask op and bufferize
97 /// the op, including its region. E.g.:
98 ///
99 /// %0 = vector.mask {
100 /// vector.transfer_write %v, %t : vector<16xf32>, tensor<?xf32>
101 /// } : vector<16xi1> -> tensor<?xf32>
102 ///
103 /// is lowered to:
104 ///
105 /// %alloc = memref.alloc
106 /// bufferization.materialize_in_destination %t in %subview
107 /// vector.mask {
108 /// vector.transfer_write %arg0, %alloc : vector<16xf32>, memref<?xf32>
109 /// } : vector<16xi1>
110 /// %0 = bufferization.to_tensor %alloc restrict writable
111 ///
112 /// In addition to rewriting the IR as shown above, this function returns the
113 /// newly allocated buffer. The `insertionPoint` parameter can be used to
114 /// specify a custom insertion point for the buffer allocation.
116  const BufferizeToAllocationOptions &options,
117  vector::MaskOp maskOp, Attribute memorySpace = {},
118  Operation *insertionPoint = nullptr);
119 
120 /// Materialize a buffer allocation for the given bufferization.alloc_tensor op
121 /// and lower the op to memref.alloc + memref.tensor_store.
122 ///
123 /// In addition to rewriting the IR, this function returns the newly allocated
124 /// buffer. The `insertionPoint` parameter can be used to specify a custom
125 /// insertion point for the buffer allocation.
126 Value bufferizeToAllocation(RewriterBase &rewriter,
127  const BufferizeToAllocationOptions &options,
128  bufferization::AllocTensorOp allocTensorOp,
129  Attribute memorySpace = {},
130  Operation *insertionPoint = nullptr);
131 
132 /// Bufferize the given op with tensor semantics and materialize the result in
133 /// a newly allocated buffer.
134 ///
135 /// Only bufferizable ops that bufferize to a memory write or have an
136 /// aliasing OpOperand (and do not themselves bufferize to an allocation) are
137 /// supported. They are bufferized using their BufferizableOpInterface
138 /// implementation.
139 ///
140 /// Selected ops that bufferize to an allocation (or need special handling) are
141 /// also supported:
142 /// - tensor.pad
143 /// - vector.mask
144 ///
145 /// This function returns the newly allocated buffer. The `insertionPoint`
146 /// parameter can be used to specify a custom insertion point for the buffer
147 /// allocation.
148 Value bufferizeToAllocation(RewriterBase &rewriter,
149  const BufferizeToAllocationOptions &options,
150  Operation *op, Attribute memorySpace = {},
151  Operation *insertionPoint = nullptr);
152 
153 /// Try to eliminate tensor::EmptyOps inside `op` that are anchored on a
154 /// LinalgOp. This transforms looks for LinalgOps that have an unused output
155 /// operand and an input operand that is rooted in a tensor::EmptyOp. The
156 /// tensor::EmptyOp uses are replaced with the output operand and the two
157 /// operands of the LinalgOp are swapped.
158 ///
159 /// Example:
160 /// %0 = tensor.empty()
161 /// %1 = linalg.matmul ins(...) outs(%0)
162 /// %2 = linalg.generic ins(%1) outs(%dest) {
163 /// ^bb0(%in: f32, %out: f32):
164 /// // out not used
165 /// }
166 ///
167 /// The IR is transformed as follows:
168 /// %0 = tensor.empty()
169 /// %1 = linalg.matmul ins(...) outs(%dest)
170 /// %2 = linalg.generic ins(%0) outs(%1) {
171 /// ^bb0(%in: f32, %out: f32):
172 /// // Use %out instead of %in
173 /// }
174 ///
175 /// The "ins" operand has no uses inside the body of the LinalgOp and can be
176 /// folded away with existing cleanup patterns. Afterwards, the tensor::EmptyOp
177 /// can also fold away.
179  RewriterBase &rewriter, Operation *op,
180  bufferization::OneShotAnalysisState &state);
181 
182 //===----------------------------------------------------------------------===//
183 // Structs that configure the behavior of various transformations.
184 //===----------------------------------------------------------------------===//
185 
187  std::function<SmallVector<Value, 4>(OpBuilder &, Operation *)>;
188 
190  /// Computation function that returns the tile sizes for each operation.
191  /// Delayed construction of constant tile sizes should occur to interoperate
192  /// with folding.
194 
197  tileSizeComputationFunction = std::move(fun);
198  return *this;
199  }
200  /// Set the `tileSizeComputationFunction` to return the values `ts`. The
201  /// values must not fold away when tiling. Otherwise, use a more robust
202  /// `tileSizeComputationFunction`.
204  tileSizeComputationFunction = [=](OpBuilder &, Operation *) { return ts; };
205  return *this;
206  }
207  /// Convenience function to set the `tileSizeComputationFunction` to a
208  /// function that computes tile sizes at the point they are needed. Allows
209  /// proper interaction with folding.
211 
212  /// Tile all dynamic dimensions by 1. I.e., scalarize those dimensions.
213  /// Note: `scalarizeDynamicDims` and `setTileSizes` cannot be used together.
215 
216  /// The interchange vector to reorder the tiled loops.
218 
220  interchangeVector.assign(interchange.begin(), interchange.end());
221  return *this;
222  }
223 
224  /// The type of tile loops to generate.
226 
228  loopType = lt;
229  return *this;
230  }
231 
232  /// When specified, specifies distribution of generated tile loops to
233  /// processors.
234  std::optional<LinalgLoopDistributionOptions> distribution;
235 
238  distribution = std::move(distributionOptions);
239  return *this;
240  }
241 
242  /// Specification markers of how to distribute the `linalg.tiled_loop`.
244 
246  distributionTypes.assign(types.begin(), types.end());
247  return *this;
248  }
249 
250  /// Peel the specified loops.
252 
254  peeledLoops.clear();
255  peeledLoops.append(loops.begin(), loops.end());
256  return *this;
257  }
258 };
259 
261  /// Tile sizes used to tile the root operation.
264  tileSizes.assign(ts.begin(), ts.end());
265  return *this;
266  }
267  /// Tile interchange used to permute the tile loops.
269  /// When specified, specifies distribution of generated tile loops to
270  /// processors.
271  std::optional<LinalgLoopDistributionOptions> tileDistribution;
274  tileDistribution = std::move(distributionOptions);
275  return *this;
276  }
277 };
278 
280  /// A padding value for every operand.
283  paddingValues.assign(pv.begin(), pv.end());
284  return *this;
285  }
286  /// A list of iterator dimensions to pad.
289  paddingDimensions.assign(pd.begin(), pd.end());
290  return *this;
291  }
292  /// A list of multiples to which each padding dimension should be padded to.
293  std::optional<SmallVector<int64_t>> padToMultipleOf;
295  padToMultipleOf.emplace(m.begin(), m.end());
296  return *this;
297  }
298  /// A flag for every operand to mark the PadOp as nofold which enables
299  /// packing for statically shaped operands.
302  packPaddings.assign(pp.begin(), pp.end());
303  return *this;
304  }
305  /// A number of loops to hoist the PadOp out for every operand.
308  hoistPaddings.assign(hp.begin(), hp.end());
309  return *this;
310  }
311  /// A permutation vector for every operand used to transpose the packed
312  /// PadOp results.
316  transposePaddings.assign(tp.begin(), tp.end());
317  return *this;
318  }
319  enum class CopyBackOp : int8_t {
320  None = 0,
322  LinalgCopy = 2
323  };
324  /// The op to be used for copying the padded result to the original
325  /// destination tensor.
328  copyBackOp = op;
329  return *this;
330  }
331 };
332 
333 /// Callback function type used to perform the allocation for the promoted
334 /// `subView`. In `boundingSubViewsize` a best attempt is made to find the
335 /// smallest constant value for the size of the buffer needed for each
336 /// dimension. If that is not possible, contains the dynamic size of the
337 /// subview. The call back should return the buffer to use.
338 using AllocBufferCallbackFn = std::function<std::optional<Value>(
339  OpBuilder &b, memref::SubViewOp subView,
340  ArrayRef<Value> boundingSubViewSize, DataLayout &layout)>;
341 
342 /// Callback function type used to deallocate the buffers used to hold the
343 /// promoted subview.
345  std::function<LogicalResult(OpBuilder &b, Value buffer)>;
346 
347 /// Callback function type used to insert copy from original subview to
348 /// subview of the promoted region for the read operands/subview of promoted
349 /// region to original subview for the results. The copy has to happen from
350 /// `src` to `dst`.
352  std::function<LogicalResult(OpBuilder &b, Value src, Value dst)>;
353 
355  /// Indices of subViews to promote. If `std::nullopt`, try to promote all
356  /// operands.
357  std::optional<DenseSet<unsigned>> operandsToPromote;
360  operandsToPromote->insert(operands.begin(), operands.end());
361  return *this;
362  }
363  /// If ith element of `useFullTiles` is true the full view should be used
364  /// for the promoted buffer of the ith operand in `operandsToPromote`.
365  /// Otherwise the partial view will be used. The decision is defaulted to
366  /// `useFullTileBuffersDefault` when `useFullTileBuffers` is std::nullopt and
367  /// for operands missing from `useFullTileBuffers`.
368  std::optional<llvm::SmallBitVector> useFullTileBuffers;
370  unsigned size = useFullTiles.size();
371  llvm::SmallBitVector tmp(size, false);
372  for (unsigned i = 0; i < size; ++i)
373  tmp[i] = useFullTiles[i];
374  useFullTileBuffers = tmp;
375  return *this;
376  }
377  /// If true all operands unspecified by `useFullTileBuffers` will use the
378  /// full view, otherwise the partial view.
382  return *this;
383  }
384  /// Alignment of promoted buffer. If `std::nullopt` do not specify alignment.
385  std::optional<unsigned> alignment;
387  alignment = align;
388  return *this;
389  }
390  /// Memory space of promoted buffer. If `std::nullopt` do not specify memory
391  /// space.
392  std::optional<Attribute> memorySpace;
394  memorySpace = memorySpc;
395  return *this;
396  }
397  /// Use alloca with the default allocation scheme.
398  bool useAlloca = false;
400  useAlloca = use;
401  return *this;
402  }
403  /// Callback function to do the allocation of the promoted buffer. If
404  /// std::nullopt, then the default allocation scheme of allocating a
405  /// memref<?xi8> buffer followed by a view operation is used.
406  std::optional<AllocBufferCallbackFn> allocationFn;
407  std::optional<DeallocBufferCallbackFn> deallocationFn;
410  DeallocBufferCallbackFn const &deallocFn) {
411  allocationFn = allocFn;
412  deallocationFn = deallocFn;
413  return *this;
414  }
415  /// Callback function to do the copy of data to and from the promoted
416  /// subview. If std::nullopt then a memref.copy is used.
417  std::optional<CopyCallbackFn> copyInFn;
418  std::optional<CopyCallbackFn> copyOutFn;
420  CopyCallbackFn const &copyOut) {
421  copyInFn = copyIn;
422  copyOutFn = copyOut;
423  return *this;
424  }
425 };
426 
427 /// Split Reduction options.
429  // Ratio used to split the reduction dimension. If the ratio is <= 1,
430  // nothing will be done.
431  int64_t ratio = 0;
432  // Index where the extra dimension is added to the intermediate tensor
433  // shape.
434  unsigned index = 0;
435  // If the inner dimension after splitting is parallel or reduction.
436  bool innerParallel = false;
437 };
438 
439 /// Function signature to control reduction splitting. This returns
440 /// `SplitReductionOptions`.
441 // TODO: don't use unsigned unless doing bit manipulation.
443  std::function<SplitReductionOptions(LinalgOp op)>;
444 
445 //===----------------------------------------------------------------------===//
446 // Preconditions that ensure the corresponding transformation succeeds and can
447 // be applied as a rewrite pattern.
448 //===----------------------------------------------------------------------===//
449 
450 /// Return true if two `linalg.generic` operations with producer/consumer
451 /// relationship through `fusedOperand` can be fused using elementwise op
452 /// fusion.
453 bool areElementwiseOpsFusable(OpOperand *fusedOperand);
454 
455 /// Promote memref.subviews feeding linalg-on-buffers operations.
458 
459 /// Return success if the operation can be vectorized.
461  ArrayRef<int64_t> inputVectorSizes = {},
462  ArrayRef<bool> inputScalableVecDims = {},
463  bool vectorizeNDExtract = false);
464 
465 //===----------------------------------------------------------------------===//
466 // Transformations exposed as functional-style API calls.
467 //===----------------------------------------------------------------------===//
468 
470 
471 /// Transformation to drop unit-extent dimensions from `linalg.generic`
472 /// operations.
475 
478 
479  using ControlFnTy = std::function<SmallVector<unsigned>(Operation *)>;
481  if (auto genericOp = dyn_cast_or_null<GenericOp>(op)) {
482  return llvm::to_vector(llvm::seq<unsigned>(0, genericOp.getNumLoops()));
483  }
484  return SmallVector<unsigned>{};
485  };
486 };
487 LogicalResult dropUnitDims(RewriterBase &rewriter, GenericOp genericOp,
488  const ControlDropUnitDims &options);
489 
490 /// Fuse two `linalg.generic` operations that have a producer-consumer
491 /// relationship captured through `fusedOperand`. The method expects
492 /// that `areElementwiseOpsFusable` returns true for the given `fusedOperand`.
496  static llvm::SmallDenseSet<int>
497  getPreservedProducerResults(GenericOp producer, GenericOp consumer);
498 };
500 fuseElementwiseOps(RewriterBase &rewriter, OpOperand *fusedOperand);
501 
502 /// Try to peel and canonicalize loop `op` and return the new result.
503 /// Also applies affine_min/max bounds simplification on the fly where relevant.
504 // TODO: Add support for scf.parallel and affine.for loops.
506 
507 /// Peel 'loops' and applies affine_min/max bounds simplification on the fly
508 /// where relevant.
509 void peelLoops(RewriterBase &rewriter, ArrayRef<scf::ForOp> loops);
510 
511 /// Pad the iterator dimensions `paddingDimensions` of all `opToPad` operands
512 /// to a static bounding box. The original `opToPad` is cloned and operates on
513 /// the padded tensors.
514 ///
515 /// * "options.padToMultipleOf" indicates that each padding dimension should be
516 /// padded to the specified multiple.
517 /// * Use "options.paddingValues" and "options.packPaddings" to set padding
518 /// value and nofold attribute of the created tensor::PadOps, respectively.
519 /// * The unpadded results (extracted slice of the cloned operation) are
520 /// returned via `replacements`.
521 /// * The tensor::PadOps are returned via `padOps`.
522 /// * "options.copyBackOp" specifies the op type for copying back the unpadded
523 /// result to the original destination tensor.
524 LogicalResult rewriteAsPaddedOp(RewriterBase &rewriter, LinalgOp opToPad,
526  LinalgOp &paddedOp,
527  SmallVector<Value> &replacements,
529 
530 namespace detail {
531 
532 /// Helper struct to hold the results of building a packing loop nest.
536  GenericOp maybeTransposeOp;
537  tensor::PadOp hoistedPadOp;
538 };
539 
540 /// Build the packing loop nest required to hoist `opToHoist` above
541 /// `outermostEnclosingForOp`.
542 /// The loop nest is built just before `outermostEnclosingForOp`.
544 buildPackingLoopNest(RewriterBase &rewriter, tensor::PadOp opToHoist,
545  scf::ForOp outermostEnclosingForOp,
546  ArrayRef<int64_t> transposeVector);
547 
548 } // namespace detail
549 
550 /// Mechanically hoist padding operations on tensors by `numLoops` into a new,
551 /// generally larger tensor. This achieves packing of multiple padding ops into
552 /// a larger tensor. On success, `opToHoist` is replaced by the cloned version
553 /// in the packing loop so the caller can continue reasoning about the padding
554 /// operation. If `transposeVector` is non-empty, hoist padding introduces a
555 /// GenericOp to transpose the padded tensor before inserting it into the packed
556 /// tensor. A `transposeVector` can change the storage order of the padded
557 /// tensor but does not change the order of the pack or compute loops.
558 ///
559 /// TODO: In the future, we should consider rewriting as a tensor.pack after
560 /// hoisting since this abstraction is now available.
561 ///
562 /// Example in pseudo-mlir:
563 /// =======================
564 ///
565 /// If hoistPaddingOnTensors is called with `nLoops` = 2 on the following IR.
566 /// ```
567 /// scf.for (%i, %j, %k)
568 /// %st0 = tensor.extract_slice f(%i, %k) : ... to tensor<?x?xf32>
569 /// %0 = tensor.pad %st0 low[0, 0] high[...] {
570 /// ^bb0( ... ):
571 /// linalg.yield %pad
572 /// } : tensor<?x?xf32> to tensor<4x8xf32>
573 /// compute(%0)
574 /// ```
575 ///
576 /// IR resembling the following is produced:
577 ///
578 /// ```
579 /// scf.for (%i) {
580 /// %packed_init = tensor.empty range(%j) : tensor<?x4x8xf32>
581 /// %packed = scf.for (%k) iter_args(%p : %packed_init) {
582 /// %st0 = tensor.extract_slice f(%i, %k) : ... to tensor<?x?xf32>
583 /// %0 = tensor.pad %st0 low[0, 0] high[...] {
584 /// ^bb0( ... ):
585 /// linalg.yield %pad
586 /// } : tensor<?x?xf32> to tensor<4x8xf32>
587 /// %1 = tensor.insert_slice %0 ...
588 /// : tensor<4x8xf32> to tensor<?x4x8xf32>
589 /// scf.yield %1: tensor<?x4x8xf32>
590 /// } -> tensor<?x4x8xf32>
591 /// scf.for (%j, %k) {
592 /// %st0 = tensor.extract_slice %packed [%k, 0, 0][1, 4, 8][1, 1, 1] :
593 /// tensor<?x4x8xf32> to tensor<4x8xf32>
594 /// compute(%st0)
595 /// }
596 /// }
597 /// ```
599 hoistPaddingOnTensors(RewriterBase &rewriter, tensor::PadOp opToHoist,
600  int64_t numLoops, ArrayRef<int64_t> transposeVector,
601  tensor::PadOp &hoistedOp,
602  SmallVectorImpl<GenericOp> &transposeOps);
603 /// Calls into `hoistPaddingOnTensors` with a local IRRewriter.
605 hoistPaddingOnTensors(tensor::PadOp opToHoist, int64_t numLoops,
606  ArrayRef<int64_t> transposeVector,
607  tensor::PadOp &hoistedOp,
608  SmallVectorImpl<GenericOp> &transposeOps);
609 
610 /// Apply padding and hoisting to `linalgOp` according to the configuration
611 /// specified in `options`.
613  LinalgOp linalgOp,
615 
616 /// Split the given `op` into two parts along the given iteration space
617 /// `dimension` at the specified `splitPoint`, and return the two parts.
618 /// If the second part is statically known to be empty, do not create it
619 /// and return nullptr instead. Error state is signalled by returning
620 /// a pair of nullptrs.
621 ///
622 /// For example, the following op:
623 ///
624 /// linalg.matmul ins(%0, %1 : tensor<128x32xf32>, tensor<32x64xf32>)
625 /// outs(%2 : tensor<128x64xf32>)
626 ///
627 /// split along the first dimension at position 42 will result in:
628 ///
629 /// %3 = tensor.extract_slice %0[0, 0][42, 32][1, 1]
630 /// %4 = tensor.extract_slice %2[0, 0][42, 64][1, 1]
631 /// %5 = linalg.matmul ins(%3, %1 : tensor<42x32xf32>, tensor<32x64xf32>)
632 /// outs(%5 : tensor<42x64xf32>)
633 /// %6 = tensor.insert_slice %5 into %2[0, 0][42, 64][1, 1]
634 ///
635 /// %7 = tensor.extract_slice %0[42, 0][86, 32][1, 1]
636 /// %8 = tensor.extract_slice %6[42, 0][86, 64][1, 1]
637 /// %9 = linalg.matmul ins(%7, %1 : tensor<86x32xf32>, tensor<32x64xf32>)
638 /// outs(%8 : tensor<86x64xf32>)
639 /// tensor.insert_slice %5 into %6[42, 0][86, 64][1, 1]
640 ///
641 /// Note that there is no simplification other than constant propagation applied
642 /// to slice extraction and insertion.
643 std::pair<TilingInterface, TilingInterface> splitOp(RewriterBase &rewriter,
644  TilingInterface op,
645  unsigned dimension,
646  OpFoldResult splitPoint);
647 
648 /// Perform standalone tiling of a single LinalgOp by `tileSizes`.
649 /// and permute the loop nest according to `interchangeVector`
650 /// The permutation is expressed as a list of integers that specify
651 /// the new ordering of the loop nest. The length of `interchangeVector`
652 /// must be equal to the length of `tileSizes`.
653 /// An empty vector is interpreted as the identity permutation and the
654 /// transformation returns early.
655 ///
656 /// Return a struct containing the tiled loops in the specified order
657 /// and the cloned op if successful, std::nullopt otherwise.
658 ///
659 /// E.g. the permutation `(i,j,k) -> (j,k,i)` is expressed by
660 /// `interchangeVector = [1,2,0]`. All values in `interchangeVector` must be
661 /// integers, in the range 0..`tileSizes.size()` without duplications
662 /// (i.e. `[1,1,2]` is an invalid permutation).
664  LinalgOp op;
667 };
670 
671 /// Interchange the `iterator_types` and `iterator_maps` dimensions and adapts
672 /// the index accesses of `op`. This is an in-place transformation controlled
673 /// by `interchangeVector`. An empty vector is interpreted as the identity
674 /// permutation and the transformation returns early.
675 ///
676 /// E.g. the permutation `(i,j,k) -> (j,k,i)` is expressed with
677 /// `interchangeVector = [1,2,0]`. All values in `interchangeVector` must be
678 /// integers, in the range 0..`op.rank` without duplications
679 /// (i.e. `[1,1,2]` is an invalid permutation).
680 ///
681 /// Return failure if the permutation is not valid.
683  GenericOp genericOp,
684  ArrayRef<unsigned> interchangeVector);
685 
686 /// Create a GenericOp from the given named operation `namedOp` and replace
687 /// namedOp.
688 /// Return failure if `namedOp` is a GenericOp or misses a region builder.
690  LinalgOp namedOp);
691 
692 /// Create a namedOp from the given GenericOp and replace the GenericOp.
693 /// Currently we can specialize only trivial linalg copy operations.
695  GenericOp genericOp);
696 
697 /// Create a new buffer using the `allocationFn` provided. The size of this
698 /// buffer is the smallest constant bounding size along each dimension that
699 /// can be computed for the size of the result of `subView`. Returns the
700 /// allocated buffer as `fullLocalView` and the view that matches the size of
701 /// the result of subview operation as `partialLocalView`.
705 };
707 promoteSubviewAsNewBuffer(OpBuilder &b, Location loc, memref::SubViewOp subView,
708  const AllocBufferCallbackFn &allocationFn,
709  DataLayout &layout);
710 
711 /// Promote the `subViews` into a new buffer allocated at the insertion point
712 /// `b`. Promotion occurs in 3 steps:
713 /// 1. Create a new buffer for a full tile (i.e. not clipped at the
714 /// boundary).
715 /// 2. Take a full view on the buffer.
716 /// 3. Take a partial slice of the full view in step 2. and copy into it.
717 ///
718 /// Return the modified linalg op (the modification happens in place) as well
719 /// as all the copy ops created.
722 
723 /// Allocate the subview in the GPU workgroup memory.
724 std::optional<Value> allocateWorkgroupMemory(OpBuilder &builder,
725  memref::SubViewOp subview,
726  ArrayRef<Value> sizeBounds,
727  DataLayout &);
728 
729 /// In case of GPU group memory there is no need to deallocate.
731 
732 /// Create Memref copy operations and add gpu barrier guards before and after
733 /// the copy operation to ensure data integrity.
735 
736 /// Allocate the subview in the GPU private memory.
737 std::optional<Value> allocateGPUPrivateMemory(OpBuilder &builder,
738  memref::SubViewOp subview,
739  ArrayRef<Value> sizeBounds,
740  DataLayout &);
741 
742 /// Normal copy to between src and dst.
744 
745 /// In case of GPU private memory there is no need to deallocate since the
746 /// memory is freed when going outside of the scope.
748 
749 /// Emit a suitable vector form for an operation. If provided,
750 /// `inputVectorSizes` are used to vectorize this operation. `inputVectorSizes`
751 /// must match the rank of the iteration space of the operation and the sizes
752 /// must be smaller or equal than their counterpart interation space sizes, if
753 /// static. `inputVectorShapes` also allows the vectorization of operations with
754 /// dynamic shapes.
756  ArrayRef<int64_t> inputVectorSizes = {},
757  ArrayRef<bool> inputScalableVecDims = {},
758  bool vectorizeNDExtract = false,
759  bool flatten1DDepthwiseConv = false);
760 
761 /// Emit a suitable vector form for a Copy op with fully static shape.
762 LogicalResult vectorizeCopy(RewriterBase &builder, memref::CopyOp copyOp);
763 
764 /// Emit a loop nest of `scf.for` with the proper body for `linalgOp`.
765 FailureOr<LinalgLoops> linalgOpToLoops(RewriterBase &rewriter,
766  LinalgOp linalgOp);
767 
768 /// Emit a loop nest of `scf.parallel` with the proper body for `linalgOp`.
769 FailureOr<LinalgLoops> linalgOpToParallelLoops(RewriterBase &rewriter,
770  LinalgOp linalgOp);
771 
772 /// Emit a loop nest of `affine.for` with the proper body for `linalgOp`.
773 FailureOr<LinalgLoops> linalgOpToAffineLoops(RewriterBase &rewriter,
774  LinalgOp linalgOp);
775 
776 /// Creates a number of ranges equal to the number of non-zero in `tileSizes`.
777 /// One for each loop of the LinalgOp that is tiled. The `tileSizes` argument
778 /// has one entry per surrounding loop. It uses zero as the convention that a
779 /// particular loop is not tiled. This convention simplifies implementations
780 /// by avoiding affine map manipulations. The returned ranges correspond to
781 /// the loop ranges, in the proper order, that are tiled and for which new
782 /// loops will be created. Also the function returns a map from loop indices
783 /// of the LinalgOp to the corresponding non-empty range indices of newly
784 /// created loops.
786 std::tuple<SmallVector<Range, 4>, LoopIndexToRangeIndexMap>
788  ArrayRef<OpFoldResult> allShapeSizes,
789  ArrayRef<OpFoldResult> allTileSizes);
790 
791 namespace detail {
792 template <typename T>
794  /// Tile sizes.
796  /// Number of tiles associated with each size.
798 };
799 } // namespace detail
800 
801 /// A description of a multi-size tiling comprising tile sizes and numbers of
802 /// tiles, expressed as Values which may or may not be constant. Multi-size
803 /// currently means two-size.
805  : public detail::MultiSizeSpecificationBase<Value> {};
807  : public detail::MultiSizeSpecificationBase<int64_t> {};
808 
809 /// Emits the IR computing the multi-sized tiling specification with two tile
810 /// sizes not exceeding `targetSize`, each divisible by `sizeDivisor`, such
811 /// that there exist numbers of tiles with these sizes that fully cover the
812 /// given iteration space `dimension` of the structured `op`.
813 ///
814 /// The computation is as follows:
815 ///
816 /// b = originalTripCount floordiv sizeDivisor
817 /// t = (targetSize + sizeDivisor - 1) floordiv sizeDivisor
818 /// d = (b + t - 1) floordiv t
819 /// s = (b floordiv d) * sizeDivisor
820 /// v = b % d
821 /// u = d - v
822 ///
823 /// where the tile sizes are `s` and `s` + `sizeDivisor`, and the numbers of
824 /// the corresponding tiles are `u` and `v`, respectively. Alternatively,
825 ///
826 /// s * u + (s + sizeDivisor) * v == original size,
827 /// where s mod sizeDivisor = 0.
828 ///
829 /// Expects all values to be positive. In some cases with the target tile size
830 /// sufficiently close to the dimension shape and non-unit divisor, it is
831 /// impossible to compute such sizes. If `emitAssertion` is set, also emit the
832 /// assertion that size computation succeeded.
833 ///
834 /// Returns the specification consisting of both tile values and the number of
835 /// tiles of each size.
837 computeMultiTileSizes(OpBuilder &builder, LinalgOp op, unsigned dimension,
838  OpFoldResult targetSize, OpFoldResult divisor,
839  bool emitAssertions = true);
841 computeStaticMultiTileSizes(LinalgOp op, unsigned dimension, int64_t targetSize,
842  int64_t divisor);
843 
844 /// Rewrite a TilingInterface `op` to a tiled `scf.forall`, applying
845 /// tiling by `numThreads`.
846 /// If non-empty, the `mapping` is added as an attribute to the
847 /// resulting `scf.forall`.
848 /// Zero tile sizes indicate that the dimension is not tiled, and can be
849 /// thought of as tiling by the full size of data. It is the user's
850 /// responsibility to ensure that `numThreads` is a valid tiling specification
851 /// (i.e. that only tiles parallel dimensions, e.g. in the Linalg case).
855 };
857  TilingInterface op,
858  ArrayRef<OpFoldResult> numThreads,
859  std::optional<ArrayAttr> mapping);
860 
861 /// Same as `tileToForallOp`, but calculate the number of threads
862 /// required using the given tileSizes.
864 tileToForallOpUsingTileSizes(RewriterBase &builder, TilingInterface op,
865  ArrayRef<OpFoldResult> tileSizes,
866  std::optional<ArrayAttr> mapping);
867 
868 /// Transformation information returned after reduction tiling.
870  /// The partial reduction tiled op generated.
872  /// The final reduction operation merging all the partial reductions.
874  /// The op initializing the tensor used for partial reductions.
876  /// The `scf.forall` operation that iterate over the tiles.
877  scf::ForallOp loops;
878 };
879 
880 /// Method to tile a reduction to parallel iterations computing partial
881 /// reductions. After the loop all the partial reduction are merged into a final
882 /// reduction. For example for the following sequence
883 ///
884 /// ```mlir
885 /// %0 = linalg.generic %in ["parallel", "reduction"]
886 /// : tensor<7x9xf32> -> tensor<7xf32>
887 /// ```
888 ///
889 /// into:
890 ///
891 /// ```mlir
892 /// %0 = linalg.fill ... : tensor<7x4xf32>
893 /// %1 = scf.forall (%iv) in (%c4) shared_outs(%arg0 = %0)
894 /// -> (tensor<7x4xf32>) {
895 /// %2 = tensor.extract_slice %arg3 : tensor<7x4xf32> to tensor<7xf32>
896 /// %3 = tensor.extract_slice %in : tensor<7x9xf32> -> tensor<7x?xf32>
897 /// %4 = linalg.generic %2, %3 ["parallel", "reduction"]
898 /// : tensor<7x?xf32> -> tensor<7xf32>
899 /// %5 = tensor.insert_slice %3, %arg0[0, %iv] : tensor<7x4xf32>
900 /// }
901 /// %6 = linalg.generic %1 ["parallel", "reduction"]
902 /// : tensor<7x4xf32> -> tensor<7xf32>
903 /// ```
905 tileReductionUsingForall(RewriterBase &b, PartialReductionOpInterface op,
906  ArrayRef<OpFoldResult> numThreads,
907  ArrayRef<OpFoldResult> tileSizes = {},
908  std::optional<ArrayAttr> mapping = std::nullopt);
909 
910 /// All indices returned by IndexOp should be invariant with respect to
911 /// tiling. Therefore, if an operation is tiled, we have to transform the
912 /// indices accordingly, i.e. offset them by the values of the corresponding
913 /// induction variables that are captured implicitly in the body of the op.
914 ///
915 /// Example. `linalg.generic` before tiling:
916 ///
917 /// #id_2d = (i, j) -> (i, j)
918 /// #pointwise_2d_trait = {
919 /// indexing_maps = [#id_2d, #id_2d],
920 /// iterator_types = ["parallel", "parallel"]
921 /// }
922 /// linalg.generic #pointwise_2d_trait %operand, %result {
923 /// ^bb0(%operand_in: f32, %result_in: f32):
924 /// %i = linalg.index 0 : index
925 /// %j = linalg.index 1 : index
926 /// <some operations that use %i, %j>
927 /// }: memref<50x100xf32>, memref<50x100xf32>
928 ///
929 /// After tiling pass with tiles sizes 10 and 25:
930 ///
931 /// #strided = (i, j)[s0, s1, s2] -> (i * s1 + s0 + j * s2)
932 ///
933 /// %c1 = arith.constant 1 : index
934 /// %c0 = arith.constant 0 : index
935 /// %c25 = arith.constant 25 : index
936 /// %c10 = arith.constant 10 : index
937 /// operand_dim_0 = dim %operand, 0 : memref<50x100xf32>
938 /// operand_dim_1 = dim %operand, 1 : memref<50x100xf32>
939 /// scf.for %k = %c0 to operand_dim_0 step %c10 {
940 /// scf.for %l = %c0 to operand_dim_1 step %c25 {
941 /// %4 = memref.subview %operand[%k, %l][%c10, %c25][%c1, %c1]
942 /// : memref<50x100xf32> to memref<?x?xf32, #strided>
943 /// %5 = memref.subview %result[%k, %l][%c10, %c25][%c1, %c1]
944 /// : memref<50x100xf32> to memref<?x?xf32, #strided>
945 /// linalg.generic pointwise_2d_trait %4, %5 {
946 /// ^bb0(%operand_in: f32, %result_in: f32):
947 /// %i = linalg.index 0 : index
948 /// %j = linalg.index 1 : index
949 /// // Indices `k` and `l` are implicitly captured in the body.
950 /// %transformed_i = arith.addi %i, %k : index // index `i` is offset by
951 /// %k %transformed_j = arith.addi %j, %l : index // index `j` is offset
952 /// by %l
953 /// // Every use of %i, %j is replaced with %transformed_i,
954 /// %transformed_j <some operations that use %transformed_i,
955 /// %transformed_j>
956 /// }: memref<?x?xf32, #strided>, memref<?x?xf32, #strided>
957 /// }
958 /// }
959 ///
960 /// TODO: Investigate whether mixing implicit and explicit indices
961 /// does not lead to losing information.
962 void transformIndexOps(RewriterBase &b, LinalgOp op,
964  const LoopIndexToRangeIndexMap &loopIndexToRangeIndex);
965 
966 /// Apply transformation to split the single linalg op reduction into a
967 /// parallel and reduction dimension. Then create a new linalg.generic op
968 /// doing the rest of the reduction. Return the new linalg op with an extra
969 /// parallel dimension or failure if the transformation didn't happen.
970 ///
971 /// Example:
972 /// ```
973 /// %r = linalg.generic {indexing_maps = [affine_map<(d0) -> (d0)>,
974 /// affine_map<(d0) -> ()>],
975 /// iterator_types = ["reduction"]}
976 /// ins(%in : tensor<32xf32>)
977 /// outs(%out : tensor<f32>) {
978 /// ^bb0(%arg1: f32, %arg2: f32):
979 /// %y = arith.addf %arg1, %arg2 : f32
980 /// linalg.yield %y : f32
981 /// } -> tensor<f32>
982 /// ```
983 /// To:
984 /// ```
985 /// %cst = arith.constant 0.000000e+00 : f32
986 /// %0 = tensor.expand_shape %in [[0, 1]] : tensor<32xf32> into
987 /// tensor<4x8xf32> %1 = tensor.empty [4] : tensor<4xf32> %2 = linalg.fill
988 /// ins(%cst : f32) outs(%1 : tensor<4xf32>) -> tensor<4xf32> %3 =
989 /// linalg.generic {indexing_maps = [affine_map<(d0, d1) -> (d0, d1)>,
990 /// affine_map<(d0, d1) -> (d0)>],
991 /// iterator_types = ["parallel", "reduction"]}
992 /// ins(%0 : tensor<4x8xf32>) outs(%2 : tensor<4xf32>) {
993 /// ^bb0(%arg3: f32, %arg5: f32):
994 /// %5 = arith.addf %arg3, %arg4 : f32
995 /// linalg.yield %5 : f32
996 /// } -> tensor<4xf32>
997 /// %r = linalg.generic {indexing_maps = [affine_map<(d0) -> (d0)>,
998 /// affine_map<(d0) -> ()>],
999 /// iterator_types = ["reduction"]}
1000 /// ins(%3 : tensor<4xf32>) outs(%out : tensor<f32>) {
1001 /// ^bb0(%arg3: f32, %arg4: f32):
1002 /// %5 = arith.addf %arg3, %arg4 : f32
1003 /// linalg.yield %5 : f32
1004 /// } -> tensor<f32>
1005 /// ```
1008  FillOp fillOp;
1009  LinalgOp splitLinalgOp;
1011 };
1013 splitReduction(RewriterBase &b, LinalgOp op,
1014  const ControlSplitReductionFn &controlSplitReductionFn,
1015  bool useAlloc = false);
1016 
1017 /// Scaling-based implementation of the split reduction transformation.
1018 /// Instead of introducing an ExpandShapeOp, this rewrites a reduction
1019 /// dimension `k` into `k * scale + kk`.
1020 ///
1021 /// Example:
1022 /// ```
1023 /// %0 = linalg.matmul ins(%A, %B: tensor<16x256xf32>, tensor<256x32xf32>)
1024 /// outs(%C: tensor<16x32xf32>) -> tensor<16x32xf32>
1025 /// ```
1026 ///
1027 /// Is transformed to:
1028 ///
1029 /// ```
1030 /// #map0 = affine_map<(d0, d1, d2, d3) -> (d0, d2 * 4 + d3)>
1031 /// #map1 = affine_map<(d0, d1, d2, d3) -> (d2 * 4 + d3, d1)>
1032 /// #map2 = affine_map<(d0, d1, d2, d3) -> (d2, d3)>
1033 /// #map3 = affine_map<(d0, d1, d2, d3) -> (d0, d1, d2)>
1034 /// #map4 = affine_map<(d0, d1, d2) -> (d0, d1, d2)>
1035 /// #map5 = affine_map<(d0, d1, d2) -> (d0, d1)>
1036 /// %0 = tensor.empty [16, 32, 64] : tensor<16x32x64xf32>
1037 /// %cst = arith.constant 0.000000e+00 : f32
1038 /// %1 = linalg.fill ins(%cst : f32) outs(%0 : tensor<16x32x64xf32>) ->
1039 /// tensor<16x32x64xf32>
1040 /// %2 = tensor.empty [64, 4] : tensor<64x4xi1>
1041 ///
1042 /// %3 = linalg.generic {indexing_maps = [#map0, #map1, #map2, #map3],
1043 /// iterator_types = ["parallel", "parallel", "parallel", "reduction"]}
1044 /// ins(%A, %B, %2 : tensor<16x256xf32>, tensor<256x32xf32>,
1045 /// tensor<64x4xi1>)
1046 /// outs(%1 : tensor<16x32x64xf32>) {
1047 /// ^bb0(%arg3: f32, %arg4: f32, %arg5: i1, %arg6: f32):
1048 /// %5 = arith.mulf %arg3, %arg4 : f32
1049 /// %6 = arith.addf %arg6, %5 : f32
1050 /// linalg.yield %6 : f32
1051 /// } -> tensor<16x32x64xf32>
1052 ///
1053 /// %4 = linalg.generic {indexing_maps = [#map4, #map5],
1054 /// iterator_types = ["parallel", "parallel", "reduction"]}
1055 // ins(%3 : tensor<16x32x64xf32>)
1056 /// outs(%C : tensor<16x32xf32>) {
1057 /// ^bb0(%arg3: f32, %arg4: f32):
1058 /// %5 = arith.addf %arg3, %arg4 : f32
1059 /// linalg.yield %5 : f32
1060 /// } -> tensor<16x32xf32>
1061 ///
1062 /// return %4 : tensor<16x32xf32>
1063 /// ```
1065 splitReductionByScaling(RewriterBase &b, LinalgOp op,
1066  const ControlSplitReductionFn &controlSplitReductionFn,
1067  bool useAlloc = false);
1068 
1069 /// Return `true` if a given sequence of dimensions are contiguous in the
1070 /// range of the specified indexing map.
1072 /// Return `true` if all sequences of dimensions specified in `dimSequences` are
1073 /// contiguous in all the ranges of the `maps`.
1075  ArrayRef<ReassociationIndices> dimSequences);
1076 
1077 /// Collapses dimensions of linalg.generic/linalg.copy operation. A precondition
1078 /// to calling this method is that for each list in `foldedIterationDim`, the
1079 /// sequence of dimensions is contiguous in domains of all `indexing_maps` of
1080 /// the `linalgOp`. This can be checked using `areDimSequencePreserved` method.
1081 /// When valid, the method also collapses the operands of the op. Returns
1082 /// replacement values of the results of the original `linalgOp` by inserting
1083 /// reshapes to get back values of compatible types.
1084 template <typename LinalgType>
1086 collapseOpIterationDims(LinalgType op,
1087  ArrayRef<ReassociationIndices> foldedIterationDims,
1088  RewriterBase &rewriter);
1089 
1091  tensor::PadOp padOp;
1092  tensor::ExpandShapeOp expandShapeOp;
1093  linalg::TransposeOp transposeOp;
1094 };
1095 
1096 /// Rewrite pack as pad + reshape + transpose.
1098  tensor::PackOp packOp);
1099 
1101  tensor::EmptyOp emptyOp;
1102  linalg::TransposeOp transposeOp;
1103  tensor::CollapseShapeOp collapseShapeOp;
1104  tensor::ExtractSliceOp extractSliceOp;
1105 };
1106 
1107 /// Rewrite pack as empty + transpose + reshape + extract_slice.
1109  tensor::UnPackOp unPackOp);
1110 
1111 /// Struct to hold the result of a `pack` call.
1112 struct PackResult {
1114  linalg::LinalgOp packedLinalgOp;
1116 };
1117 /// Implement packing of a single LinalgOp by `packedSizes`.
1118 /// There must be one packedSizes entry per `linalgOp` iterator.
1119 /// Return the packed Linalg op on success, failure otherwise.
1120 FailureOr<PackResult> pack(RewriterBase &rewriter, linalg::LinalgOp linalgOp,
1121  ArrayRef<OpFoldResult> packedSizes);
1122 
1123 /// Struct to hold the result of a `packTranspose` call.
1125  tensor::PackOp transposedPackOp;
1126  linalg::LinalgOp transposedLinalgOp;
1127  tensor::UnPackOp transposedUnPackOp;
1128 };
1129 /// Transpose a single PackOp -> LinalgOp -> UnPackOp chain and return the
1130 /// transposed PackOp -> LinalgOp -> UnPackOp chain after replacements.
1131 /// Return failure if either:
1132 /// 1. the `packOp` does not have the `linalgOp` as its unique use.
1133 /// 2. the `maybeUnPackOp`, if specified must be a consumer of the result tied
1134 /// to the unique `packOp` use.
1135 /// 3. `outerPerm` (resp. `innerPerm`) must be valid permutations of
1136 /// `packOp.getOuterDimsPerm` (resp. `packOp.getInnerDimsPerm`) or empty.
1138 packTranspose(RewriterBase &rewriter, tensor::PackOp packOp,
1139  linalg::LinalgOp linalgOp, tensor::UnPackOp maybeUnPackOp,
1140  ArrayRef<int64_t> outerPerm, ArrayRef<int64_t> innerPerm);
1141 
1142 /// Pack a LinalgOp by greedily inferring matmul dimensions (m, n, k) where m
1143 /// and n are proper parallel dimensions and k is a proper reduction
1144 /// dimension. Packing occurs by rewriting the op as a linalg.generic and
1145 /// calling linalg::pack by `mnkPackedSizes`. The order of the packed
1146 /// dimensions is customizable: the `mnkOrder` is a permutation of {0, 1, 2}
1147 /// to reorder {m, n, k} into one of the 8 possible forms. The outer
1148 /// dimensions of the operands are not permuted at this time, this is left for
1149 /// future work.
1151 packMatmulGreedily(RewriterBase &rewriter, LinalgOp linalgOp,
1152  ArrayRef<OpFoldResult> mnkPackedSizes,
1153  ArrayRef<int64_t> mnkPaddedSizesNextMultipleOf,
1154  ArrayRef<int64_t> mnkOrder);
1155 
1156 /// Rewrite tensor.from_elements to linalg.generic.
1159  tensor::FromElementsOp fromElementsOp);
1160 
1161 /// Rewrite tensor.generate to linalg.generic.
1164  tensor::GenerateOp generateOp);
1165 
1166 /// Rewrite tensor.pad to linalg.generic + tensor.insert_slice.
1168  tensor::PadOp padOp);
1169 
1170 /// Convert linalg.conv_2d_nhwc_hwcf into linalg.generic (for img2col packing)
1171 /// and linalg.matmul.
1172 ///
1173 /// A convolution operation can be written as a matrix-matrix multiplication by
1174 /// unfolding the cross-correlation between input and filter and explicitly copy
1175 /// overlapped sliding window inputs.
1176 ///
1177 /// Consider 2D input X with single channel input and output and 2x2 filter W:
1178 /// [x(0, 0) , x(0, 1) , ..., x(0, n) ]
1179 /// [x(1, 0) , x(1, 1) , ..., x(1, n) ]
1180 /// [. , . ,. , . ] [w(0, 0), w(0, 1)]
1181 /// [. , . , . , . ] (conv) [w(1, 0), w(1, 1)]
1182 /// [. , . , ., . ]
1183 /// [x(n-1, 0), x(n-1, 1), ..., x(n-1, n-1)]
1184 ///
1185 /// The packed input data (img2col) is a matrix with |rows| = output spatial
1186 /// size, |columns| = filter spatial size. To compute the output Y(i, j) we need
1187 /// to calculate the dot product between filter window at input X(x, y)) and the
1188 /// filter which will look like the following where r.h.s is the img2col matrix
1189 /// and l.h.s is the flattened filter:
1190 ///
1191 /// [x(0,0), x(0,1), x(1,0), x(1,1)]
1192 /// [x(0,1), x(1,1), x(0,2), x(1,2)] (matmul) [w(0,0), w(0,1), w(1,0), w(1,1)]
1193 /// [x(0,1), x(1,1), x(0,2), x(1,2)]
1194 /// [ . , . , . , . ]
1195 ///
1196 /// In general for 2D case with (N, H, W, C) input and (Kh, Kw, C, D) filter
1197 /// and output (N, Ho, Wo, D) the convolution is the following matrix-matrix
1198 /// multiplication (Ho x Wo, Kh x Kw x C) * (Kh x Kw x C, D) for each input in
1199 /// the N input. For the case where N > 1 its a batched matrix-matrix
1200 /// multiplication.
1201 ///
1202 /// On success, return both the operation that produces the img2col tensor and
1203 /// the final operation of the sequence that replaces the original convolution.
1205 rewriteInIm2Col(RewriterBase &rewriter, linalg::Conv2DNhwcHwcfOp convOp);
1206 
1207 /// Same as the above but for Fhwc channel orderings in the filter. In this case
1208 /// the matrix multiplication is actually a row-wise dot-product rather than a
1209 /// row-column dot-product. This is to avoid transposing the filter matrix which
1210 /// would be required for a regular matrix multiplication to produce the correct
1211 /// output dimensions.
1213 rewriteInIm2Col(RewriterBase &rewriter, linalg::Conv2DNhwcFhwcOp convOp);
1214 
1215 /// Similar to rewriteInIm2Col with linalg::Conv2DNhwcHwcfOp except there is no
1216 /// reduction among the input channels so each convolution can be a
1217 /// matrix-vector product and by transposing both input filter so channels are
1218 /// outer most the computation is a batched matrix-vector product.
1220 rewriteInIm2Col(RewriterBase &rewriter,
1221  linalg::DepthwiseConv2DNhwcHwcOp convOp);
1222 
1223 /// Similar to rewriteInIm2Col with linalg::Conv2DNhwcHwcfOp except because the
1224 /// channels are to the left of the image shape dimensions, the position of the
1225 /// contraction dimension in the resulting matmul is reversed. This swaps the
1226 /// LHS and RHS of the matmul when compared with nhwc (i.e. (D, C x Kh x Kw) *
1227 /// (C x Kh x Kw, Ho x Wo))
1229 rewriteInIm2Col(RewriterBase &rewriter, linalg::Conv2DNchwFchwOp convOp);
1230 
1231 /// Convert linalg.conv_2d_nhwc_fhwc(_q) to linalg.conv_2d_nhwc_hwcf(_q) by
1232 /// materializing transpose.
1234  linalg::Conv2DNhwcFhwcOp op);
1236  linalg::Conv2DNhwcFhwcQOp op);
1237 
1238 //===----------------------------------------------------------------------===//
1239 // Rewrite patterns wrapping transformations.
1240 // TODO: every single such pattern should be a close to noop wrapper around a
1241 // functional-stye API call.
1242 //===----------------------------------------------------------------------===//
1243 
1244 /// Rewrites 2-D convolution ops with size-1 window dimensions into 1-D
1245 /// convolution ops.
1246 template <typename Conv2DOp, typename Conv1DOp>
1248  : public OpRewritePattern<Conv2DOp> {
1250 
1252  PatternRewriter &rewriter) const;
1253 
1255  PatternRewriter &rewriter) const override {
1256  return returningMatchAndRewrite(convOp, rewriter);
1257  }
1258 };
1259 
1260 extern template struct DownscaleSizeOneWindowed2DConvolution<Conv2DNhwcHwcfOp,
1261  Conv1DNwcWcfOp>;
1262 extern template struct DownscaleSizeOneWindowed2DConvolution<Conv2DNchwFchwOp,
1263  Conv1DNcwFcwOp>;
1264 
1265 /// Rewrites 2-D depthwise convolution ops with size-1 (w, kw) or (h, kh)
1266 /// dimensions into 1-D depthwise convolution ops.
1268  : public OpRewritePattern<DepthwiseConv2DNhwcHwcOp> {
1270  PatternBenefit benefit = 1)
1271  : OpRewritePattern<DepthwiseConv2DNhwcHwcOp>(context, benefit) {}
1272 
1274  returningMatchAndRewrite(DepthwiseConv2DNhwcHwcOp convOp,
1275  PatternRewriter &rewriter) const;
1276 
1277  LogicalResult matchAndRewrite(DepthwiseConv2DNhwcHwcOp convOp,
1278  PatternRewriter &rewriter) const override {
1279  return returningMatchAndRewrite(convOp, rewriter);
1280  }
1281 };
1282 
1283 struct DownscaleConv2DOp final : public OpRewritePattern<Conv2DOp> {
1285  : OpRewritePattern<Conv2DOp>(context, benefit) {}
1286 
1288  PatternRewriter &rewriter) const;
1289 
1291  PatternRewriter &rewriter) const override {
1292  return returningMatchAndRewrite(convOp, rewriter);
1293  }
1294 };
1295 
1296 ///
1297 /// Linalg generalization pattern.
1298 ///
1299 /// Apply the `generalization` transformation as a pattern.
1300 /// See `generalization` for more details.
1301 //
1302 // TODO: Automatic default pattern class that just unwraps a function
1303 // returning FailureOr<GenericOp>.
1305  : public OpInterfaceRewritePattern<LinalgOp> {
1307 
1308  /// `matchAndRewrite` implementation that returns the significant
1309  /// transformed pieces of IR.
1311  returningMatchAndRewrite(LinalgOp op, PatternRewriter &rewriter) const {
1312  return generalizeNamedOp(rewriter, op);
1313  }
1314 
1316  PatternRewriter &rewriter) const override {
1317  return returningMatchAndRewrite(op, rewriter);
1318  }
1319 };
1320 
1321 /// Vectorization pattern for memref::CopyOp.
1322 struct CopyVectorizationPattern : public OpRewritePattern<memref::CopyOp> {
1324 
1325  LogicalResult matchAndRewrite(memref::CopyOp copyOp,
1326  PatternRewriter &rewriter) const override;
1327 };
1328 
1330  std::function<LogicalResult(RewriterBase &, tensor::PadOp, Value)>;
1331 
1332 /// Rewrite a tensor::PadOp into a sequence of EmptyOp, FillOp and
1333 /// InsertSliceOp. For now, only constant padding values are supported.
1334 /// `OptimizeCopyFn` can be used to customize copying step optimization.
1335 struct GeneralizePadOpPattern : public OpRewritePattern<tensor::PadOp> {
1337  OptimizeCopyFn optimizeCopyFn = nullptr,
1338  PatternBenefit benefit = 1)
1339  : OpRewritePattern<tensor::PadOp>(context, benefit),
1340  optimizeCopyFn(std::move(optimizeCopyFn)) {}
1341  LogicalResult matchAndRewrite(tensor::PadOp padOp,
1342  PatternRewriter &rewriter) const override;
1343 
1344 protected:
1346  Value createFillOrGenerateOp(RewriterBase &rewriter, tensor::PadOp padOp,
1347  Value dest,
1348  const SmallVector<Value> &dynSizes) const;
1349 };
1350 
1351 /// Rewrites a tensor::PackOp into a sequence of tensor.pad + linalg.transpose +
1352 /// tensor.insert_slice ops, where the tensor::PackOp has outer dims being all
1353 /// 1s.
1355  : public OpRewritePattern<tensor::PackOp> {
1357  LogicalResult matchAndRewrite(tensor::PackOp packOp,
1358  PatternRewriter &rewriter) const override;
1359 };
1360 
1361 /// Rewrites a tensor::UnPackOp into a sequence of rank-reduced extract_slice op
1362 /// + transpose op + insert_slice op, where the tensor::UnPackOp has outer dims
1363 /// being all 1s.
1365  : public OpRewritePattern<tensor::UnPackOp> {
1367  LogicalResult matchAndRewrite(tensor::UnPackOp unpackOp,
1368  PatternRewriter &rewriter) const override;
1369 };
1370 
1371 /// Match and rewrite for the pattern:
1372 /// ```
1373 /// %alloc = ...
1374 /// [optional] %view = memref.view %alloc ...
1375 /// %subView = subview %allocOrView ...
1376 /// [optional] linalg.fill(%allocOrView, %cst) ...
1377 /// ...
1378 /// memref.copy(%in, %subView) ...
1379 /// vector.transfer_read %allocOrView[...], %cst ...
1380 /// ```
1381 /// into
1382 /// ```
1383 /// [unchanged] %alloc = ...
1384 /// [unchanged] [optional] %view = memref.view %alloc ...
1385 /// [unchanged] [unchanged] %subView = subview %allocOrView ...
1386 /// ...
1387 /// vector.transfer_read %in[...], %cst ...
1388 /// ```
1389 /// Where there is no interleaved use between memref.copy and transfer_read as
1390 /// well as no interleaved use between linalg.fill and memref.copy (if
1391 /// linalg.fill is specified).
1392 /// This is a custom rewrite to forward partial reads (with optional fills) to
1393 /// vector.transfer_read.
1395  : public OpRewritePattern<vector::TransferReadOp> {
1397 
1398  LogicalResult matchAndRewrite(vector::TransferReadOp xferOp,
1399  PatternRewriter &rewriter) const override;
1400 };
1401 
1402 /// Match and rewrite for the pattern:
1403 /// ```
1404 /// %alloc = ...
1405 /// [optional] %view = memref.view %alloc ...
1406 /// %subView = subview %allocOrView...
1407 /// ...
1408 /// vector.transfer_write %..., %allocOrView[...]
1409 /// memref.copy(%subView, %out)
1410 /// ```
1411 /// into
1412 /// ```
1413 /// [unchanged] %alloc = ...
1414 /// [unchanged] [optional] %view = memref.view %alloc ...
1415 /// [unchanged] %subView = subview %allocOrView...
1416 /// ...
1417 /// vector.transfer_write %..., %out[...]
1418 /// ```
1419 /// Where there is no interleaved use between transfer_write and memref.copy.
1420 /// This is a custom rewrite to forward partial writes to
1421 /// vector.transfer_write.
1423  : public OpRewritePattern<vector::TransferWriteOp> {
1425 
1426  LogicalResult matchAndRewrite(vector::TransferWriteOp xferOp,
1427  PatternRewriter &rewriter) const override;
1428 };
1429 
1430 /// Rewrite extract_slice(tensor.pad(x)) into tensor.pad(extract_slice(x)).
1432  : public OpRewritePattern<tensor::ExtractSliceOp> {
1433  /// A function to control pattern application and rewrite logic.
1434  ///
1435  /// The function will be given the slice op and should return:
1436  /// - std::nullopt: to fail the match and not apply the pattern;
1437  /// - true: to apply the pattern with zero slice guard;
1438  /// - false: to apply the pattern without zero slice guard.
1439  ///
1440  /// See the documentation for tensor::bubbleUpPadSlice regarding zero slice
1441  /// guard.
1442  using ControlFn = std::function<std::optional<bool>(tensor::ExtractSliceOp)>;
1443 
1445  ControlFn controlFn = nullptr,
1446  PatternBenefit benefit = 1)
1447  : OpRewritePattern(context, benefit), controlFn(std::move(controlFn)) {}
1448 
1449  LogicalResult matchAndRewrite(tensor::ExtractSliceOp sliceOp,
1450  PatternRewriter &rewriter) const override;
1451 
1452 private:
1453  ControlFn controlFn;
1454 };
1455 
1456 //===----------------------------------------------------------------------===//
1457 // Populate functions.
1458 //===----------------------------------------------------------------------===//
1459 
1460 /// Canonicalization patterns relevant to apply after tiling patterns. These
1461 /// are applied automatically by the tiling pass but need to be applied
1462 /// manually when tiling is called programmatically.
1465 
1466 /// Linalg generalization patterns
1467 
1468 /// Populates `patterns` with patterns to convert spec-generated named ops to
1469 /// linalg.generic ops.
1471 
1472 /// Linalg decompose convolutions patterns
1473 
1474 /// Populates patterns to decompose high-D convolution ops into low-D ones.
1475 /// This is a step in progressive lowering for convolution ops, afterwards we
1476 /// can vectorize the low-D convolution ops.
1478  PatternBenefit benefit = 1);
1479 
1480 /// Populates patterns to transform linalg.conv_2d_xxx operations into
1481 /// linalg.generic (for img2col packing) and linalg.matmul.
1482 /// \see rewriteInIm2Col for more details.
1484 
1485 /// Populates `patterns` with patterns that vectorize tensor.pad.
1486 /// These patterns are meant to apply in a complementary fashion. Benefits
1487 /// are used to encode a certain ordering of pattern application. To avoid
1488 /// scattering magic constants throughout the code base, the patterns must be
1489 /// added with this function. `baseBenefit` can be used to offset the benefit
1490 /// of all tensor::PadOp vectorization patterns by a certain value.
1492  PatternBenefit baseBenefit = 1);
1493 
1494 /// Populate patterns for splitting a `LinalgOp` with multiple statements within
1495 /// its payload into multiple `GenericOp` that have a single statement.
1496 /// The option `removeDeadArgsAndResults` adds patterns to remove dead arguments
1497 /// and results from the generated decomposed ops. This is default `true` since
1498 /// the core decomposition patterns relies on these clean up patterns. It is set
1499 /// to false only for testing purposes.
1501  bool removeDeadArgsAndResults = true);
1502 
1503 /// Populate patterns that convert non-destination-style ops to destination
1504 /// style ops.
1506 
1507 /// Populate patterns for vectorizing low-D convolution ops. This is a step in
1508 /// progressive lowering for convolution ops, it assume high-D convolution ops
1509 /// were decomposed previously.
1511  PatternBenefit benefit = 1);
1512 
1513 /// Populate patterns that convert `ElementwiseMappable` ops to linalg
1514 /// parallel loops.
1516 
1517 /// Populate patterns that are only useful in the context of sparse tensors.
1519 
1520 /// Function type which is used to control when to stop fusion. It is expected
1521 /// that OpOperand is not modified in the callback. The OpOperand is not marked
1522 /// as const to allow callers to use non-const methods.
1523 using ControlFusionFn = std::function<bool(OpOperand *fusedOperand)>;
1524 
1525 /// Patterns for fusing linalg operation on tensors.
1526 
1527 /// Pattern to fuse `linalg.generic` -> `linalg.generic` operations
1528 /// when both operations are fusable elementwise operations.
1530  RewritePatternSet &patterns,
1531  const ControlFusionFn &controlElementwiseOpFusion);
1532 
1533 /// Function type which is used to control propagation of tensor.pack/unpack
1534 /// ops.
1535 using ControlPropagationFn = std::function<bool(Operation *op)>;
1536 
1537 /// Patterns to bubble up or down data layout ops across other operations.
1539  RewritePatternSet &patterns,
1540  const ControlPropagationFn &controlPackUnPackPropagation);
1541 
1542 /// Pattern to remove dead operands and results of `linalg.generic` operations.
1543 /// This is effectively DCE for a linalg op.
1545 
1546 /// Patterns to promote inputs to outputs and remove unused inputs of
1547 /// `linalg.generic` ops.
1549 
1550 /// Function type to control generic op dimension collapsing. It is expected
1551 /// to return an array of `ReassociationIndices` representing dimensions that
1552 /// should be merged.
1554  std::function<SmallVector<ReassociationIndices>(linalg::LinalgOp)>;
1555 
1556 /// Pattern to collapse dimensions in a linalg.generic op. This will collapse
1557 /// tensor operands when needed and expand back the result tensors.
1559  RewritePatternSet &patterns,
1560  const GetCollapsableDimensionsFn &controlCollapseDimensions);
1561 
1562 /// Patterns to fold an expanding (collapsing) tensor_reshape operation with its
1563 /// producer (consumer) generic operation by expanding the dimensionality of the
1564 /// loop in the generic op.
1566  RewritePatternSet &patterns, const ControlFusionFn &controlFoldingReshapes);
1567 
1568 /// Patterns to fold an expanding tensor.expand_shape operation with its
1569 /// producer generic operation by collapsing the dimensions of the generic op.
1571  RewritePatternSet &patterns, const ControlFusionFn &controlFoldingReshapes);
1572 
1573 /// Patterns to constant fold Linalg operations.
1575  const ControlFusionFn &controlFn);
1576 
1577 /// Pattern to fuse a `tensor.pad` operation with the producer of its source,
1578 /// if the producer is a `linalg` operation with all parallel iterator types.
1580  RewritePatternSet &patterns);
1581 
1582 /// Patterns to convert from one named op to another. These can be seen as
1583 /// canonicalizations of named ops into another named op.
1585 
1586 /// Patterns to fold unit-extent dimensions in operands/results of linalg ops on
1587 /// tensors via reassociative reshape ops.
1590 
1591 /// A pattern that converts init operands to input operands.
1593 
1594 /// Patterns that are used to inline constant operands into linalg generic ops.
1596 
1597 /// Patterns that are used to bubble up extract slice op above linalg op.
1599 
1600 /// Adds patterns that waps tensor.extract_slice(linalg.fill(%cst, %init)) into
1601 /// linalg.fill(%cst, tensor.extract_slice(%init)).
1603 
1604 /// Patterns to apply `splitReduction` below.
1606  RewritePatternSet &patterns,
1607  const ControlSplitReductionFn &controlSplitReductionFn,
1608  bool useAlloc = false);
1609 
1610 } // namespace linalg
1611 } // namespace mlir
1612 
1613 #endif // MLIR_DIALECT_LINALG_TRANSFORMS_TRANSFORMS_H
static llvm::ManagedStatic< PassManagerOptions > options
A multi-dimensional affine map Affine map's are immutable like Type's, and they are uniqued.
Definition: AffineMap.h:47
Attributes are known-constant values of operations.
Definition: Attributes.h:25
The main mechanism for performing data layout queries.
This class provides support for representing a failure result, or a valid value of type T.
Definition: LogicalResult.h:78
This class defines the main interface for locations in MLIR and acts as a non-nullable wrapper around...
Definition: Location.h:63
MLIRContext is the top-level object for a collection of MLIR operations.
Definition: MLIRContext.h:60
This class helps build Operations.
Definition: Builders.h:209
This class represents a single result from folding an operation.
Definition: OpDefinition.h:266
This class represents an operand of an operation.
Definition: Value.h:263
Operation is the basic unit of execution within MLIR.
Definition: Operation.h:88
This class represents the benefit of a pattern match in a unitless scheme that ranges from 0 (very li...
Definition: PatternMatch.h:33
A special type of RewriterBase that coordinates the application of a rewrite pattern on the current I...
Definition: PatternMatch.h:748
This class coordinates the application of a rewrite on a set of IR, providing a way for clients to tr...
Definition: PatternMatch.h:399
This class represents an instance of an SSA value in the MLIR system, representing a computable value...
Definition: Value.h:96
FailureOr< PackingResult > buildPackingLoopNest(RewriterBase &rewriter, tensor::PadOp opToHoist, scf::ForOp outermostEnclosingForOp, ArrayRef< int64_t > transposeVector)
Build the packing loop nest required to hoist opToHoist above outermostEnclosingForOp.
void populateLinalgNamedOpConversionPatterns(RewritePatternSet &patterns)
Patterns to convert from one named op to another.
void populateMoveInitOperandsToInputPattern(RewritePatternSet &patterns)
A pattern that converts init operands to input operands.
FailureOr< GenericOp > generalizeNamedOp(RewriterBase &rewriter, LinalgOp namedOp)
Create a GenericOp from the given named operation namedOp and replace namedOp.
LogicalResult rewriteAsPaddedOp(RewriterBase &rewriter, LinalgOp opToPad, const LinalgPaddingOptions &options, LinalgOp &paddedOp, SmallVector< Value > &replacements, SmallVector< tensor::PadOp > &padOps)
Pad the iterator dimensions paddingDimensions of all opToPad operands to a static bounding box.
Definition: Padding.cpp:151
void populateSplitReductionPattern(RewritePatternSet &patterns, const ControlSplitReductionFn &controlSplitReductionFn, bool useAlloc=false)
Patterns to apply splitReduction below.
void populateFuseTensorPadWithProducerLinalgOpPatterns(RewritePatternSet &patterns)
Pattern to fuse a tensor.pad operation with the producer of its source, if the producer is a linalg o...
FailureOr< std::pair< Operation *, Operation * > > rewriteInIm2Col(RewriterBase &rewriter, linalg::Conv2DNhwcHwcfOp convOp)
Convert linalg.conv_2d_nhwc_hwcf into linalg.generic (for img2col packing) and linalg....
bool areDimSequencesPreserved(ArrayRef< AffineMap > maps, ArrayRef< ReassociationIndices > dimSequences)
Return true if all sequences of dimensions specified in dimSequences are contiguous in all the ranges...
FailureOr< ForallTilingResult > tileToForallOpUsingTileSizes(RewriterBase &builder, TilingInterface op, ArrayRef< OpFoldResult > tileSizes, std::optional< ArrayAttr > mapping)
Same as tileToForallOp, but calculate the number of threads required using the given tileSizes.
Definition: Tiling.cpp:433
FailureOr< LowerUnPackOpResult > lowerUnPack(RewriterBase &rewriter, tensor::UnPackOp unPackOp)
Rewrite pack as empty + transpose + reshape + extract_slice.
Definition: Transforms.cpp:355
LogicalResult vectorizeOpPrecondition(Operation *op, ArrayRef< int64_t > inputVectorSizes={}, ArrayRef< bool > inputScalableVecDims={}, bool vectorizeNDExtract=false)
Return success if the operation can be vectorized.
void populateBubbleUpExtractSliceOpPatterns(RewritePatternSet &patterns)
Patterns that are used to bubble up extract slice op above linalg op.
void transformIndexOps(RewriterBase &b, LinalgOp op, SmallVectorImpl< Value > &ivs, const LoopIndexToRangeIndexMap &loopIndexToRangeIndex)
All indices returned by IndexOp should be invariant with respect to tiling.
Definition: Tiling.cpp:78
std::function< std::optional< Value >(OpBuilder &b, memref::SubViewOp subView, ArrayRef< Value > boundingSubViewSize, DataLayout &layout)> AllocBufferCallbackFn
Callback function type used to perform the allocation for the promoted subView.
Definition: Transforms.h:340
FailureOr< SmallVector< Value > > collapseOpIterationDims(LinalgType op, ArrayRef< ReassociationIndices > foldedIterationDims, RewriterBase &rewriter)
Collapses dimensions of linalg.generic/linalg.copy operation.
void populateConvertConv2DToImg2ColPatterns(RewritePatternSet &patterns)
Populates patterns to transform linalg.conv_2d_xxx operations into linalg.generic (for img2col packin...
DenseMap< int, int > LoopIndexToRangeIndexMap
Creates a number of ranges equal to the number of non-zero in tileSizes.
Definition: Transforms.h:785
std::optional< Value > allocateWorkgroupMemory(OpBuilder &builder, memref::SubViewOp subview, ArrayRef< Value > sizeBounds, DataLayout &)
Allocate the subview in the GPU workgroup memory.
Definition: Promotion.cpp:472
Value bufferizeToAllocation(RewriterBase &rewriter, const BufferizeToAllocationOptions &options, tensor::PadOp padOp, Attribute memorySpace={}, Operation *insertionPoint=nullptr)
Materialize a buffer allocation for the given tensor.pad op and lower the op to linalg....
std::function< bool(OpOperand *fusedOperand)> ControlFusionFn
Function type which is used to control when to stop fusion.
Definition: Transforms.h:1523
bool isDimSequencePreserved(AffineMap map, ReassociationIndicesRef dimSequence)
Return true if a given sequence of dimensions are contiguous in the range of the specified indexing m...
std::function< bool(Operation *op)> ControlPropagationFn
Function type which is used to control propagation of tensor.pack/unpack ops.
Definition: Transforms.h:1535
FailureOr< LinalgOp > specializeGenericOp(RewriterBase &rewriter, GenericOp genericOp)
Create a namedOp from the given GenericOp and replace the GenericOp.
Definition: Specialize.cpp:24
void populateFoldReshapeOpsByCollapsingPatterns(RewritePatternSet &patterns, const ControlFusionFn &controlFoldingReshapes)
Patterns to fold an expanding tensor.expand_shape operation with its producer generic operation by co...
LinalgTilingLoopType
The type of loops to be generated during tiling.
Definition: Utils.h:103
std::function< LogicalResult(OpBuilder &b, Value buffer)> DeallocBufferCallbackFn
Callback function type used to deallocate the buffers used to hold the promoted subview.
Definition: Transforms.h:345
void populateDataLayoutPropagationPatterns(RewritePatternSet &patterns, const ControlPropagationFn &controlPackUnPackPropagation)
Patterns to bubble up or down data layout ops across other operations.
void populatePadOpVectorizationPatterns(RewritePatternSet &patterns, PatternBenefit baseBenefit=1)
Populates patterns with patterns that vectorize tensor.pad.
void populateLinalgTilingCanonicalizationPatterns(RewritePatternSet &patterns)
Definition: Tiling.cpp:877
LogicalResult deallocateGPUPrivateMemory(OpBuilder &, Value)
In case of GPU private memory there is no need to deallocate since the memory is freed when going out...
Definition: Promotion.cpp:513
void populateSparseTensorRewriting(RewritePatternSet &patterns)
Populate patterns that are only useful in the context of sparse tensors.
FailureOr< ElementwiseOpFusionResult > fuseElementwiseOps(RewriterBase &rewriter, OpOperand *fusedOperand)
FailureOr< PromotionInfo > promoteSubviewAsNewBuffer(OpBuilder &b, Location loc, memref::SubViewOp subView, const AllocBufferCallbackFn &allocationFn, DataLayout &layout)
Definition: Promotion.cpp:238
std::optional< Value > allocateGPUPrivateMemory(OpBuilder &builder, memref::SubViewOp subview, ArrayRef< Value > sizeBounds, DataLayout &)
Allocate the subview in the GPU private memory.
Definition: Promotion.cpp:497
FailureOr< Operation * > rewriteInDestinationPassingStyle(RewriterBase &rewriter, tensor::FromElementsOp fromElementsOp)
Rewrite tensor.from_elements to linalg.generic.
void peelLoops(RewriterBase &rewriter, ArrayRef< scf::ForOp > loops)
Peel 'loops' and applies affine_min/max bounds simplification on the fly where relevant.
Definition: Transforms.cpp:75
FailureOr< ForallTilingResult > tileToForallOp(RewriterBase &builder, TilingInterface op, ArrayRef< OpFoldResult > numThreads, std::optional< ArrayAttr > mapping)
Definition: Tiling.cpp:424
void populateConvertToDestinationStylePatterns(RewritePatternSet &patterns)
Populate patterns that convert non-destination-style ops to destination style ops.
FailureOr< Operation * > transposeConv2D(RewriterBase &rewriter, linalg::Conv2DNhwcFhwcOp op)
Convert linalg.conv_2d_nhwc_fhwc(_q) to linalg.conv_2d_nhwc_hwcf(_q) by materializing transpose.
void populateFoldUnitExtentDimsPatterns(RewritePatternSet &patterns, ControlDropUnitDims &options)
Patterns to fold unit-extent dimensions in operands/results of linalg ops on tensors via reassociativ...
LogicalResult copyToWorkgroupMemory(OpBuilder &b, Value src, Value dst)
Create Memref copy operations and add gpu barrier guards before and after the copy operation to ensur...
Definition: Promotion.cpp:488
std::function< SmallVector< Value, 4 >(OpBuilder &, Operation *)> TileSizeComputationFunction
Definition: Transforms.h:187
std::function< LogicalResult(RewriterBase &, tensor::PadOp, Value)> OptimizeCopyFn
Definition: Transforms.h:1330
FailureOr< Value > hoistPaddingOnTensors(RewriterBase &rewriter, tensor::PadOp opToHoist, int64_t numLoops, ArrayRef< int64_t > transposeVector, tensor::PadOp &hoistedOp, SmallVectorImpl< GenericOp > &transposeOps)
Mechanically hoist padding operations on tensors by numLoops into a new, generally larger tensor.
void populateElementwiseToLinalgConversionPatterns(RewritePatternSet &patterns)
Populate patterns that convert ElementwiseMappable ops to linalg parallel loops.
LogicalResult linalgOpAnchoredEmptyTensorEliminationStep(RewriterBase &rewriter, Operation *op, bufferization::OneShotAnalysisState &state)
Try to eliminate tensor::EmptyOps inside op that are anchored on a LinalgOp.
FailureOr< LinalgLoops > linalgOpToLoops(RewriterBase &rewriter, LinalgOp linalgOp)
Emit a loop nest of scf.for with the proper body for linalgOp.
Definition: Loops.cpp:369
LogicalResult vectorize(RewriterBase &rewriter, Operation *op, ArrayRef< int64_t > inputVectorSizes={}, ArrayRef< bool > inputScalableVecDims={}, bool vectorizeNDExtract=false, bool flatten1DDepthwiseConv=false)
Emit a suitable vector form for an operation.
std::tuple< SmallVector< Range, 4 >, LoopIndexToRangeIndexMap > makeTiledLoopRanges(RewriterBase &b, Location loc, AffineMap map, ArrayRef< OpFoldResult > allShapeSizes, ArrayRef< OpFoldResult > allTileSizes)
Definition: Tiling.cpp:49
LogicalResult promoteSubviewsPrecondition(Operation *op, LinalgPromotionOptions options)
Promote memref.subviews feeding linalg-on-buffers operations.
Definition: Promotion.cpp:401
LogicalResult copyToGPUPrivateMemory(OpBuilder &b, Value src, Value dst)
Normal copy to between src and dst.
Definition: Promotion.cpp:505
void populateDecomposeConvolutionPatterns(RewritePatternSet &patterns, PatternBenefit benefit=1)
Linalg decompose convolutions patterns.
void populateConvolutionVectorizationPatterns(RewritePatternSet &patterns, PatternBenefit benefit=1)
Populate patterns for vectorizing low-D convolution ops.
LogicalResult vectorizeCopy(RewriterBase &builder, memref::CopyOp copyOp)
Emit a suitable vector form for a Copy op with fully static shape.
FailureOr< GenericOp > interchangeGenericOp(RewriterBase &rewriter, GenericOp genericOp, ArrayRef< unsigned > interchangeVector)
Interchange the iterator_types and iterator_maps dimensions and adapts the index accesses of op.
Definition: Interchange.cpp:50
void populateCollapseDimensions(RewritePatternSet &patterns, const GetCollapsableDimensionsFn &controlCollapseDimensions)
Pattern to collapse dimensions in a linalg.generic op.
bool areElementwiseOpsFusable(OpOperand *fusedOperand)
Return true if two linalg.generic operations with producer/consumer relationship through fusedOperand...
FailureOr< StaticMultiSizeSpecification > computeStaticMultiTileSizes(LinalgOp op, unsigned dimension, int64_t targetSize, int64_t divisor)
Definition: Tiling.cpp:111
FailureOr< LinalgLoops > linalgOpToAffineLoops(RewriterBase &rewriter, LinalgOp linalgOp)
Emit a loop nest of affine.for with the proper body for linalgOp.
Definition: Loops.cpp:364
void populateEraseUnusedOperandsAndResultsPatterns(RewritePatternSet &patterns)
Pattern to remove dead operands and results of linalg.generic operations.
std::function< LogicalResult(OpBuilder &b, Value src, Value dst)> CopyCallbackFn
Callback function type used to insert copy from original subview to subview of the promoted region fo...
Definition: Transforms.h:352
FailureOr< SplitReductionResult > splitReduction(RewriterBase &b, LinalgOp op, const ControlSplitReductionFn &controlSplitReductionFn, bool useAlloc=false)
FailureOr< LinalgOp > padAndHoistLinalgOp(RewriterBase &rewriter, LinalgOp linalgOp, const LinalgPaddingOptions &options)
Apply padding and hoisting to linalgOp according to the configuration specified in options.
Definition: Padding.cpp:263
void populateDecomposeLinalgOpsPattern(RewritePatternSet &patterns, bool removeDeadArgsAndResults=true)
Populate patterns for splitting a LinalgOp with multiple statements within its payload into multiple ...
FailureOr< ForallReductionTilingResult > tileReductionUsingForall(RewriterBase &b, PartialReductionOpInterface op, ArrayRef< OpFoldResult > numThreads, ArrayRef< OpFoldResult > tileSizes={}, std::optional< ArrayAttr > mapping=std::nullopt)
Method to tile a reduction to parallel iterations computing partial reductions.
Definition: Tiling.cpp:613
FailureOr< PackResult > packMatmulGreedily(RewriterBase &rewriter, LinalgOp linalgOp, ArrayRef< OpFoldResult > mnkPackedSizes, ArrayRef< int64_t > mnkPaddedSizesNextMultipleOf, ArrayRef< int64_t > mnkOrder)
Pack a LinalgOp by greedily inferring matmul dimensions (m, n, k) where m and n are proper parallel d...
Definition: Transforms.cpp:777
LogicalResult dropUnitDims(RewriterBase &rewriter, GenericOp genericOp, const ControlDropUnitDims &options)
FailureOr< PackResult > pack(RewriterBase &rewriter, linalg::LinalgOp linalgOp, ArrayRef< OpFoldResult > packedSizes)
Implement packing of a single LinalgOp by packedSizes.
Definition: Transforms.cpp:488
void populateEraseUnnecessaryInputsPatterns(RewritePatternSet &patterns)
Patterns to promote inputs to outputs and remove unused inputs of linalg.generic ops.
FailureOr< TiledLinalgOp > tileLinalgOp(RewriterBase &b, LinalgOp op, const LinalgTilingOptions &options)
Definition: Tiling.cpp:837
std::function< SmallVector< ReassociationIndices >(linalg::LinalgOp)> GetCollapsableDimensionsFn
Function type to control generic op dimension collapsing.
Definition: Transforms.h:1554
FailureOr< LowerPackResult > lowerPack(RewriterBase &rewriter, tensor::PackOp packOp)
Rewrite pack as pad + reshape + transpose.
Definition: Transforms.cpp:219
void populateFoldReshapeOpsByExpansionPatterns(RewritePatternSet &patterns, const ControlFusionFn &controlFoldingReshapes)
Patterns to fold an expanding (collapsing) tensor_reshape operation with its producer (consumer) gene...
void populateSwapExtractSliceWithFillPatterns(RewritePatternSet &patterns)
Adds patterns that waps tensor.extract_slice(linalg.fill(cst, init)) into linalg.fill(cst,...
void populateInlineConstantOperandsPatterns(RewritePatternSet &patterns)
Patterns that are used to inline constant operands into linalg generic ops.
FailureOr< LinalgOp > promoteSubViews(OpBuilder &b, LinalgOp op, const LinalgPromotionOptions &options)
Promote the subViews into a new buffer allocated at the insertion point b.
Definition: Promotion.cpp:423
void populateConstantFoldLinalgOperations(RewritePatternSet &patterns, const ControlFusionFn &controlFn)
Patterns to constant fold Linalg operations.
std::function< SplitReductionOptions(LinalgOp op)> ControlSplitReductionFn
Function signature to control reduction splitting.
Definition: Transforms.h:443
LogicalResult deallocateWorkgroupMemory(OpBuilder &, Value)
In case of GPU group memory there is no need to deallocate.
Definition: Promotion.cpp:481
void populateLinalgNamedOpsGeneralizationPatterns(RewritePatternSet &patterns)
Linalg generalization patterns.
std::optional< vector::CombiningKind > getCombinerOpKind(Operation *combinerOp)
Return vector::CombiningKind for the given op.
SmallVector< Value > peelLoop(RewriterBase &rewriter, Operation *op)
Try to peel and canonicalize loop op and return the new result.
Definition: Transforms.cpp:59
RewritePatternSet getLinalgTilingCanonicalizationPatterns(MLIRContext *ctx)
Canonicalization patterns relevant to apply after tiling patterns.
Definition: Tiling.cpp:871
FailureOr< PackTransposeResult > packTranspose(RewriterBase &rewriter, tensor::PackOp packOp, linalg::LinalgOp linalgOp, tensor::UnPackOp maybeUnPackOp, ArrayRef< int64_t > outerPerm, ArrayRef< int64_t > innerPerm)
Transpose a single PackOp -> LinalgOp -> UnPackOp chain and return the transposed PackOp -> LinalgOp ...
Definition: Transforms.cpp:686
std::pair< TilingInterface, TilingInterface > splitOp(RewriterBase &rewriter, TilingInterface op, unsigned dimension, OpFoldResult splitPoint)
Split the given op into two parts along the given iteration space dimension at the specified splitPoi...
Definition: Split.cpp:67
void populateElementwiseOpsFusionPatterns(RewritePatternSet &patterns, const ControlFusionFn &controlElementwiseOpFusion)
Patterns for fusing linalg operation on tensors.
FailureOr< SplitReductionResult > splitReductionByScaling(RewriterBase &b, LinalgOp op, const ControlSplitReductionFn &controlSplitReductionFn, bool useAlloc=false)
Scaling-based implementation of the split reduction transformation.
FailureOr< MultiSizeSpecification > computeMultiTileSizes(OpBuilder &builder, LinalgOp op, unsigned dimension, OpFoldResult targetSize, OpFoldResult divisor, bool emitAssertions=true)
Emits the IR computing the multi-sized tiling specification with two tile sizes not exceeding targetS...
Definition: Tiling.cpp:137
FailureOr< LinalgLoops > linalgOpToParallelLoops(RewriterBase &rewriter, LinalgOp linalgOp)
Emit a loop nest of scf.parallel with the proper body for linalgOp.
Definition: Loops.cpp:376
Include the generated interface declarations.
This class represents an efficient way to signal success or failure.
Definition: LogicalResult.h:26
OpInterfaceRewritePattern is a wrapper around RewritePattern that allows for matching and rewriting a...
Definition: PatternMatch.h:372
OpRewritePattern is a wrapper around RewritePattern that allows for matching and rewriting against an...
Definition: PatternMatch.h:357
bool bufferizeDestinationOnly
If set to "true", only the destination tensor operands are bufferized to a new allocation (and wrappe...
Definition: Transforms.h:65
bool emitDealloc
If set to "true", a memref.dealloc operation will be emitted for each allocated buffer.
Definition: Transforms.h:71
Transformation to drop unit-extent dimensions from linalg.generic operations.
Definition: Transforms.h:473
RankReductionStrategy rankReductionStrategy
Definition: Transforms.h:476
std::function< SmallVector< unsigned >(Operation *)> ControlFnTy
Definition: Transforms.h:479
Vectorization pattern for memref::CopyOp.
Definition: Transforms.h:1322
LogicalResult matchAndRewrite(memref::CopyOp copyOp, PatternRewriter &rewriter) const override
Definition: Transforms.cpp:928
LogicalResult matchAndRewrite(Conv2DOp convOp, PatternRewriter &rewriter) const override
Definition: Transforms.h:1290
FailureOr< Conv1DOp > returningMatchAndRewrite(Conv2DOp convOp, PatternRewriter &rewriter) const
DownscaleConv2DOp(MLIRContext *context, PatternBenefit benefit=1)
Definition: Transforms.h:1284
Rewrites 2-D depthwise convolution ops with size-1 (w, kw) or (h, kh) dimensions into 1-D depthwise c...
Definition: Transforms.h:1268
FailureOr< DepthwiseConv1DNwcWcOp > returningMatchAndRewrite(DepthwiseConv2DNhwcHwcOp convOp, PatternRewriter &rewriter) const
LogicalResult matchAndRewrite(DepthwiseConv2DNhwcHwcOp convOp, PatternRewriter &rewriter) const override
Definition: Transforms.h:1277
DownscaleDepthwiseConv2DNhwcHwcOp(MLIRContext *context, PatternBenefit benefit=1)
Definition: Transforms.h:1269
Rewrites 2-D convolution ops with size-1 window dimensions into 1-D convolution ops.
Definition: Transforms.h:1248
LogicalResult matchAndRewrite(Conv2DOp convOp, PatternRewriter &rewriter) const override
Definition: Transforms.h:1254
FailureOr< Conv1DOp > returningMatchAndRewrite(Conv2DOp convOp, PatternRewriter &rewriter) const
Fuse two linalg.generic operations that have a producer-consumer relationship captured through fusedO...
Definition: Transforms.h:493
llvm::DenseMap< Value, Value > replacements
Definition: Transforms.h:495
static llvm::SmallDenseSet< int > getPreservedProducerResults(GenericOp producer, GenericOp consumer)
Returns a set of indices of the producer's results which would be preserved after the fusion.
Rewrite extract_slice(tensor.pad(x)) into tensor.pad(extract_slice(x)).
Definition: Transforms.h:1432
std::function< std::optional< bool >(tensor::ExtractSliceOp)> ControlFn
A function to control pattern application and rewrite logic.
Definition: Transforms.h:1442
LogicalResult matchAndRewrite(tensor::ExtractSliceOp sliceOp, PatternRewriter &rewriter) const override
ExtractSliceOfPadTensorSwapPattern(MLIRContext *context, ControlFn controlFn=nullptr, PatternBenefit benefit=1)
Definition: Transforms.h:1444
Transformation information returned after reduction tiling.
Definition: Transforms.h:869
Operation * parallelTiledOp
The partial reduction tiled op generated.
Definition: Transforms.h:871
Operation * initialOp
The op initializing the tensor used for partial reductions.
Definition: Transforms.h:875
scf::ForallOp loops
The scf.forall operation that iterate over the tiles.
Definition: Transforms.h:877
Operation * mergeOp
The final reduction operation merging all the partial reductions.
Definition: Transforms.h:873
Rewrite a TilingInterface op to a tiled scf.forall, applying tiling by numThreads.
Definition: Transforms.h:852
Rewrites a tensor::PackOp into a sequence of tensor.pad + linalg.transpose + tensor....
Definition: Transforms.h:1355
LogicalResult matchAndRewrite(tensor::PackOp packOp, PatternRewriter &rewriter) const override
Rewrites a tensor::UnPackOp into a sequence of rank-reduced extract_slice op.
Definition: Transforms.h:1365
LogicalResult matchAndRewrite(tensor::UnPackOp unpackOp, PatternRewriter &rewriter) const override
Rewrite a tensor::PadOp into a sequence of EmptyOp, FillOp and InsertSliceOp.
Definition: Transforms.h:1335
LogicalResult matchAndRewrite(tensor::PadOp padOp, PatternRewriter &rewriter) const override
Definition: Transforms.cpp:952
Value createFillOrGenerateOp(RewriterBase &rewriter, tensor::PadOp padOp, Value dest, const SmallVector< Value > &dynSizes) const
Filling dest using FillOp constant padding value if possible.
Definition: Transforms.cpp:935
GeneralizePadOpPattern(MLIRContext *context, OptimizeCopyFn optimizeCopyFn=nullptr, PatternBenefit benefit=1)
Definition: Transforms.h:1336
Match and rewrite for the pattern:
Definition: Transforms.h:1395
LogicalResult matchAndRewrite(vector::TransferReadOp xferOp, PatternRewriter &rewriter) const override
TODO: use interfaces, side-effects and aliasing analysis as appropriate, when available.
Match and rewrite for the pattern:
Definition: Transforms.h:1423
LogicalResult matchAndRewrite(vector::TransferWriteOp xferOp, PatternRewriter &rewriter) const override
TODO: use interfaces, side-effects and aliasing analysis as appropriate, when available.
Linalg generalization pattern.
Definition: Transforms.h:1305
LogicalResult matchAndRewrite(LinalgOp op, PatternRewriter &rewriter) const override
Definition: Transforms.h:1315
FailureOr< GenericOp > returningMatchAndRewrite(LinalgOp op, PatternRewriter &rewriter) const
matchAndRewrite implementation that returns the significant transformed pieces of IR.
Definition: Transforms.h:1311
Options that allow distribution of loops generated in Linalg transforms to processors while generatin...
Definition: Utils.h:305
SmallVector< Attribute > paddingValues
A padding value for every operand.
Definition: Transforms.h:281
LinalgPaddingOptions & setPadToMultipleOf(ArrayRef< int64_t > m)
Definition: Transforms.h:294
SmallVector< bool > packPaddings
A flag for every operand to mark the PadOp as nofold which enables packing for statically shaped oper...
Definition: Transforms.h:300
std::optional< SmallVector< int64_t > > padToMultipleOf
A list of multiples to which each padding dimension should be padded to.
Definition: Transforms.h:293
LinalgPaddingOptions & setPaddingDimensions(ArrayRef< int64_t > pd)
Definition: Transforms.h:288
LinalgPaddingOptions & setTransposePaddings(ArrayRef< SmallVector< int64_t >> tp)
Definition: Transforms.h:315
SmallVector< SmallVector< int64_t > > transposePaddings
A permutation vector for every operand used to transpose the packed PadOp results.
Definition: Transforms.h:313
LinalgPaddingOptions & setPaddingValues(ArrayRef< Attribute > pv)
Definition: Transforms.h:282
LinalgPaddingOptions & setPackPaddings(ArrayRef< bool > pp)
Definition: Transforms.h:301
LinalgPaddingOptions & setCopyBackOp(CopyBackOp op)
Definition: Transforms.h:327
LinalgPaddingOptions & setHoistPaddings(ArrayRef< int64_t > hp)
Definition: Transforms.h:307
SmallVector< int64_t > hoistPaddings
A number of loops to hoist the PadOp out for every operand.
Definition: Transforms.h:306
SmallVector< int64_t > paddingDimensions
A list of iterator dimensions to pad.
Definition: Transforms.h:287
CopyBackOp copyBackOp
The op to be used for copying the padded result to the original destination tensor.
Definition: Transforms.h:326
std::optional< unsigned > alignment
Alignment of promoted buffer. If std::nullopt do not specify alignment.
Definition: Transforms.h:385
LinalgPromotionOptions & setUseFullTileBuffersByDefault(bool use)
Definition: Transforms.h:380
bool useAlloca
Use alloca with the default allocation scheme.
Definition: Transforms.h:398
LinalgPromotionOptions & setAlignment(unsigned align)
Definition: Transforms.h:386
std::optional< Attribute > memorySpace
Memory space of promoted buffer.
Definition: Transforms.h:392
std::optional< CopyCallbackFn > copyOutFn
Definition: Transforms.h:418
std::optional< CopyCallbackFn > copyInFn
Callback function to do the copy of data to and from the promoted subview.
Definition: Transforms.h:417
LinalgPromotionOptions & setUseAlloca(bool use)
Definition: Transforms.h:399
std::optional< DenseSet< unsigned > > operandsToPromote
Indices of subViews to promote.
Definition: Transforms.h:357
LinalgPromotionOptions & setCopyInOutFns(CopyCallbackFn const &copyIn, CopyCallbackFn const &copyOut)
Definition: Transforms.h:419
LinalgPromotionOptions & setUseFullTileBuffers(ArrayRef< bool > useFullTiles)
Definition: Transforms.h:369
std::optional< AllocBufferCallbackFn > allocationFn
Callback function to do the allocation of the promoted buffer.
Definition: Transforms.h:406
bool useFullTileBuffersDefault
If true all operands unspecified by useFullTileBuffers will use the full view, otherwise the partial ...
Definition: Transforms.h:379
std::optional< DeallocBufferCallbackFn > deallocationFn
Definition: Transforms.h:407
LinalgPromotionOptions & setMemorySpace(Attribute memorySpc)
Definition: Transforms.h:393
LinalgPromotionOptions & setAllocationDeallocationFns(AllocBufferCallbackFn const &allocFn, DeallocBufferCallbackFn const &deallocFn)
Definition: Transforms.h:409
std::optional< llvm::SmallBitVector > useFullTileBuffers
If ith element of useFullTiles is true the full view should be used for the promoted buffer of the it...
Definition: Transforms.h:368
LinalgPromotionOptions & setOperandsToPromote(ArrayRef< int64_t > operands)
Definition: Transforms.h:358
std::optional< LinalgLoopDistributionOptions > tileDistribution
When specified, specifies distribution of generated tile loops to processors.
Definition: Transforms.h:271
LinalgTilingAndFusionOptions & setTileSizes(ArrayRef< int64_t > ts)
Definition: Transforms.h:263
SmallVector< int64_t > tileInterchange
Tile interchange used to permute the tile loops.
Definition: Transforms.h:268
LinalgTilingAndFusionOptions & setDistributionOptions(LinalgLoopDistributionOptions distributionOptions)
Definition: Transforms.h:273
SmallVector< int64_t > tileSizes
Tile sizes used to tile the root operation.
Definition: Transforms.h:262
LinalgTilingOptions & setLoopType(LinalgTilingLoopType lt)
Definition: Transforms.h:227
LinalgTilingOptions & setDistributionTypes(ArrayRef< StringRef > types)
Definition: Transforms.h:245
LinalgTilingOptions & setInterchange(ArrayRef< unsigned > interchange)
Definition: Transforms.h:219
LinalgTilingLoopType loopType
The type of tile loops to generate.
Definition: Transforms.h:225
LinalgTilingOptions & setTileSizeComputationFunction(TileSizeComputationFunction fun)
Definition: Transforms.h:196
LinalgTilingOptions & setTileSizes(const SmallVector< Value, 4 > &ts)
Set the tileSizeComputationFunction to return the values ts.
Definition: Transforms.h:203
LinalgTilingOptions & setPeeledLoops(ArrayRef< int64_t > loops)
Definition: Transforms.h:253
SmallVector< int64_t > peeledLoops
Peel the specified loops.
Definition: Transforms.h:251
LinalgTilingOptions & setDistributionOptions(LinalgLoopDistributionOptions distributionOptions)
Definition: Transforms.h:237
SmallVector< unsigned, 4 > interchangeVector
The interchange vector to reorder the tiled loops.
Definition: Transforms.h:217
TileSizeComputationFunction tileSizeComputationFunction
Computation function that returns the tile sizes for each operation.
Definition: Transforms.h:193
LinalgTilingOptions & scalarizeDynamicDims()
Tile all dynamic dimensions by 1.
std::optional< LinalgLoopDistributionOptions > distribution
When specified, specifies distribution of generated tile loops to processors.
Definition: Transforms.h:234
SmallVector< StringRef, 2 > distributionTypes
Specification markers of how to distribute the linalg.tiled_loop.
Definition: Transforms.h:243
linalg::TransposeOp transposeOp
Definition: Transforms.h:1093
tensor::ExpandShapeOp expandShapeOp
Definition: Transforms.h:1092
tensor::ExtractSliceOp extractSliceOp
Definition: Transforms.h:1104
linalg::TransposeOp transposeOp
Definition: Transforms.h:1102
tensor::CollapseShapeOp collapseShapeOp
Definition: Transforms.h:1103
A description of a multi-size tiling comprising tile sizes and numbers of tiles, expressed as Values ...
Definition: Transforms.h:805
Struct to hold the result of a pack call.
Definition: Transforms.h:1112
linalg::LinalgOp packedLinalgOp
Definition: Transforms.h:1114
SmallVector< tensor::PackOp > packOps
Definition: Transforms.h:1113
SmallVector< tensor::UnPackOp > unPackOps
Definition: Transforms.h:1115
Struct to hold the result of a packTranspose call.
Definition: Transforms.h:1124
linalg::LinalgOp transposedLinalgOp
Definition: Transforms.h:1126
tensor::UnPackOp transposedUnPackOp
Definition: Transforms.h:1127
Create a new buffer using the allocationFn provided.
Definition: Transforms.h:702
Split Reduction options.
Definition: Transforms.h:428
Apply transformation to split the single linalg op reduction into a parallel and reduction dimension.
Definition: Transforms.h:1006
Perform standalone tiling of a single LinalgOp by tileSizes.
Definition: Transforms.h:663
SmallVector< Operation *, 8 > loops
Definition: Transforms.h:665
SmallVector< Value, 4 > tensorResults
Definition: Transforms.h:666
T lowTripCount
Number of tiles associated with each size.
Definition: Transforms.h:797
Helper struct to hold the results of building a packing loop nest.
Definition: Transforms.h:533
SmallVector< OpFoldResult > strides
Definition: Transforms.h:534
SmallVector< Value > leadingPackedTensorIndexings
Definition: Transforms.h:535
SmallVector< Value > clonedLoopIvs
Definition: Transforms.h:535
SmallVector< OpFoldResult > sizes
Definition: Transforms.h:534
SmallVector< OpFoldResult > offsets
Definition: Transforms.h:534