MLIR  15.0.0git
Transforms.h
Go to the documentation of this file.
1 //===- Transforms.h - Linalg transformations as patterns --------*- C++ -*-===//
2 //
3 // Part of the LLVM Project, under the Apache License v2.0 with LLVM Exceptions.
4 // See https://llvm.org/LICENSE.txt for license information.
5 // SPDX-License-Identifier: Apache-2.0 WITH LLVM-exception
6 //
7 //===----------------------------------------------------------------------===//
8 
9 #ifndef MLIR_DIALECT_LINALG_TRANSFORMS_TRANSFORMS_H
10 #define MLIR_DIALECT_LINALG_TRANSFORMS_TRANSFORMS_H
11 
12 #include <utility>
13 
22 #include "mlir/IR/PatternMatch.h"
24 #include "llvm/ADT/SmallBitVector.h"
25 #include "llvm/ADT/SmallSet.h"
26 
27 namespace mlir {
28 namespace bufferization {
29 class BufferizeTypeConverter;
30 } // namespace bufferization
31 
32 class FrozenRewritePatternSet;
33 
34 namespace linalg {
35 
36 struct LinalgElementwiseFusionOptions;
37 struct LinalgFusionOptions;
38 struct LinalgTilingOptions;
39 
40 //===----------------------------------------------------------------------===//
41 // Transformations exposed as function calls.
42 //===----------------------------------------------------------------------===//
44 
47 
48 /// Populate patterns for vectorizing low-D convolution ops. This is a step in
49 /// progressive lowering for convolution ops, it assume high-D convolution ops
50 /// were decomposed previously.
52  PatternBenefit benefit = 1);
53 
54 /// Populate patterns that convert `ElementwiseMappable` ops to linalg
55 /// parallel loops.
57 
58 /// Populate patterns that are only useful in the context of sparse tensors.
60 
61 /// Function type which is used to control when to stop fusion. It is expected
62 /// that OpOperand is not modified in the callback. The OpOperand is not marked
63 /// as const to allow callers to use non-const methods.
64 using ControlFusionFn =
65  std::function<bool(const OpResult &producer, OpOperand &consumer)>;
66 
67 /// Patterns for fusing linalg operation on tensors.
68 
69 /// Pattern to fuse `linalg.generic` -> `linalg.generic` operations
70 /// when both operations are fusable elementwise operations.
72  RewritePatternSet &patterns,
73  const ControlFusionFn &controlElementwiseOpFusion);
74 
75 /// Patterns to fold an expanding (collapsing) tensor_reshape operation with its
76 /// producer (consumer) generic operation by expanding the dimensionality of the
77 /// loop in the generic op.
79  RewritePatternSet &patterns, const ControlFusionFn &controlFoldingReshapes);
80 
81 /// Patterns to fold an expanding tensor.expand_shape operation with its
82 /// producer generic operation by collapsing the dimensions of the generic op.
84  RewritePatternSet &patterns, const ControlFusionFn &controlFoldingReshapes);
85 
86 /// Patterns to constant fold Linalg operations.
88  const ControlFusionFn &controlFn);
89 
90 /// Pattern to fuse a `tensor.pad` operation with the producer of its source,
91 /// if the producer is a `linalg` operation with all parallel iterator types.
93  RewritePatternSet &patterns);
94 
95 /// Patterns to convert from one named op to another. These can be seen as
96 /// canonicalizations of named ops into another named op.
98 
99 /// Patterns to fold unit-extent dimensions in operands/results of linalg ops on
100 /// tensors.
102 
103 /// Patterns that are used to inline constant operands into linalg generic ops.
105 
106 /// Patterns that are used to bubble up extract slice op above linalg op.
108 
109 /// Perform standalone tiling of a single LinalgOp by `tileSizes`.
110 /// and permute the loop nest according to `interchangeVector`
111 /// The permutation is expressed as a list of integers that specify
112 /// the new ordering of the loop nest. The length of `interchangeVector`
113 /// must be equal to the length of `tileSizes`.
114 /// An empty vector is interpreted as the identity permutation and the
115 /// transformation returns early.
116 ///
117 /// Return a struct containing the tiled loops in the specified order
118 /// and the cloned op if successful, llvm::None otherwise.
119 ///
120 /// E.g. the permutation `(i,j,k) -> (j,k,i)` is expressed by
121 /// `interchangeVector = [1,2,0]`. All values in `interchangeVector` must be
122 /// integers, in the range 0..`tileSizes.size()` without duplications
123 /// (i.e. `[1,1,2]` is an invalid permutation).
125  LinalgOp op;
128 };
131 
132 /// Peel the loops of a TiledLinalgOp.
133 void peelTiledLinalgOp(RewriterBase &rewriter, TiledLinalgOp &res,
134  ArrayRef<int64_t> peeledLoops,
135  LinalgTilingLoopType loopType);
136 
137 /// Fuse a sequence of linalg operations (`ops`) using tile-and-fuse. This
138 /// proceeds as follows:
139 /// - Find outer parallel loops in these ops that can be fused.
140 /// - Tile fusable outer parallel loops of the last operation in the sequence.
141 /// - Fuse the remaining operations with the tiled operation
142 ///
143 /// For example, consider the sequence of matmul below
144 ///
145 /// linalg.matmul ins(%arg0, %arg1 : memref<256x32xf32>, memref<32x32xf32>)
146 /// outs(%arg2 : memref<256x32xf32>)
147 /// linalg.matmul ins(%arg2, %arg3 : memref<256x32xf32>, memref<32x32xf32>)
148 /// outs(%arg4 : memref<256x32xf32>)
149 ///
150 /// It is legal to fuse the RAW dependence (through %arg2) by only fusing the
151 /// matmuls row-wise. For example, the fused computation for the above is shown
152 /// below. The outer `scf.parallel` loop is the "fused" loop obtained by tiling
153 /// along the rows of the matrix. The entire rows of the first matmul operation
154 /// need to be computed before they can be used for the second matmul. The
155 /// second matmul is further tiled (similar to normal tiling).
156 ///
157 /// #map0 = affine_map<(d0, d1)[s0] -> (d0 * 32 + s0 + d1)>
158 /// #map1 = affine_map<(d0, d1) -> (d0 * 32 + d1)>
159 /// scf.parallel (%arg5) = (%c0) to (%c256) step (%c16) {
160 /// %0 = subview %arg2[%arg5, 0] [16, 32] [1, 1]
161 /// : memref<256x32xf32> to memref<16x32xf32, #map0>
162 /// %1 = subview %arg4[%arg5, 0] [16, 32] [1, 1]
163 /// : memref<256x32xf32> to memref<16x32xf32, #map0>
164 /// %2 = subview %arg0[%arg5, 0] [16, 32] [1, 1]
165 /// : memref<256x32xf32> to memref<16x32xf32, #map0>
166 /// %3 = subview %arg1[0, 0] [32, 32] [1, 1]
167 /// : memref<32x32xf32> to memref<32x32xf32, #map1>
168 /// %4 = subview %arg3[0, 0] [32, 32] [1, 1]
169 /// : memref<32x32xf32> to memref<32x32xf32, #map1>
170 /// linalg.matmul
171 /// ins(%2, %3 : memref<16x32xf32, #map0>, memref<32x32xf32, #map1>)
172 /// outs(%0 : memref<16x32xf32, #map0>)
173 /// linalg.matmul
174 /// ins(%0, %4 : memref<16x4xf32, #map0>, memref<4x8xf32, #map0>)
175 /// outs(%1 : memref<16x8xf32, #map0>)
176 /// }
177 ///
178 /// `tilingOptions` are used to tile the corresponding operation in `ops` (the
179 /// size of the former should be same as size of the latter. Based on how
180 /// tile+fuse is implemented, the fused loops are generated based on the last
181 /// operation in the sequence. For example, the tile sizes for the fused loops
182 /// is obtained from `tilingOptions.back()`. The following tiling options are
183 /// handled differently in tile+fuse (compared to tile only)
184 /// - Interchange of the tiling loops is not supported right now.
185 /// - Only the fused loops are distributed.
187  /// Operation obtained by tiling the last operation in sequence of `ops`
188  /// passed to `tileAndFuseLinalgOps`.
189  LinalgOp op;
190  /// The dimension of the loops that are fused.
191  std::set<unsigned> fusedLoopDims;
192  /// The generated fused operations (created within the fused loops).
194  /// The fused loop generated.
196 };
199  const LinalgDependenceGraph &dependenceGraph,
200  const LinalgTilingOptions &tilingOptions);
201 
202 /// Interchange the `iterator_types` and `iterator_maps` dimensions and adapts
203 /// the index accesses of `op`. This is an in-place transformation controlled by
204 /// `interchangeVector`. An empty vector is interpreted as the identity
205 /// permutation and the transformation returns early.
206 ///
207 /// E.g. the permutation `(i,j,k) -> (j,k,i)` is expressed with
208 /// `interchangeVector = [1,2,0]`. All values in `interchangeVector` must be
209 /// integers, in the range 0..`op.rank` without duplications
210 /// (i.e. `[1,1,2]` is an invalid permutation).
212  GenericOp genericOp,
213  ArrayRef<unsigned> interchangeVector);
214 
215 /// Create a GenericOp from the given named operation `namedOp` and replace
216 /// namedOp.
217 /// Return failure if `namedOp` is a GenericOp or misses a region builder.
219  LinalgOp namedOp);
220 
221 /// Callback function type used to perform the allocation for the promoted
222 /// `subView`. In `boundingSubViewsize` a best attempt is made to find the
223 /// smallest constant value for the size of the buffer needed for each
224 /// dimension. If that is not possible, contains the dynamic size of the
225 /// subview. The call back should return the buffer to use.
226 using AllocBufferCallbackFn = std::function<Optional<Value>(
227  OpBuilder &b, memref::SubViewOp subView,
228  ArrayRef<Value> boundingSubViewSize, DataLayout &layout)>;
229 
230 /// Callback function type used to deallocate the buffers used to hold the
231 /// promoted subview.
233  std::function<LogicalResult(OpBuilder &b, Value buffer)>;
234 
235 /// Callback function type used to insert copy from original subview to subview
236 /// of the promoted region for the read operands/subview of promoted region to
237 /// original subview for the results. The copy has to happen from `src` to
238 /// `dst`.
239 using CopyCallbackFn =
240  std::function<LogicalResult(OpBuilder &b, Value src, Value dst)>;
241 
243  /// Indices of subViews to promote. If `None`, try to promote all operands.
244  Optional<DenseSet<unsigned>> operandsToPromote = None;
246  operandsToPromote = DenseSet<unsigned>();
247  operandsToPromote->insert(operands.begin(), operands.end());
248  return *this;
249  }
250  /// If ith element of `useFullTiles` is true the full view should be used for
251  /// the promoted buffer of the ith operand in `operandsToPromote`. Otherwise
252  /// the partial view will be used.
253  /// The decision is defaulted to `useFullTileBuffersDefault` when
254  /// `useFullTileBuffers` is None and for operands missing from
255  /// `useFullTileBuffers`.
256  Optional<llvm::SmallBitVector> useFullTileBuffers = None;
258  unsigned size = useFullTiles.size();
259  llvm::SmallBitVector tmp(size, false);
260  for (unsigned i = 0; i < size; ++i)
261  tmp[i] = useFullTiles[i];
262  useFullTileBuffers = tmp;
263  return *this;
264  }
265  /// If true all operands unspecified by `useFullTileBuffers` will use the full
266  /// view, otherwise the partial view.
267  bool useFullTileBuffersDefault = false;
269  useFullTileBuffersDefault = use;
270  return *this;
271  }
272  /// Allow the use of dynamically-sized buffers.
273  bool dynamicBuffers = false;
275  dynamicBuffers = dynamic;
276  return *this;
277  }
278  /// Alignment of promoted buffer. If `None` do not specify alignment.
281  alignment = align;
282  return *this;
283  }
284  /// Use alloca with the default allocation scheme.
285  bool useAlloca = false;
287  useAlloca = use;
288  return *this;
289  }
290  /// Callback function to do the allocation of the promoted buffer. If None,
291  /// then the default allocation scheme of allocating a memref<?xi8> buffer
292  /// followed by a view operation is used.
297  DeallocBufferCallbackFn const &deallocFn) {
298  allocationFn = allocFn;
299  deallocationFn = deallocFn;
300  return *this;
301  }
302  /// Callback function to do the copy of data to and from the promoted
303  /// subview. If None then a memref.copy is used.
307  CopyCallbackFn const &copyOut) {
308  copyInFn = copyIn;
309  copyOutFn = copyOut;
310  return *this;
311  }
312 };
313 
314 /// Create a new buffer using the `allocationFn` provided. The size of this
315 /// buffer is the smallest constant bounding size along each dimension that can
316 /// be computed for the size of the result of `subView`. Returns the allocated
317 /// buffer as `fullLocalView` and the view that matches the size of the result
318 /// of subview operation as `partialLocalView`.
322 };
324 promoteSubviewAsNewBuffer(OpBuilder &b, Location loc, memref::SubViewOp subView,
325  const AllocBufferCallbackFn &allocationFn,
326  DataLayout &layout);
327 
328 /// Promote the `subViews` into a new buffer allocated at the insertion point
329 /// `b`. Promotion occurs in 3 steps:
330 /// 1. Create a new buffer for a full tile (i.e. not clipped at the boundary).
331 /// 2. Take a full view on the buffer.
332 /// 3. Take a partial slice of the full view in step 2. and copy into it.
333 /// Infers statically sized buffers from subViews unless `dynamicBuffers` is
334 /// true.
335 ///
336 /// Return the modified linalg op (the modification happens in place) as well
337 /// as all the copy ops created.
340 
341 /// Emit a suitable vector form for a Linalg op with fully static shape.
342 LogicalResult vectorize(RewriterBase &builder, LinalgOp linalgOp);
343 
344 /// Emit a suitable vector form for a Copy op with fully static shape.
345 LogicalResult vectorizeCopy(RewriterBase &builder, memref::CopyOp copyOp);
346 
347 /// Emit a loop nest of `scf.for` with the proper body for `linalgOp`.
349  LinalgOp linalgOp);
350 
351 /// Emit a loop nest of `scf.parallel` with the proper body for `linalgOp`.
353  LinalgOp linalgOp);
354 
355 /// Emit a loop nest of `affine.for` with the proper body for `linalgOp`.
357  LinalgOp linalgOp);
358 
359 //===----------------------------------------------------------------------===//
360 // Preconditions that ensure the corresponding transformation succeeds and can
361 // be applied as a rewrite pattern.
362 //===----------------------------------------------------------------------===//
363 /// Promote memref.subviews feeding linalg-on-buffers operations.
366 
367 //===----------------------------------------------------------------------===//
368 // Transformations exposed as rewrite patterns.
369 //===----------------------------------------------------------------------===//
370 // Marker used as attribute name in generated Linalg rewriting transformations.
372  static const StringLiteral kLinalgTransformMarker;
373 };
374 
375 /// Helper class to control application of linalg transformation patterns.
376 /// Control comes in 2 forms:
377 /// 1. attribute matching and setting behavior using the attribute named
378 /// `kLinalgTransformMarker`. This can be used to build a state machine
379 /// using attributes and incrementally applying patterns to advance states.
380 /// 2. filter function, which is a simple lambda on the Operation* that
381 /// returns a LogicalResult.
383  using FilterFunction = std::function<LogicalResult(Operation *)>;
384 
386  ArrayRef<StringAttr> matchDisjunction = {},
387  Optional<StringAttr> replacement = None);
388 
390  const FilterFunction &f, ArrayRef<StringAttr> matchDisjunction = {},
391  Optional<StringAttr> replacement = None);
392 
395  LogicalResult checkAndNotify(PatternRewriter &rewriter, Operation *op) const;
396  void replaceLinalgTransformationFilter(PatternRewriter &rewriter,
397  Operation *op) const;
398  bool hasReplacementFilter(Operation *op) const;
399 
401  if (f)
402  filters.push_back(f);
403  return *this;
404  }
405 
406  template <typename... OpTypes>
408  return addFilter(
409  [](Operation *op) { return success(isa<OpTypes...>(op)); });
410  }
411 
413  return addFilter([opName](Operation *op) {
414  return success(op->getName().getStringRef() == opName);
415  });
416  }
417 
419  matchByDefault = true;
420  return *this;
421  }
422 
423 private:
425  SmallVector<StringAttr> matchDisjunction;
426  Optional<StringAttr> replacement;
427  /// When set to true, if the attribute is not set, it will be treated as
428  /// a match. Default is false.
429  bool matchByDefault;
430 };
431 
433  std::function<SmallVector<Value, 4>(OpBuilder &, Operation *)>;
434 
435 /// Creates a number of ranges equal to the number of non-zero in `tileSizes`.
436 /// One for each loop of the LinalgOp that is tiled. The `tileSizes` argument
437 /// has one entry per surrounding loop. It uses zero as the convention that a
438 /// particular loop is not tiled. This convention simplifies implementations by
439 /// avoiding affine map manipulations.
440 /// The returned ranges correspond to the loop ranges, in the proper order, that
441 /// are tiled and for which new loops will be created. Also the function returns
442 /// a map from loop indices of the LinalgOp to the corresponding non-empty range
443 /// indices of newly created loops.
445 std::tuple<SmallVector<Range, 4>, LoopIndexToRangeIndexMap>
447  ValueRange allShapeSizes, ValueRange allTileSizes);
448 
449 /// All indices returned by IndexOp should be invariant with respect to tiling.
450 /// Therefore, if an operation is tiled, we have to transform the indices
451 /// accordingly, i.e. offset them by the values of the corresponding induction
452 /// variables that are captured implicitly in the body of the op.
453 ///
454 /// Example. `linalg.generic` before tiling:
455 ///
456 /// #id_2d = (i, j) -> (i, j)
457 /// #pointwise_2d_trait = {
458 /// indexing_maps = [#id_2d, #id_2d],
459 /// iterator_types = ["parallel", "parallel"]
460 /// }
461 /// linalg.generic #pointwise_2d_trait %operand, %result {
462 /// ^bb0(%operand_in: f32, %result_in: f32):
463 /// %i = linalg.index 0 : index
464 /// %j = linalg.index 1 : index
465 /// <some operations that use %i, %j>
466 /// }: memref<50x100xf32>, memref<50x100xf32>
467 ///
468 /// After tiling pass with tiles sizes 10 and 25:
469 ///
470 /// #strided = (i, j)[s0, s1, s2] -> (i * s1 + s0 + j * s2)
471 ///
472 /// %c1 = arith.constant 1 : index
473 /// %c0 = arith.constant 0 : index
474 /// %c25 = arith.constant 25 : index
475 /// %c10 = arith.constant 10 : index
476 /// operand_dim_0 = dim %operand, 0 : memref<50x100xf32>
477 /// operand_dim_1 = dim %operand, 1 : memref<50x100xf32>
478 /// scf.for %k = %c0 to operand_dim_0 step %c10 {
479 /// scf.for %l = %c0 to operand_dim_1 step %c25 {
480 /// %4 = memref.subview %operand[%k, %l][%c10, %c25][%c1, %c1]
481 /// : memref<50x100xf32> to memref<?x?xf32, #strided>
482 /// %5 = memref.subview %result[%k, %l][%c10, %c25][%c1, %c1]
483 /// : memref<50x100xf32> to memref<?x?xf32, #strided>
484 /// linalg.generic pointwise_2d_trait %4, %5 {
485 /// ^bb0(%operand_in: f32, %result_in: f32):
486 /// %i = linalg.index 0 : index
487 /// %j = linalg.index 1 : index
488 /// // Indices `k` and `l` are implicitly captured in the body.
489 /// %transformed_i = arith.addi %i, %k : index // index `i` is offset by
490 /// %k %transformed_j = arith.addi %j, %l : index // index `j` is offset
491 /// by %l
492 /// // Every use of %i, %j is replaced with %transformed_i, %transformed_j
493 /// <some operations that use %transformed_i, %transformed_j>
494 /// }: memref<?x?xf32, #strided>, memref<?x?xf32, #strided>
495 /// }
496 /// }
497 ///
498 /// TODO: Investigate whether mixing implicit and explicit indices
499 /// does not lead to losing information.
500 void transformIndexOps(RewriterBase &b, LinalgOp op,
502  const LoopIndexToRangeIndexMap &loopIndexToRangeIndex);
503 
505  /// A padding value for every operand.
508  paddingValues.assign(pv.begin(), pv.end());
509  return *this;
510  }
511  /// A list of iterator dimensions to pad.
514  paddingDimensions.assign(pd.begin(), pd.end());
515  return *this;
516  }
517  /// A flag for every operand to mark the PadOp as nofold which enables packing
518  /// for statically shaped operands.
521  packPaddings.assign(pp.begin(), pp.end());
522  return *this;
523  }
524  /// A number of loops to hoist the PadOp out for every operand.
527  hoistPaddings.assign(hp.begin(), hp.end());
528  return *this;
529  }
530  /// A permutation vector for every operand used to transpose the packed PadOp
531  /// results.
535  transposePaddings.assign(tp.begin(), tp.end());
536  return *this;
537  }
538 };
539 
541  /// Tile sizes used to tile the root operation.
544  tileSizes.assign(ts.begin(), ts.end());
545  return *this;
546  }
547  /// Tile interchange used to permute the tile loops.
549  /// When specified, specifies distribution of generated tile loops to
550  /// processors.
554  tileDistribution = std::move(distributionOptions);
555  return *this;
556  }
557 };
558 
560  /// Computation function that returns the tile sizes for each operation.
561  /// Delayed construction of constant tile sizes should occur to interoperate
562  /// with folding.
563  TileSizeComputationFunction tileSizeComputationFunction = nullptr;
564 
567  tileSizeComputationFunction = std::move(fun);
568  return *this;
569  }
570  /// Set the `tileSizeComputationFunction` to return the values `ts`. The
571  /// values must not fold away when tiling. Otherwise, use a more robust
572  /// `tileSizeComputationFunction`.
574  tileSizeComputationFunction = [=](OpBuilder &, Operation *) { return ts; };
575  return *this;
576  }
577  /// Convenience function to set the `tileSizeComputationFunction` to a
578  /// function that computes tile sizes at the point they are needed. Allows
579  /// proper interaction with folding.
580  LinalgTilingOptions &setTileSizes(ArrayRef<int64_t> ts);
581 
582  /// Tile all dynamic dimensions by 1. I.e., scalarize those dimensions.
583  /// Note: `scalarizeDynamicDims` and `setTileSizes` cannot be used together.
584  LinalgTilingOptions &scalarizeDynamicDims();
585 
586  /// The interchange vector to reorder the tiled loops.
587  SmallVector<unsigned, 4> interchangeVector = {};
588 
590  interchangeVector.assign(interchange.begin(), interchange.end());
591  return *this;
592  }
593 
594  /// The type of tile loops to generate.
596 
598  loopType = lt;
599  return *this;
600  }
601 
602  /// When specified, specifies distribution of generated tile loops to
603  /// processors.
605 
608  distribution = std::move(distributionOptions);
609  return *this;
610  }
611 
612  /// Specification markers of how to distribute the `linalg.tiled_loop`.
613  SmallVector<StringRef, 2> distributionTypes = {};
614 
616  distributionTypes.assign(types.begin(), types.end());
617  return *this;
618  }
619 
620  /// Peel the specified loops.
622 
624  peeledLoops.clear();
625  peeledLoops.append(loops.begin(), loops.end());
626  return *this;
627  }
628 };
629 
630 /// Canonicalization patterns relevant to apply after tiling patterns. These are
631 /// applied automatically by the tiling pass but need to be applied manually
632 /// when tiling is called programmatically.
635 
636 ///
637 /// Linalg tiling pattern.
638 ///
639 /// Apply the `tiling` transformation as a pattern.
640 /// `filter` controls LinalgTransformMarker matching and update when specified.
641 /// See `tiling` for more details.
642 // TODO: TiledOpInterface
644  /// Construct a generic pattern applied to all LinalgOp that verify `filter`.
648  PatternBenefit benefit = 1);
649 
650  /// Construct a pattern specifically applied to `opName`.
652  StringRef opName, MLIRContext *context, LinalgTilingOptions options,
654  PatternBenefit benefit = 1);
655 
656  /// `matchAndRewrite` implementation that returns the significant transformed
657  /// pieces of IR.
659  returningMatchAndRewrite(LinalgOp op, PatternRewriter &rewriter) const;
660 
662  PatternRewriter &rewriter) const override {
663  return returningMatchAndRewrite(op, rewriter);
664  }
665 
666 private:
667  /// LinalgTransformMarker handles special attribute manipulations.
669  /// Options to control tiling;
671 };
672 
673 ///
674 /// Linalg padding pattern.
675 ///
676 /// Apply the `padding` transformation as a pattern.
677 /// `filter` controls LinalgTransformMarker matching and update when specified.
678 /// See `padding` for more details.
680  /// Construct a generic pattern applied to all LinalgOp that verify `filter`.
682  MLIRContext *context,
685  PatternBenefit benefit = 1);
686 
687  /// Construct a pattern specifically applied to `opName`.
689  StringRef opName, MLIRContext *context,
692  PatternBenefit benefit = 1);
693 
694  /// `matchAndRewrite` implementation that returns the significant transformed
695  /// pieces of IR.
696  FailureOr<LinalgOp> returningMatchAndRewrite(LinalgOp op,
697  PatternRewriter &rewriter) const;
698 
700  PatternRewriter &rewriter) const override {
701  return returningMatchAndRewrite(op, rewriter);
702  }
703 
704 private:
705  /// LinalgTransformMarker handles special attribute manipulations.
707  /// Options to control padding and hoisting.
709 };
710 
712  /// List of operands indices to use for fusion.
713  llvm::SmallSet<unsigned, 1> indicesToFuse = {};
715  indicesToFuse.insert(operands.begin(), operands.end());
716  return *this;
717  }
718 };
719 
722  StringRef opName, MLIRContext *context,
723  const LinalgDependenceGraph &dependenceGraph,
724  LinalgTilingOptions tilingOptions, LinalgFusionOptions fusionOptions,
727  LinalgTransformationFilter originalOpMarker =
729  PatternBenefit benefit = 1);
730  LogicalResult matchAndRewrite(Operation *op,
731  PatternRewriter &rewriter) const override;
732 
733 private:
734  /// Dependence graph needed for fusion.
735  const LinalgDependenceGraph &dependenceGraph;
736  /// Options to control tiling.
737  LinalgTilingOptions tilingOptions;
738  /// Options to control fusion.
739  LinalgFusionOptions fusionOptions;
740  /// Marker to control application of the pattern.
742  /// Marker set on the fused op after tile and fuse.
743  LinalgTransformationFilter fusedOpMarker;
744  /// The dependenceGraph is not modifiable, i.e. if the Linalg operations used
745  /// to build the dependence graph changes then the dependenceGraph needs to be
746  /// recomputed right now. To not invalidate the dependenceGraph as
747  /// transformation happens, the original producer can be tagged with a filter
748  /// that can be later used to delete the original operations.
749  LinalgTransformationFilter originalOpMarker;
750 };
751 
752 template <typename OpTy>
755  MLIRContext *context, const LinalgDependenceGraph &dependenceGraph,
756  LinalgTilingOptions tilingOptions, LinalgFusionOptions fusionOptions,
759  LinalgTransformationFilter originalOpMarker =
761  PatternBenefit benefit = 1)
763  OpTy::getOperationName(), context, dependenceGraph, tilingOptions,
764  fusionOptions, f, fusedOpMarker, originalOpMarker, benefit) {}
765 };
766 
767 ///
768 /// Linalg tile and fuse tensor ops pattern.
769 ///
770 /// Apply tiling and fusion as a pattern.
771 /// `filter` controls LinalgTransformMarker matching and update when specified.
772 /// See `tileConsumerAndFuseProducers` for more details.
774  // Entry point to match any LinalgOp.
778  PatternBenefit benefit = 1);
779  // Entry point to match a specific LinalgOp.
781  StringRef opName, MLIRContext *context,
784  PatternBenefit benefit = 1);
785 
786  /// `matchAndRewrite` implementation that returns the significant transformed
787  /// pieces of IR.
789  returningMatchAndRewrite(Operation *op, PatternRewriter &rewriter) const;
790 
792  PatternRewriter &rewriter) const override {
793  return returningMatchAndRewrite(op, rewriter);
794  }
795 
796 private:
797  /// LinalgTransformMarker handles special attribute manipulations.
799  /// Tile sizes and interchange used to tile the root operation.
801 };
802 
803 ///
804 /// Linalg generic interchange pattern.
805 ///
806 /// Apply the `interchange` transformation on a RewriterBase.
807 /// `filter` controls LinalgTransformMarker matching and update when specified.
808 /// See `interchange` for more details.
809 struct GenericOpInterchangePattern : public OpRewritePattern<GenericOp> {
811 
812  /// GenericOp-specific constructor with an optional `filter`.
814  MLIRContext *context, ArrayRef<unsigned> interchangeVector,
816  PatternBenefit benefit = 1);
817 
818  /// `matchAndRewrite` implementation that returns the significant transformed
819  /// pieces of IR.
821  returningMatchAndRewrite(GenericOp op, PatternRewriter &rewriter) const;
822 
824  PatternRewriter &rewriter) const override {
825  return returningMatchAndRewrite(op, rewriter);
826  }
827 
828 private:
829  /// LinalgTransformMarker handles special attribute manipulations.
831  /// The interchange vector to reorder the iterators and indexing_maps dims.
832  SmallVector<unsigned, 8> interchangeVector;
833 };
834 
835 ///
836 /// Linalg generalization pattern.
837 ///
838 /// Apply the `generalization` transformation as a pattern.
839 /// `filter` controls LinalgTransformMarker matching and update when specified.
840 /// See `generalization` for more details.
842  : public OpInterfaceRewritePattern<LinalgOp> {
843  /// Construct a generic pattern applied to all LinalgOp that verify `filter`.
845  MLIRContext *context,
847  PatternBenefit benefit = 1);
848 
849  /// Construct a pattern specifically applied to `opName`.
851  StringRef opName, MLIRContext *context,
853  PatternBenefit benefit = 1);
854 
855  /// `matchAndRewrite` implementation that returns the significant transformed
856  /// pieces of IR.
858  returningMatchAndRewrite(LinalgOp op, PatternRewriter &rewriter) const;
859 
861  PatternRewriter &rewriter) const override {
862  return returningMatchAndRewrite(op, rewriter);
863  }
864 
865 private:
866  /// LinalgTransformMarker handles special attribute manipulations.
868 };
869 
870 ///
871 /// Linalg promotion patterns.
872 ///
873 /// Apply the `promoteSubViews` transformation as a pattern.
874 /// `filter` controls LinalgTransformMarker matching and update when specified.
875 /// See `promoteSubViews` for more details.
877  /// Entry point to match any LinalgOp OpInterface.
878  /// MatchAnyOpTag-based constructor with a mandatory `filter`.
882  PatternBenefit benefit = 1);
883  /// Entry point to match a specific Linalg op.
885  StringRef opName, MLIRContext *context, LinalgPromotionOptions options,
887  PatternBenefit benefit = 1);
888 
889  LogicalResult matchAndRewrite(Operation *op,
890  PatternRewriter &rewriter) const override;
891 
892 private:
893  /// LinalgTransformMarker handles special attribute manipulations.
895  /// Promotion options.
897 };
898 
899 template <typename OpTy>
901  /// SFINAE: This constructor can only trigger for concrete ops that have a
902  /// static `getOperationName` method.
903  template <typename ConcreateOpTy = OpTy>
907  PatternBenefit benefit = 1)
908  : LinalgBasePromotionPattern(OpTy::getOperationName(), context, options,
909  f, benefit) {}
910  /// This constructor is available to anyone.
912  StringRef opName, MLIRContext *context, LinalgPromotionOptions options,
914  PatternBenefit benefit = 1)
915  : LinalgBasePromotionPattern(opName, context, options, f, benefit) {}
916 };
917 
918 ///
919 /// Linalg vectorization patterns.
920 ///
921 /// Empty for now, used for SFINAE purposes only.
923 
924 /// `filter` controls LinalgTransformMarker matching and update when specified.
925 /// See `vectorizeLinalgOp` for more details.
927  /// Construct a generic pattern applied to all LinalgOp that verify `filter`.
929  MLIRContext *context,
932  PatternBenefit benefit = 1);
933 
934  /// Construct a pattern specifically applied to `opName`.
936  StringRef opName, MLIRContext *context,
939  PatternBenefit benefit = 1);
940 
941  LogicalResult matchAndRewrite(LinalgOp linalgOp,
942  PatternRewriter &rewriter) const override;
943 
944 private:
945  /// LinalgTransformMarker handles special attribute manipulations.
947 };
948 
949 /// `filter` controls LinalgTransformMarker matching and update when specified.
950 /// See `vectorizeLinalgOp` for more details.
951 struct CopyVectorizationPattern : public OpRewritePattern<memref::CopyOp> {
953 
954  LogicalResult matchAndRewrite(memref::CopyOp copyOp,
955  PatternRewriter &rewriter) const override;
956 };
957 
958 /// Return vector::CombiningKind for the given op.
960 
961 //===----------------------------------------------------------------------===//
962 // Transformation and lowering options exposed as auxiliary structs.
963 //===----------------------------------------------------------------------===//
964 /// Options to control the application of enabling transformations.
965 /// Hoisting transformations are always deemed beneficial and must be disabled
966 /// explicitly.
968  /// Enable LICM.
969  bool licm = true;
970  LinalgEnablingOptions &enableLICM(bool val = true) {
971  licm = val;
972  return *this;
973  }
974  /// Enable hoisting of redundant vector transfer ops.
977  hoistRedundantVectorTransfers = val;
978  return *this;
979  }
980  /// Enable hoisting of redundant vector transfer ops on tensor.
984  hoistRedundantVectorTransfersOnTensor = val;
985  return *this;
986  }
987 };
988 
989 /// Vector lowering options control how ops are lowered down to 1-D and scf.for
990 /// form.
992  /// Enable lowering of vector.contract.
993  /// In a progressive lowering of vectors, this would be the 1st step.
994  bool contractionLowering = false;
996  contractionLowering = val;
997  return *this;
998  }
999  /// Enable lowering of vector.multi_reduce.
1000  /// In a progressive lowering of vectors, this would be the 2nd step.
1001  bool multiReductionLowering = false;
1003  multiReductionLowering = val;
1004  return *this;
1005  }
1006  /// Trigger full / partial vector.transfer splits.
1007  /// In a progressive lowering of vectors, this would be the 3rd step.
1008  bool transferPartialRewrite = false;
1010  transferPartialRewrite = val;
1011  return *this;
1012  }
1013  /// Enable lowering of vector.transfer to scf.
1014  /// In a progressive lowering of vectors, this would be the 4th step.
1015  bool transferToSCFConversion = false;
1017  transferToSCFConversion = val;
1018  return *this;
1019  }
1020  /// Maximal transfer rank under which we do not lower further.
1021  int64_t maxTransferRank = 1;
1023  maxTransferRank = val;
1024  return *this;
1025  }
1026  /// Vector lowering operations may result in surprising behavior when
1027  /// composing multiple codegen strategies and must be enabled explicitly.
1028  /// In a progressive lowering of vectors, this would be the 5th step.
1029  bool transferLowering = true;
1031  transferLowering = val;
1032  return *this;
1033  }
1034  /// Enable lowering of vector.shape_cast to insert/extract.
1035  /// In a progressive lowering of vectors, this would be the 6th step.
1036  bool shapeCastLowering = true;
1038  shapeCastLowering = val;
1039  return *this;
1040  }
1041  /// Enable lowering of vector.transpose.
1042  /// In a progressive lowering of vectors, this would be the 7th step.
1043  bool transposeLowering = false;
1045  transposeLowering = val;
1046  return *this;
1047  }
1048  /// Enable AVX2-specific lowerings.
1049  bool avx2Lowering = false;
1051  avx2Lowering = val;
1052  return *this;
1053  }
1054 
1055  /// Configure the post staged-patterns late vector.transfer to scf
1056  /// conversion.
1060  vectorTransferToSCFOptions = options;
1061  return *this;
1062  }
1063  /// Configure late vector transformations.
1067  vectorTransformOptions = options;
1068  return *this;
1069  }
1070  /// Configure specialized vector lowerings.
1074  avx2LoweringOptions = options;
1075  return *this;
1076  }
1077 };
1078 
1079 //===----------------------------------------------------------------------===//
1080 // Transformations exposed as rewrite patterns.
1081 //===----------------------------------------------------------------------===//
1082 ///
1083 /// Linalg lowering patterns.
1084 ///
1085 /// Apply the `linalgLowerOpToLoops` transformation as a pattern.
1086 /// `filter` controls LinalgTransformMarker matching and update when specified.
1087 /// See `linalgLowerOpToLoops` for more details.
1089  LibraryCall = 0,
1090  Loops = 1,
1091  AffineLoops = 2,
1092  ParallelLoops = 3
1093 };
1094 
1095 template <typename OpTy>
1098  MLIRContext *context, LinalgLoweringType loweringType,
1100  PatternBenefit benefit = 1)
1101  : RewritePattern(OpTy::getOperationName(), benefit, context),
1102  filter(std::move(f)), loweringType(loweringType) {}
1103 
1104  // TODO: Move implementation to .cpp once named ops are auto-generated.
1106  PatternRewriter &rewriter) const override {
1107  LinalgOp linalgOp = dyn_cast<LinalgOp>(op);
1108  if (!linalgOp)
1109  return failure();
1110  if (failed(filter.checkAndNotify(rewriter, linalgOp)))
1111  return failure();
1112 
1113  switch (loweringType) {
1114  case LinalgLoweringType::LibraryCall:
1115  // TODO: Move lowering to library calls here.
1116  return failure();
1118  if (failed(linalgOpToLoops(rewriter, op)))
1119  return failure();
1120  break;
1121  case LinalgLoweringType::AffineLoops:
1122  if (failed(linalgOpToAffineLoops(rewriter, op)))
1123  return failure();
1124  break;
1125  case LinalgLoweringType::ParallelLoops:
1126  if (failed(linalgOpToParallelLoops(rewriter, op)))
1127  return failure();
1128  break;
1129  }
1130 
1131  rewriter.eraseOp(op);
1132  return success();
1133  }
1134 
1135 private:
1136  /// LinalgTransformMarker handles special attribute manipulations.
1138  /// Controls whether the pattern lowers to library calls, scf.for, affine.for
1139  /// or scf.parallel.
1140  LinalgLoweringType loweringType;
1141 };
1142 
1143 /// Linalg generalization patterns
1144 
1145 /// Populates `patterns` with patterns to convert spec-generated named ops to
1146 /// linalg.generic ops.
1148  RewritePatternSet &patterns,
1150 
1151 /// Linalg decompose convolutions patterns
1152 
1153 /// Populates patterns to decompose high-D convolution ops into low-D ones. This
1154 /// is a step in progressive lowering for convolution ops, afterwards we can
1155 /// vectorize the low-D convolution ops.
1157  RewritePatternSet &patterns,
1159  PatternBenefit benefit = 1);
1160 
1161 //===----------------------------------------------------------------------===//
1162 // Op-specific patterns.
1163 //===----------------------------------------------------------------------===//
1164 
1165 /// tensor::PadOp is not canonicalized away yet, so we provide a transformation
1166 /// to `linalg.generic`.
1167 struct PadOpTransformationPattern : public OpRewritePattern<tensor::PadOp> {
1169 
1170  LogicalResult matchAndRewrite(tensor::PadOp padOp,
1171  PatternRewriter &rewriter) const override;
1172 };
1173 
1174 /// Pad the iterator dimensions `paddingDimensions` of all `opToPad` operands to
1175 /// a static bounding box. Use `paddingValues` and `packPaddings` to set padding
1176 /// value and nofold attribute of the created tensor::PadOps, respectively.
1177 /// Update `paddedOp` to the cloned operation with statically shaped
1178 /// `paddingDimensions` and return the extracted dynamically shaped results. If
1179 /// padding fails, return failure.
1181 rewriteAsPaddedOp(OpBuilder &b, LinalgOp opToPad,
1182  ArrayRef<int64_t> paddingDimensions,
1183  ArrayRef<Attribute> paddingValues,
1184  ArrayRef<bool> packPaddings, LinalgOp &paddedOp);
1185 
1186 using OptimizeCopyFn =
1187  std::function<LogicalResult(PatternRewriter &, tensor::PadOp, Value)>;
1188 
1189 /// Rewrite a tensor::PadOp into a sequence of InitTensorOp, FillOp and
1190 /// InsertSliceOp. For now, only constant padding values are supported.
1191 /// `OptimizeCopyFn` can be used to customize copying step optimization.
1192 struct GeneralizePadOpPattern : public OpRewritePattern<tensor::PadOp> {
1194  OptimizeCopyFn optimizeCopyFn = nullptr,
1195  PatternBenefit benefit = 1)
1196  : OpRewritePattern<tensor::PadOp>(context, benefit),
1197  optimizeCopyFn(std::move(optimizeCopyFn)) {}
1198  LogicalResult matchAndRewrite(tensor::PadOp padOp,
1199  PatternRewriter &rewriter) const override;
1200 
1201 protected:
1203  Value createFillOrGenerateOp(PatternRewriter &rewriter, tensor::PadOp padOp,
1204  Value dest,
1205  const SmallVector<Value> &dynSizes) const;
1206 };
1207 
1208 /// Populates `patterns` with patterns that vectorize linalg.pad_tensor.
1209 /// These patterns are meant to apply in a complementary fashion. Benefits
1210 /// are used to encode a certain ordering of pattern application. To avoid
1211 /// scattering magic constants throughout the code base, the patterns must be
1212 /// added with this function. `baseBenefit` can be used to offset the benefit
1213 /// of all tensor::PadOp vectorization patterns by a certain value.
1215  PatternBenefit baseBenefit = 1);
1216 
1217 /// Match and rewrite for the pattern:
1218 /// ```
1219 /// %alloc = ...
1220 /// [optional] %view = memref.view %alloc ...
1221 /// %subView = subview %allocOrView ...
1222 /// [optional] linalg.fill(%allocOrView, %cst) ...
1223 /// ...
1224 /// memref.copy(%in, %subView) ...
1225 /// vector.transfer_read %allocOrView[...], %cst ...
1226 /// ```
1227 /// into
1228 /// ```
1229 /// [unchanged] %alloc = ...
1230 /// [unchanged] [optional] %view = memref.view %alloc ...
1231 /// [unchanged] [unchanged] %subView = subview %allocOrView ...
1232 /// ...
1233 /// vector.transfer_read %in[...], %cst ...
1234 /// ```
1235 /// Where there is no interleaved use between memref.copy and transfer_read as
1236 /// well as no interleaved use between linalg.fill and memref.copy (if
1237 /// linalg.fill is specified).
1238 /// This is a custom rewrite to forward partial reads (with optional fills) to
1239 /// vector.transfer_read.
1241  : public OpRewritePattern<vector::TransferReadOp> {
1243 
1244  LogicalResult matchAndRewrite(vector::TransferReadOp xferOp,
1245  PatternRewriter &rewriter) const override;
1246 };
1247 
1248 /// Match and rewrite for the pattern:
1249 /// ```
1250 /// %alloc = ...
1251 /// [optional] %view = memref.view %alloc ...
1252 /// %subView = subview %allocOrView...
1253 /// ...
1254 /// vector.transfer_write %..., %allocOrView[...]
1255 /// memref.copy(%subView, %out)
1256 /// ```
1257 /// into
1258 /// ```
1259 /// [unchanged] %alloc = ...
1260 /// [unchanged] [optional] %view = memref.view %alloc ...
1261 /// [unchanged] %subView = subview %allocOrView...
1262 /// ...
1263 /// vector.transfer_write %..., %out[...]
1264 /// ```
1265 /// Where there is no interleaved use between transfer_write and memref.copy.
1266 /// This is a custom rewrite to forward partial writes to vector.transfer_write.
1268  : public OpRewritePattern<vector::TransferWriteOp> {
1270 
1271  LogicalResult matchAndRewrite(vector::TransferWriteOp xferOp,
1272  PatternRewriter &rewriter) const override;
1273 };
1274 
1275 //===----------------------------------------------------------------------===//
1276 // Support for staged pattern application.
1277 //===----------------------------------------------------------------------===//
1278 /// Helper function to allow applying rewrite patterns, interleaved with more
1279 /// global transformations, in a staged fashion:
1280 /// 1. the first stage consists of a list of FrozenRewritePatternSet. Each
1281 /// FrozenRewritePatternSet in this list is applied once, in order.
1282 /// 2. the second stage consists of a single RewritePattern that is applied
1283 /// greedily until convergence.
1284 /// 3. the third stage consists of applying a lambda, generally used for
1285 /// non-local transformation effects. This allows creating custom fused
1286 /// transformations where patterns can be ordered and applied at a finer
1287 /// granularity than a sequence of traditional compiler passes.
1289  Operation *op, ArrayRef<FrozenRewritePatternSet> stage1Patterns,
1290  const FrozenRewritePatternSet &stage2Patterns,
1291  function_ref<LogicalResult(Operation *)> stage3Lambda = nullptr);
1292 
1293 /// Rewrite extract_slice(pad_tensor(x)) into pad_tensor(extract_slice(x)).
1295  : public OpRewritePattern<tensor::ExtractSliceOp> {
1296  /// A function to control pattern application and rewrite logic.
1297  ///
1298  /// The function will be given the slice op and should return:
1299  /// - None: to fail the match and not apply the pattern;
1300  /// - true: to apply the pattern with zero slice guard;
1301  /// - false: to apply the pattern without zero slice guard.
1302  ///
1303  /// See the documentation for tensor::bubbleUpPadSlice regarding zero slice
1304  /// guard.
1305  using ControlFn = std::function<llvm::Optional<bool>(tensor::ExtractSliceOp)>;
1306 
1308  ControlFn controlFn = nullptr,
1309  PatternBenefit benefit = 1)
1310  : OpRewritePattern(context, benefit), controlFn(std::move(controlFn)) {}
1311 
1312  LogicalResult matchAndRewrite(tensor::ExtractSliceOp sliceOp,
1313  PatternRewriter &rewriter) const override;
1314 
1315 private:
1316  ControlFn controlFn;
1317 };
1318 
1319 //===----------------------------------------------------------------------===//
1320 // Helper classes for type list expansion.
1321 //===----------------------------------------------------------------------===//
1322 template <typename... OpTypes>
1324 
1325 template <>
1327 public:
1328  static void insert(RewritePatternSet &patterns,
1329  const LinalgVectorizationOptions &options,
1330  const LinalgTransformationFilter &f) {}
1331 };
1332 
1333 template <typename OpTy, typename... OpTypes>
1334 class VectorizationPatterns<OpTy, OpTypes...> {
1335 public:
1336  static void insert(RewritePatternSet &patterns,
1337  const LinalgVectorizationOptions &options,
1338  const LinalgTransformationFilter &f) {
1339  patterns.add<LinalgVectorizationPattern>(OpTy::getOperationName(),
1340  patterns.getContext(), options, f);
1341  VectorizationPatterns<OpTypes...>::insert(patterns, options, f);
1342  }
1343 };
1344 
1345 template <typename... OpTypes>
1347 
1348 template <>
1350 public:
1351  static void insert(RewritePatternSet &patterns,
1352  const LinalgTilingOptions &options,
1353  const LinalgTransformationFilter &f) {}
1354 };
1355 
1356 template <typename OpTy, typename... OpTypes>
1357 class TilingPatterns<OpTy, OpTypes...> {
1358 public:
1359  static void insert(RewritePatternSet &patterns,
1360  const LinalgTilingOptions &options,
1361  const LinalgTransformationFilter &f) {
1362  patterns.add<LinalgTilingPattern>(OpTy::getOperationName(),
1363  patterns.getContext(), options, f);
1364  TilingPatterns<OpTypes...>::insert(patterns, options, f);
1365  }
1366 };
1367 
1368 /// Function signature to control reduction splitting. This returns a pair
1369 /// containing a ratio and a dimension index. The ratio is used to split the
1370 /// reduction dimension. The dimension index is used to control where the extra
1371 /// dimension is added to the intermediate tensor shape. If the ratio value is
1372 /// less or equal to 1 then nothing will be done.
1374  std::function<std::pair<int64_t, unsigned>(LinalgOp op)>;
1375 
1376 /// Patterns to apply `splitReduction` below.
1378  RewritePatternSet &patterns,
1379  const ControlSplitReductionFn &controlSplitReductionFn,
1381 
1382 /// Apply transformation to split the single linalg op reduction into a parallel
1383 /// and reduction dimension. Then create a new linalg.generic op doing the rest
1384 /// of the reduction. Return the new linalg op with an extra parallel dimension
1385 /// or failure if the transformation didn't happen.
1386 /// Example:
1387 /// ```
1388 /// %r = linalg.generic {indexing_maps = [affine_map<(d0) -> (d0)>,
1389 /// affine_map<(d0) -> ()>],
1390 /// iterator_types = ["reduction"]}
1391 /// ins(%in : tensor<32xf32>)
1392 /// outs(%out : tensor<f32>) {
1393 /// ^bb0(%arg1: f32, %arg2: f32):
1394 /// %y = arith.addf %arg1, %arg2 : f32
1395 /// linalg.yield %y : f32
1396 /// } -> tensor<f32>
1397 /// ```
1398 /// To:
1399 /// ```
1400 /// %cst = arith.constant 0.000000e+00 : f32
1401 /// %0 = tensor.expand_shape %in [[0, 1]] : tensor<32xf32> into tensor<4x8xf32>
1402 /// %1 = linalg.init_tensor [4] : tensor<4xf32>
1403 /// %2 = linalg.fill ins(%cst : f32) outs(%1 : tensor<4xf32>) -> tensor<4xf32>
1404 /// %3 = linalg.generic {indexing_maps = [affine_map<(d0, d1) -> (d0, d1)>,
1405 /// affine_map<(d0, d1) -> (d0)>],
1406 /// iterator_types = ["parallel", "reduction"]}
1407 /// ins(%0 : tensor<4x8xf32>) outs(%2 : tensor<4xf32>) {
1408 /// ^bb0(%arg3: f32, %arg5: f32):
1409 /// %5 = arith.addf %arg3, %arg4 : f32
1410 /// linalg.yield %5 : f32
1411 /// } -> tensor<4xf32>
1412 /// %r = linalg.generic {indexing_maps = [affine_map<(d0) -> (d0)>,
1413 /// affine_map<(d0) -> ()>],
1414 /// iterator_types = ["reduction"]}
1415 /// ins(%3 : tensor<4xf32>) outs(%out : tensor<f32>) {
1416 /// ^bb0(%arg3: f32, %arg4: f32):
1417 /// %5 = arith.addf %arg3, %arg4 : f32
1418 /// linalg.yield %5 : f32
1419 /// } -> tensor<f32>
1420 /// ```
1422 splitReduction(PatternRewriter &b, LinalgOp op,
1423  const ControlSplitReductionFn &controlSplitReductionFn,
1424  const LinalgTransformationFilter &f);
1425 
1426 } // namespace linalg
1427 } // namespace mlir
1428 
1429 #endif // MLIR_DIALECT_LINALG_TRANSFORMS_TRANSFORMS_H
Include the generated interface declarations.
LinalgPromotionOptions & setCopyInOutFns(CopyCallbackFn const &copyIn, CopyCallbackFn const &copyOut)
Definition: Transforms.h:306
LogicalResult matchAndRewrite(Operation *op, PatternRewriter &rewriter) const override
Attempt to match against code rooted at the specified operation, which is the same operation code as ...
Definition: Transforms.h:791
SmallVector< SmallVector< int64_t > > transposePaddings
A permutation vector for every operand used to transpose the packed PadOp results.
Definition: Transforms.h:532
Helper class to control application of linalg transformation patterns.
Definition: Transforms.h:382
LinalgVectorLoweringOptions & enableContractionLowering(bool val=true)
Definition: Transforms.h:995
LinalgPaddingOptions & setHoistPaddings(ArrayRef< int64_t > hp)
Definition: Transforms.h:526
LogicalResult matchAndRewrite(Operation *op, PatternRewriter &rewriter) const override
Attempt to match against code rooted at the specified operation, which is the same operation code as ...
Definition: Transforms.h:1105
LinalgLoweringType
Linalg lowering patterns.
Definition: Transforms.h:1088
Linalg tiling pattern.
Definition: Transforms.h:643
Options for controlling specialized AVX2 lowerings.
Definition: Transforms.h:159
A special type of RewriterBase that coordinates the application of a rewrite pattern on the current I...
Definition: PatternMatch.h:600
Operation is a basic unit of execution within MLIR.
Definition: Operation.h:28
SmallVector< Value, 4 > tensorResults
Definition: Transforms.h:127
void populateDecomposeConvolutionPatterns(RewritePatternSet &patterns, const LinalgTransformationFilter &filter=LinalgTransformationFilter(), PatternBenefit benefit=1)
Linalg decompose convolutions patterns.
SmallVector< LinalgOp, 1 > fusedProducers
The generated fused operations (created within the fused loops).
Definition: Transforms.h:193
static void insert(RewritePatternSet &patterns, const LinalgTilingOptions &options, const LinalgTransformationFilter &f)
Definition: Transforms.h:1351
This class represents a frozen set of patterns that can be processed by a pattern applicator...
std::function< LogicalResult(OpBuilder &b, Value buffer)> DeallocBufferCallbackFn
Callback function type used to deallocate the buffers used to hold the promoted subview.
Definition: Transforms.h:233
void populateFoldReshapeOpsByCollapsingPatterns(RewritePatternSet &patterns, const ControlFusionFn &controlFoldingReshapes)
Patterns to fold an expanding tensor.expand_shape operation with its producer generic operation by co...
LogicalResult matchAndRewrite(GenericOp op, PatternRewriter &rewriter) const override
Definition: Transforms.h:823
Options to control the application of enabling transformations.
Definition: Transforms.h:967
Linalg padding pattern.
Definition: Transforms.h:679
Rewrite a tensor::PadOp into a sequence of InitTensorOp, FillOp and InsertSliceOp.
Definition: Transforms.h:1192
virtual void eraseOp(Operation *op)
This method erases an operation that is known to have no uses.
FailureOr< GenericOp > generalizeNamedOp(RewriterBase &rewriter, LinalgOp namedOp)
Create a GenericOp from the given named operation namedOp and replace namedOp.
bool failed(LogicalResult result)
Utility function that returns true if the provided LogicalResult corresponds to a failure value...
Definition: LogicalResult.h:72
void populateFoldReshapeOpsByExpansionPatterns(RewritePatternSet &patterns, const ControlFusionFn &controlFoldingReshapes)
Patterns to fold an expanding (collapsing) tensor_reshape operation with its producer (consumer) gene...
LinalgPaddingOptions & setPaddingDimensions(ArrayRef< int64_t > pd)
Definition: Transforms.h:513
LinalgLoweringPattern(MLIRContext *context, LinalgLoweringType loweringType, LinalgTransformationFilter f=LinalgTransformationFilter(), PatternBenefit benefit=1)
Definition: Transforms.h:1097
SmallVector< int64_t > tileInterchange
Tile interchange used to permute the tile loops.
Definition: Transforms.h:548
Rewrite extract_slice(pad_tensor(x)) into pad_tensor(extract_slice(x)).
Definition: Transforms.h:1294
FailureOr< LinalgLoops > linalgOpToAffineLoops(PatternRewriter &rewriter, LinalgOp linalgOp)
Emit a loop nest of affine.for with the proper body for linalgOp.
Definition: Loops.cpp:358
LinalgTilingOptions & setInterchange(ArrayRef< unsigned > interchange)
Definition: Transforms.h:589
LinalgVectorLoweringOptions & enableMultiReductionLowering(bool val=true)
Definition: Transforms.h:1002
LinalgTilingOptions & setDistributionOptions(LinalgLoopDistributionOptions distributionOptions)
Definition: Transforms.h:607
LinalgVectorLoweringOptions & enableTransferToSCFConversion(bool val=true)
Definition: Transforms.h:1016
LinalgPaddingOptions & setPackPaddings(ArrayRef< bool > pp)
Definition: Transforms.h:520
LinalgPromotionOptions & setAllocationDeallocationFns(AllocBufferCallbackFn const &allocFn, DeallocBufferCallbackFn const &deallocFn)
Definition: Transforms.h:296
SmallVector< int64_t > paddingDimensions
A list of iterator dimensions to pad.
Definition: Transforms.h:512
std::function< SmallVector< Value, 4 >(OpBuilder &, Operation *)> TileSizeComputationFunction
Definition: Transforms.h:433
LogicalResult matchAndRewrite(LinalgOp op, PatternRewriter &rewriter) const override
Definition: Transforms.h:661
std::tuple< SmallVector< Range, 4 >, LoopIndexToRangeIndexMap > makeTiledLoopRanges(RewriterBase &b, Location loc, AffineMap map, ValueRange allShapeSizes, ValueRange allTileSizes)
Definition: Tiling.cpp:44
void populatePadTensorTilingPatterns(RewritePatternSet &patterns, const LinalgTilingOptions &options)
Definition: Tiling.cpp:438
LinalgVectorLoweringOptions & setMaxTransferRank(int64_t val)
Definition: Transforms.h:1022
std::set< unsigned > fusedLoopDims
The dimension of the loops that are fused.
Definition: Transforms.h:191
LinalgEnablingOptions & enableLICM(bool val=true)
Definition: Transforms.h:970
SmallVector< bool > packPaddings
A flag for every operand to mark the PadOp as nofold which enables packing for statically shaped oper...
Definition: Transforms.h:519
This class defines the main interface for locations in MLIR and acts as a non-nullable wrapper around...
Definition: Location.h:48
FailureOr< PromotionInfo > promoteSubviewAsNewBuffer(OpBuilder &b, Location loc, memref::SubViewOp subView, const AllocBufferCallbackFn &allocationFn, DataLayout &layout)
Definition: Promotion.cpp:213
LinalgVectorLoweringOptions & enableAVX2Lowering(bool val=true)
Definition: Transforms.h:1050
Match and rewrite for the pattern: ``` alloc = ...
Definition: Transforms.h:1240
FailureOr< GenericOp > interchangeGenericOp(RewriterBase &rewriter, GenericOp genericOp, ArrayRef< unsigned > interchangeVector)
Interchange the iterator_types and iterator_maps dimensions and adapts the index accesses of op...
Definition: Interchange.cpp:51
void populateElementwiseToLinalgConversionPatterns(RewritePatternSet &patterns)
Populate patterns that convert ElementwiseMappable ops to linalg parallel loops.
void populateSparseTensorRewriting(RewritePatternSet &patterns)
Populate patterns that are only useful in the context of sparse tensors.
void populateFuseTensorPadWithProducerLinalgOpPatterns(RewritePatternSet &patterns)
Pattern to fuse a tensor.pad operation with the producer of its source, if the producer is a linalg o...
std::function< Optional< Value >(OpBuilder &b, memref::SubViewOp subView, ArrayRef< Value > boundingSubViewSize, DataLayout &layout)> AllocBufferCallbackFn
Callback function type used to perform the allocation for the promoted subView.
Definition: Transforms.h:228
SmallVector< scf::ForOp, 8 > Loops
Tile a nest of standard for loops rooted at rootForOp by finding such parametric tile sizes that the ...
Definition: Utils.h:126
filter controls LinalgTransformMarker matching and update when specified.
Definition: Transforms.h:951
RewritePattern is the common base class for all DAG to DAG replacements.
Definition: PatternMatch.h:244
SmallVector< Operation *, 4 > fusedLoops
The fused loop generated.
Definition: Transforms.h:195
FailureOr< SmallVector< Value > > rewriteAsPaddedOp(OpBuilder &b, LinalgOp opToPad, ArrayRef< int64_t > paddingDimensions, ArrayRef< Attribute > paddingValues, ArrayRef< bool > packPaddings, LinalgOp &paddedOp)
Pad the iterator dimensions paddingDimensions of all opToPad operands to a static bounding box...
Definition: Transforms.cpp:255
Match and rewrite for the pattern: ``` alloc = ...
Definition: Transforms.h:1267
StringRef getStringRef() const
Return the name of this operation. This always succeeds.
LogicalResult success(bool isSuccess=true)
Utility function to generate a LogicalResult.
Definition: LogicalResult.h:56
LinalgTransformationFilter & addFilter(const FilterFunction &f)
Definition: Transforms.h:400
This class represents an efficient way to signal success or failure.
Definition: LogicalResult.h:26
static void insert(RewritePatternSet &patterns, const LinalgTilingOptions &options, const LinalgTransformationFilter &f)
Definition: Transforms.h:1359
LogicalResult failure(bool isFailure=true)
Utility function to generate a LogicalResult.
Definition: LogicalResult.h:62
std::function< llvm::Optional< bool >(tensor::ExtractSliceOp)> ControlFn
A function to control pattern application and rewrite logic.
Definition: Transforms.h:1305
LinalgPromotionPattern(MLIRContext *context, LinalgPromotionOptions options, LinalgTransformationFilter f=LinalgTransformationFilter(), PatternBenefit benefit=1)
SFINAE: This constructor can only trigger for concrete ops that have a static getOperationName method...
Definition: Transforms.h:904
LinalgPromotionOptions & setOperandsToPromote(ArrayRef< int64_t > operands)
Definition: Transforms.h:245
This class provides support for representing a failure result, or a valid value of type T...
Definition: LogicalResult.h:77
FailureOr< TiledAndFusedLinalgOps > tileAndFuseLinalgOps(OpBuilder &builder, ArrayRef< LinalgOp > ops, const LinalgDependenceGraph &dependenceGraph, const LinalgTilingOptions &tilingOptions)
Definition: Fusion.cpp:887
llvm::Optional< vector::CombiningKind > getCombinerOpKind(Operation *combinerOp)
Return vector::CombiningKind for the given op.
tensor::PadOp is not canonicalized away yet, so we provide a transformation to linalg.generic.
Definition: Transforms.h:1167
void populateConvolutionVectorizationPatterns(RewritePatternSet &patterns, PatternBenefit benefit=1)
Populate patterns for vectorizing low-D convolution ops.
LinalgVectorLoweringOptions & setAVX2LoweringOptions(x86vector::avx2::LoweringOptions options)
Definition: Transforms.h:1073
LinalgTileAndFusePattern(MLIRContext *context, const LinalgDependenceGraph &dependenceGraph, LinalgTilingOptions tilingOptions, LinalgFusionOptions fusionOptions, LinalgTransformationFilter f=LinalgTransformationFilter(), LinalgTransformationFilter fusedOpMarker=LinalgTransformationFilter(), LinalgTransformationFilter originalOpMarker=LinalgTransformationFilter(), PatternBenefit benefit=1)
Definition: Transforms.h:754
LinalgPaddingOptions & setPaddingValues(ArrayRef< Attribute > pv)
Definition: Transforms.h:507
LogicalResult promoteSubviewsPrecondition(Operation *op, LinalgPromotionOptions options)
Promote memref.subviews feeding linalg-on-buffers operations.
Definition: Promotion.cpp:359
This class represents the benefit of a pattern match in a unitless scheme that ranges from 0 (very li...
Definition: PatternMatch.h:32
void populateInlineConstantOperandsPatterns(RewritePatternSet &patterns)
Patterns that are used to inline constant operands into linalg generic ops.
LinalgFusionOptions & setIndicesToFuse(ArrayRef< int64_t > operands)
Definition: Transforms.h:714
LinalgVectorLoweringOptions & enableVectorTransposeLowering(bool val=true)
Definition: Transforms.h:1044
x86vector::avx2::LoweringOptions avx2LoweringOptions
Configure specialized vector lowerings.
Definition: Transforms.h:1071
SmallVector< int64_t > tileSizes
Tile sizes used to tile the root operation.
Definition: Transforms.h:542
Data structure for holding a dependence graph that operates on LinalgOp and views as SSA values...
LinalgOp op
Operation obtained by tiling the last operation in sequence of ops passed to tileAndFuseLinalgOps.
Definition: Transforms.h:189
Linalg generic interchange pattern.
Definition: Transforms.h:809
std::function< LogicalResult(PatternRewriter &, tensor::PadOp, Value)> OptimizeCopyFn
Definition: Transforms.h:1187
ExtractSliceOfPadTensorSwapPattern(MLIRContext *context, ControlFn controlFn=nullptr, PatternBenefit benefit=1)
Definition: Transforms.h:1307
filter controls LinalgTransformMarker matching and update when specified.
Definition: Transforms.h:926
FailureOr< LinalgOp > promoteSubViews(OpBuilder &b, LinalgOp op, const LinalgPromotionOptions &options)
Promote the subViews into a new buffer allocated at the insertion point b.
Definition: Promotion.cpp:381
LinalgTransformationFilter & addOpFilter()
Definition: Transforms.h:407
LogicalResult matchAndRewrite(LinalgOp op, PatternRewriter &rewriter) const override
Definition: Transforms.h:699
LogicalResult vectorizeCopy(RewriterBase &builder, memref::CopyOp copyOp)
Emit a suitable vector form for a Copy op with fully static shape.
void populateFoldUnitExtentDimsPatterns(RewritePatternSet &patterns)
Patterns to fold unit-extent dimensions in operands/results of linalg ops on tensors.
SmallVector< int64_t > hoistPaddings
A number of loops to hoist the PadOp out for every operand.
Definition: Transforms.h:525
A multi-dimensional affine map Affine map&#39;s are immutable like Type&#39;s, and they are uniqued...
Definition: AffineMap.h:41
static void insert(RewritePatternSet &patterns, const LinalgVectorizationOptions &options, const LinalgTransformationFilter &f)
Definition: Transforms.h:1328
LinalgVectorLoweringOptions & enableTransferPartialRewrite(bool val=true)
Definition: Transforms.h:1009
FailureOr< TiledLinalgOp > tileLinalgOp(RewriterBase &b, LinalgOp op, const LinalgTilingOptions &options)
Definition: Tiling.cpp:269
LinalgTransformationFilter & addOpNameFilter(StringRef opName)
Definition: Transforms.h:412
SmallVector< int64_t > peeledLoops
Peel the specified loops.
Definition: Transforms.h:621
LinalgTilingOptions & setDistributionTypes(ArrayRef< StringRef > types)
Definition: Transforms.h:615
Create a new buffer using the allocationFn provided.
Definition: Transforms.h:319
void populateConstantFoldLinalgOperations(RewritePatternSet &patterns, const ControlFusionFn &controlFn)
Patterns to constant fold Linalg operations.
LinalgEnablingOptions & enableHoistRedundantVectorTransfers(bool val=true)
Definition: Transforms.h:976
LogicalResult matchAndRewrite(LinalgOp op, PatternRewriter &rewriter) const override
Definition: Transforms.h:860
LinalgVectorLoweringOptions & setVectorTransformsOptions(vector::VectorTransformsOptions options)
Definition: Transforms.h:1066
void populateLinalgTilingCanonicalizationPatterns(RewritePatternSet &patterns)
Definition: Tiling.cpp:396
LinalgTransformationFilter & setMatchByDefault()
Definition: Transforms.h:418
void populateLinalgNamedOpConversionPatterns(RewritePatternSet &patterns)
Patterns to convert from one named op to another.
SmallVector< Attribute > paddingValues
A padding value for every operand.
Definition: Transforms.h:506
std::function< LogicalResult(OpBuilder &b, Value src, Value dst)> CopyCallbackFn
Callback function type used to insert copy from original subview to subview of the promoted region fo...
Definition: Transforms.h:240
This class represents an instance of an SSA value in the MLIR system, representing a computable value...
Definition: Value.h:85
FailureOr< LinalgLoops > linalgOpToLoops(PatternRewriter &rewriter, LinalgOp linalgOp)
Emit a loop nest of scf.for with the proper body for linalgOp.
Definition: Loops.cpp:364
void transformIndexOps(RewriterBase &b, LinalgOp op, SmallVectorImpl< Value > &ivs, const LoopIndexToRangeIndexMap &loopIndexToRangeIndex)
All indices returned by IndexOp should be invariant with respect to tiling.
Definition: Tiling.cpp:72
static llvm::ManagedStatic< PassManagerOptions > options
LinalgPromotionOptions & setDynamicBuffers(unsigned dynamic)
Definition: Transforms.h:274
LinalgVectorLoweringOptions & enableShapeCastLowering(bool val=true)
Definition: Transforms.h:1037
OpRewritePattern is a wrapper around RewritePattern that allows for matching and rewriting against an...
Definition: PatternMatch.h:355
void hoistRedundantVectorTransfers(func::FuncOp func)
Hoist vector.transfer_read/vector.transfer_write on buffers pairs out of immediately enclosing scf::F...
Definition: Hoisting.cpp:400
LinalgPromotionOptions & setUseAlloca(bool use)
Definition: Transforms.h:286
std::function< LogicalResult(Operation *)> FilterFunction
Definition: Transforms.h:383
LinalgTilingLoopType
The type of loops to be generated during tiling.
Definition: Utils.h:142
RewritePatternSet & add(ConstructorArg &&arg, ConstructorArgs &&... args)
Add an instance of each of the pattern types &#39;Ts&#39; to the pattern list with the given arguments...
LinalgPromotionOptions & setAlignment(unsigned align)
Definition: Transforms.h:280
static const StringLiteral kLinalgTransformMarker
Definition: Transforms.h:372
LinalgVectorLoweringOptions & enableTransferLowering(bool val=true)
Definition: Transforms.h:1030
VectorTransferToSCFOptions vectorTransferToSCFOptions
Configure the post staged-patterns late vector.transfer to scf conversion.
Definition: Transforms.h:1057
Fuse a sequence of linalg operations (ops) using tile-and-fuse.
Definition: Transforms.h:186
LinalgTilingOptions & setLoopType(LinalgTilingLoopType lt)
Definition: Transforms.h:597
LinalgTilingAndFusionOptions & setDistributionOptions(LinalgLoopDistributionOptions distributionOptions)
Definition: Transforms.h:553
Linalg generalization pattern.
Definition: Transforms.h:841
LinalgVectorLoweringOptions & setVectorTransferToSCFOptions(VectorTransferToSCFOptions options)
Definition: Transforms.h:1059
Linalg tile and fuse tensor ops pattern.
Definition: Transforms.h:773
void populateBubbleUpExtractSliceOpPatterns(RewritePatternSet &patterns)
Patterns that are used to bubble up extract slice op above linalg op.
Perform standalone tiling of a single LinalgOp by tileSizes.
Definition: Transforms.h:124
LinalgPromotionOptions & setUseFullTileBuffers(ArrayRef< bool > useFullTiles)
Definition: Transforms.h:257
LinalgTilingOptions & setPeeledLoops(ArrayRef< int64_t > loops)
Definition: Transforms.h:623
GeneralizePadOpPattern(MLIRContext *context, OptimizeCopyFn optimizeCopyFn=nullptr, PatternBenefit benefit=1)
Definition: Transforms.h:1193
Structure to control the behavior of vector transform patterns.
MLIRContext is the top-level object for a collection of MLIR operations.
Definition: MLIRContext.h:55
void populateLinalgNamedOpsGeneralizationPatterns(RewritePatternSet &patterns, const LinalgTransformationFilter &filter=LinalgTransformationFilter())
Linalg generalization patterns.
Vector lowering options control how ops are lowered down to 1-D and scf.for form. ...
Definition: Transforms.h:991
std::function< bool(const OpResult &producer, OpOperand &consumer)> ControlFusionFn
Function type which is used to control when to stop fusion.
Definition: Transforms.h:65
LogicalResult vectorize(RewriterBase &builder, LinalgOp linalgOp)
Emit a suitable vector form for a Linalg op with fully static shape.
void populateSplitReductionPattern(RewritePatternSet &patterns, const ControlSplitReductionFn &controlSplitReductionFn, const LinalgTransformationFilter &f=LinalgTransformationFilter())
Patterns to apply splitReduction below.
void hoistRedundantVectorTransfersOnTensor(func::FuncOp func)
Same behavior as hoistRedundantVectorTransfers but works on tensors instead of buffers.
Definition: Hoisting.cpp:348
FailureOr< LinalgOp > splitReduction(PatternRewriter &b, LinalgOp op, const ControlSplitReductionFn &controlSplitReductionFn, const LinalgTransformationFilter &f)
Apply transformation to split the single linalg op reduction into a parallel and reduction dimension...
void populateElementwiseOpsFusionPatterns(RewritePatternSet &patterns, const ControlFusionFn &controlElementwiseOpFusion)
Patterns for fusing linalg operation on tensors.
Linalg vectorization patterns.
Definition: Transforms.h:922
LogicalResult applyStagedPatterns(Operation *op, ArrayRef< FrozenRewritePatternSet > stage1Patterns, const FrozenRewritePatternSet &stage2Patterns, function_ref< LogicalResult(Operation *)> stage3Lambda=nullptr)
Helper function to allow applying rewrite patterns, interleaved with more global transformations, in a staged fashion:
Definition: Transforms.cpp:743
Linalg promotion patterns.
Definition: Transforms.h:876
LinalgPromotionOptions & setUseFullTileBuffersByDefault(bool use)
Definition: Transforms.h:268
void peelTiledLinalgOp(RewriterBase &rewriter, TiledLinalgOp &res, ArrayRef< int64_t > peeledLoops, LinalgTilingLoopType loopType)
Peel the loops of a TiledLinalgOp.
Definition: Transforms.cpp:327
OperationName getName()
The name of an operation is the key identifier for it.
Definition: Operation.h:57
void populatePadOpVectorizationPatterns(RewritePatternSet &patterns, PatternBenefit baseBenefit=1)
Populates patterns with patterns that vectorize linalg.pad_tensor.
std::function< std::pair< int64_t, unsigned >(LinalgOp op)> ControlSplitReductionFn
Function signature to control reduction splitting.
Definition: Transforms.h:1374
SmallVector< Operation *, 8 > loops
Definition: Transforms.h:126
LinalgTilingOptions & setTileSizeComputationFunction(TileSizeComputationFunction fun)
Definition: Transforms.h:566
FailureOr< LinalgLoops > linalgOpToParallelLoops(PatternRewriter &rewriter, LinalgOp linalgOp)
Emit a loop nest of scf.parallel with the proper body for linalgOp.
Definition: Loops.cpp:371
RewritePatternSet getLinalgTilingCanonicalizationPatterns(MLIRContext *ctx)
Canonicalization patterns relevant to apply after tiling patterns.
Definition: Tiling.cpp:390
Options that allow distribution of loops generated in Linalg transforms to processors while generatin...
Definition: Utils.h:306
This class helps build Operations.
Definition: Builders.h:177
This class provides an abstraction over the different types of ranges over Values.
LinalgEnablingOptions & enableHoistRedundantVectorTransfersOnTensor(bool val=true)
Definition: Transforms.h:983
LinalgTilingAndFusionOptions & setTileSizes(ArrayRef< int64_t > ts)
Definition: Transforms.h:543
MLIRContext * getContext() const
static void insert(RewritePatternSet &patterns, const LinalgVectorizationOptions &options, const LinalgTransformationFilter &f)
Definition: Transforms.h:1336
When lowering an N-d vector transfer op to an (N-1)-d vector transfer op, a temporary buffer is creat...
Definition: VectorToSCF.h:49
This class coordinates the application of a rewrite on a set of IR, providing a way for clients to tr...
Definition: PatternMatch.h:398
LinalgTilingOptions & setTileSizes(const SmallVector< Value, 4 > &ts)
Set the tileSizeComputationFunction to return the values ts.
Definition: Transforms.h:573
OpInterfaceRewritePattern is a wrapper around RewritePattern that allows for matching and rewriting a...
Definition: PatternMatch.h:370
LinalgPromotionPattern(StringRef opName, MLIRContext *context, LinalgPromotionOptions options, LinalgTransformationFilter f=LinalgTransformationFilter(), PatternBenefit benefit=1)
This constructor is available to anyone.
Definition: Transforms.h:911
The main mechanism for performing data layout queries.
vector::VectorTransformsOptions vectorTransformOptions
Configure late vector transformations.
Definition: Transforms.h:1064
LinalgPaddingOptions & setTransposePaddings(ArrayRef< SmallVector< int64_t >> tp)
Definition: Transforms.h:534