MLIR  21.0.0git
VectorToSPIRV.cpp
Go to the documentation of this file.
1 //===- VectorToSPIRV.cpp - Vector to SPIR-V Patterns ----------------------===//
2 //
3 // Part of the LLVM Project, under the Apache License v2.0 with LLVM Exceptions.
4 // See https://llvm.org/LICENSE.txt for license information.
5 // SPDX-License-Identifier: Apache-2.0 WITH LLVM-exception
6 //
7 //===----------------------------------------------------------------------===//
8 //
9 // This file implements patterns to convert Vector dialect to SPIRV dialect.
10 //
11 //===----------------------------------------------------------------------===//
12 
14 
21 #include "mlir/IR/Attributes.h"
23 #include "mlir/IR/BuiltinTypes.h"
24 #include "mlir/IR/Location.h"
25 #include "mlir/IR/Matchers.h"
26 #include "mlir/IR/PatternMatch.h"
27 #include "mlir/IR/TypeUtilities.h"
29 #include "llvm/ADT/ArrayRef.h"
30 #include "llvm/ADT/STLExtras.h"
31 #include "llvm/ADT/SmallVector.h"
32 #include "llvm/ADT/SmallVectorExtras.h"
33 #include "llvm/Support/FormatVariadic.h"
34 #include <cassert>
35 #include <cstdint>
36 #include <numeric>
37 
38 using namespace mlir;
39 
40 /// Returns the integer value from the first valid input element, assuming Value
41 /// inputs are defined by a constant index ops and Attribute inputs are integer
42 /// attributes.
43 static uint64_t getFirstIntValue(ArrayAttr attr) {
44  return (*attr.getAsValueRange<IntegerAttr>().begin()).getZExtValue();
45 }
46 
47 /// Returns the number of bits for the given scalar/vector type.
48 static int getNumBits(Type type) {
49  // TODO: This does not take into account any memory layout or widening
50  // constraints. E.g., a vector<3xi57> may report to occupy 3x57=171 bit, even
51  // though in practice it will likely be stored as in a 4xi64 vector register.
52  if (auto vectorType = dyn_cast<VectorType>(type))
53  return vectorType.getNumElements() * vectorType.getElementTypeBitWidth();
54  return type.getIntOrFloatBitWidth();
55 }
56 
57 namespace {
58 
59 struct VectorShapeCast final : public OpConversionPattern<vector::ShapeCastOp> {
61 
62  LogicalResult
63  matchAndRewrite(vector::ShapeCastOp shapeCastOp, OpAdaptor adaptor,
64  ConversionPatternRewriter &rewriter) const override {
65  Type dstType = getTypeConverter()->convertType(shapeCastOp.getType());
66  if (!dstType)
67  return failure();
68 
69  // If dstType is same as the source type or the vector size is 1, it can be
70  // directly replaced by the source.
71  if (dstType == adaptor.getSource().getType() ||
72  shapeCastOp.getResultVectorType().getNumElements() == 1) {
73  rewriter.replaceOp(shapeCastOp, adaptor.getSource());
74  return success();
75  }
76 
77  // Lowering for size-n vectors when n > 1 hasn't been implemented.
78  return failure();
79  }
80 };
81 
82 struct VectorBitcastConvert final
83  : public OpConversionPattern<vector::BitCastOp> {
85 
86  LogicalResult
87  matchAndRewrite(vector::BitCastOp bitcastOp, OpAdaptor adaptor,
88  ConversionPatternRewriter &rewriter) const override {
89  Type dstType = getTypeConverter()->convertType(bitcastOp.getType());
90  if (!dstType)
91  return failure();
92 
93  if (dstType == adaptor.getSource().getType()) {
94  rewriter.replaceOp(bitcastOp, adaptor.getSource());
95  return success();
96  }
97 
98  // Check that the source and destination type have the same bitwidth.
99  // Depending on the target environment, we may need to emulate certain
100  // types, which can cause issue with bitcast.
101  Type srcType = adaptor.getSource().getType();
102  if (getNumBits(dstType) != getNumBits(srcType)) {
103  return rewriter.notifyMatchFailure(
104  bitcastOp,
105  llvm::formatv("different source ({0}) and target ({1}) bitwidth",
106  srcType, dstType));
107  }
108 
109  rewriter.replaceOpWithNewOp<spirv::BitcastOp>(bitcastOp, dstType,
110  adaptor.getSource());
111  return success();
112  }
113 };
114 
115 struct VectorBroadcastConvert final
116  : public OpConversionPattern<vector::BroadcastOp> {
118 
119  LogicalResult
120  matchAndRewrite(vector::BroadcastOp castOp, OpAdaptor adaptor,
121  ConversionPatternRewriter &rewriter) const override {
122  Type resultType =
123  getTypeConverter()->convertType(castOp.getResultVectorType());
124  if (!resultType)
125  return failure();
126 
127  if (isa<spirv::ScalarType>(resultType)) {
128  rewriter.replaceOp(castOp, adaptor.getSource());
129  return success();
130  }
131 
132  SmallVector<Value, 4> source(castOp.getResultVectorType().getNumElements(),
133  adaptor.getSource());
134  rewriter.replaceOpWithNewOp<spirv::CompositeConstructOp>(castOp, resultType,
135  source);
136  return success();
137  }
138 };
139 
140 // SPIR-V does not have a concept of a poison index for certain instructions,
141 // which creates a UB hazard when lowering from otherwise equivalent Vector
142 // dialect instructions, because this index will be considered out-of-bounds.
143 // To avoid this, this function implements a dynamic sanitization that returns
144 // some arbitrary safe index. For power-of-two vector sizes, this uses a bitmask
145 // (presumably more efficient), and otherwise index 0 (always in-bounds).
146 static Value sanitizeDynamicIndex(ConversionPatternRewriter &rewriter,
147  Location loc, Value dynamicIndex,
148  int64_t kPoisonIndex, unsigned vectorSize) {
149  if (llvm::isPowerOf2_32(vectorSize)) {
150  Value inBoundsMask = rewriter.create<spirv::ConstantOp>(
151  loc, dynamicIndex.getType(),
152  rewriter.getIntegerAttr(dynamicIndex.getType(), vectorSize - 1));
153  return rewriter.create<spirv::BitwiseAndOp>(loc, dynamicIndex,
154  inBoundsMask);
155  }
156  Value poisonIndex = rewriter.create<spirv::ConstantOp>(
157  loc, dynamicIndex.getType(),
158  rewriter.getIntegerAttr(dynamicIndex.getType(), kPoisonIndex));
159  Value cmpResult =
160  rewriter.create<spirv::IEqualOp>(loc, dynamicIndex, poisonIndex);
161  return rewriter.create<spirv::SelectOp>(
162  loc, cmpResult,
163  spirv::ConstantOp::getZero(dynamicIndex.getType(), loc, rewriter),
164  dynamicIndex);
165 }
166 
167 struct VectorExtractOpConvert final
168  : public OpConversionPattern<vector::ExtractOp> {
170 
171  LogicalResult
172  matchAndRewrite(vector::ExtractOp extractOp, OpAdaptor adaptor,
173  ConversionPatternRewriter &rewriter) const override {
174  Type dstType = getTypeConverter()->convertType(extractOp.getType());
175  if (!dstType)
176  return failure();
177 
178  if (isa<spirv::ScalarType>(adaptor.getVector().getType())) {
179  rewriter.replaceOp(extractOp, adaptor.getVector());
180  return success();
181  }
182 
183  if (std::optional<int64_t> id =
184  getConstantIntValue(extractOp.getMixedPosition()[0])) {
185  if (id == vector::ExtractOp::kPoisonIndex)
186  return rewriter.notifyMatchFailure(
187  extractOp,
188  "Static use of poison index handled elsewhere (folded to poison)");
189  rewriter.replaceOpWithNewOp<spirv::CompositeExtractOp>(
190  extractOp, dstType, adaptor.getVector(),
191  rewriter.getI32ArrayAttr(id.value()));
192  } else {
193  Value sanitizedIndex = sanitizeDynamicIndex(
194  rewriter, extractOp.getLoc(), adaptor.getDynamicPosition()[0],
195  vector::ExtractOp::kPoisonIndex,
196  extractOp.getSourceVectorType().getNumElements());
197  rewriter.replaceOpWithNewOp<spirv::VectorExtractDynamicOp>(
198  extractOp, dstType, adaptor.getVector(), sanitizedIndex);
199  }
200  return success();
201  }
202 };
203 
204 struct VectorExtractStridedSliceOpConvert final
205  : public OpConversionPattern<vector::ExtractStridedSliceOp> {
207 
208  LogicalResult
209  matchAndRewrite(vector::ExtractStridedSliceOp extractOp, OpAdaptor adaptor,
210  ConversionPatternRewriter &rewriter) const override {
211  Type dstType = getTypeConverter()->convertType(extractOp.getType());
212  if (!dstType)
213  return failure();
214 
215  uint64_t offset = getFirstIntValue(extractOp.getOffsets());
216  uint64_t size = getFirstIntValue(extractOp.getSizes());
217  uint64_t stride = getFirstIntValue(extractOp.getStrides());
218  if (stride != 1)
219  return failure();
220 
221  Value srcVector = adaptor.getOperands().front();
222 
223  // Extract vector<1xT> case.
224  if (isa<spirv::ScalarType>(dstType)) {
225  rewriter.replaceOpWithNewOp<spirv::CompositeExtractOp>(extractOp,
226  srcVector, offset);
227  return success();
228  }
229 
230  SmallVector<int32_t, 2> indices(size);
231  std::iota(indices.begin(), indices.end(), offset);
232 
233  rewriter.replaceOpWithNewOp<spirv::VectorShuffleOp>(
234  extractOp, dstType, srcVector, srcVector,
235  rewriter.getI32ArrayAttr(indices));
236 
237  return success();
238  }
239 };
240 
241 template <class SPIRVFMAOp>
242 struct VectorFmaOpConvert final : public OpConversionPattern<vector::FMAOp> {
244 
245  LogicalResult
246  matchAndRewrite(vector::FMAOp fmaOp, OpAdaptor adaptor,
247  ConversionPatternRewriter &rewriter) const override {
248  Type dstType = getTypeConverter()->convertType(fmaOp.getType());
249  if (!dstType)
250  return failure();
251  rewriter.replaceOpWithNewOp<SPIRVFMAOp>(fmaOp, dstType, adaptor.getLhs(),
252  adaptor.getRhs(), adaptor.getAcc());
253  return success();
254  }
255 };
256 
257 struct VectorFromElementsOpConvert final
258  : public OpConversionPattern<vector::FromElementsOp> {
260 
261  LogicalResult
262  matchAndRewrite(vector::FromElementsOp op, OpAdaptor adaptor,
263  ConversionPatternRewriter &rewriter) const override {
264  Type resultType = getTypeConverter()->convertType(op.getType());
265  if (!resultType)
266  return failure();
267  OperandRange elements = op.getElements();
268  if (isa<spirv::ScalarType>(resultType)) {
269  // In the case with a single scalar operand / single-element result,
270  // pass through the scalar.
271  rewriter.replaceOp(op, elements[0]);
272  return success();
273  }
274  // SPIRVTypeConverter rejects vectors with rank > 1, so multi-dimensional
275  // vector.from_elements cases should not need to be handled, only 1d.
276  assert(cast<VectorType>(resultType).getRank() == 1);
277  rewriter.replaceOpWithNewOp<spirv::CompositeConstructOp>(op, resultType,
278  elements);
279  return success();
280  }
281 };
282 
283 struct VectorInsertOpConvert final
284  : public OpConversionPattern<vector::InsertOp> {
286 
287  LogicalResult
288  matchAndRewrite(vector::InsertOp insertOp, OpAdaptor adaptor,
289  ConversionPatternRewriter &rewriter) const override {
290  if (isa<VectorType>(insertOp.getSourceType()))
291  return rewriter.notifyMatchFailure(insertOp, "unsupported vector source");
292  if (!getTypeConverter()->convertType(insertOp.getDestVectorType()))
293  return rewriter.notifyMatchFailure(insertOp,
294  "unsupported dest vector type");
295 
296  // Special case for inserting scalar values into size-1 vectors.
297  if (insertOp.getSourceType().isIntOrFloat() &&
298  insertOp.getDestVectorType().getNumElements() == 1) {
299  rewriter.replaceOp(insertOp, adaptor.getSource());
300  return success();
301  }
302 
303  if (std::optional<int64_t> id =
304  getConstantIntValue(insertOp.getMixedPosition()[0])) {
305  if (id == vector::InsertOp::kPoisonIndex)
306  return rewriter.notifyMatchFailure(
307  insertOp,
308  "Static use of poison index handled elsewhere (folded to poison)");
309  rewriter.replaceOpWithNewOp<spirv::CompositeInsertOp>(
310  insertOp, adaptor.getSource(), adaptor.getDest(), id.value());
311  } else {
312  Value sanitizedIndex = sanitizeDynamicIndex(
313  rewriter, insertOp.getLoc(), adaptor.getDynamicPosition()[0],
314  vector::InsertOp::kPoisonIndex,
315  insertOp.getDestVectorType().getNumElements());
316  rewriter.replaceOpWithNewOp<spirv::VectorInsertDynamicOp>(
317  insertOp, insertOp.getDest(), adaptor.getSource(), sanitizedIndex);
318  }
319  return success();
320  }
321 };
322 
323 struct VectorExtractElementOpConvert final
324  : public OpConversionPattern<vector::ExtractElementOp> {
326 
327  LogicalResult
328  matchAndRewrite(vector::ExtractElementOp extractOp, OpAdaptor adaptor,
329  ConversionPatternRewriter &rewriter) const override {
330  Type resultType = getTypeConverter()->convertType(extractOp.getType());
331  if (!resultType)
332  return failure();
333 
334  if (isa<spirv::ScalarType>(adaptor.getVector().getType())) {
335  rewriter.replaceOp(extractOp, adaptor.getVector());
336  return success();
337  }
338 
339  APInt cstPos;
340  if (matchPattern(adaptor.getPosition(), m_ConstantInt(&cstPos)))
341  rewriter.replaceOpWithNewOp<spirv::CompositeExtractOp>(
342  extractOp, resultType, adaptor.getVector(),
343  rewriter.getI32ArrayAttr({static_cast<int>(cstPos.getSExtValue())}));
344  else
345  rewriter.replaceOpWithNewOp<spirv::VectorExtractDynamicOp>(
346  extractOp, resultType, adaptor.getVector(), adaptor.getPosition());
347  return success();
348  }
349 };
350 
351 struct VectorInsertElementOpConvert final
352  : public OpConversionPattern<vector::InsertElementOp> {
354 
355  LogicalResult
356  matchAndRewrite(vector::InsertElementOp insertOp, OpAdaptor adaptor,
357  ConversionPatternRewriter &rewriter) const override {
358  Type vectorType = getTypeConverter()->convertType(insertOp.getType());
359  if (!vectorType)
360  return failure();
361 
362  if (isa<spirv::ScalarType>(vectorType)) {
363  rewriter.replaceOp(insertOp, adaptor.getSource());
364  return success();
365  }
366 
367  APInt cstPos;
368  if (matchPattern(adaptor.getPosition(), m_ConstantInt(&cstPos)))
369  rewriter.replaceOpWithNewOp<spirv::CompositeInsertOp>(
370  insertOp, adaptor.getSource(), adaptor.getDest(),
371  cstPos.getSExtValue());
372  else
373  rewriter.replaceOpWithNewOp<spirv::VectorInsertDynamicOp>(
374  insertOp, vectorType, insertOp.getDest(), adaptor.getSource(),
375  adaptor.getPosition());
376  return success();
377  }
378 };
379 
380 struct VectorInsertStridedSliceOpConvert final
381  : public OpConversionPattern<vector::InsertStridedSliceOp> {
383 
384  LogicalResult
385  matchAndRewrite(vector::InsertStridedSliceOp insertOp, OpAdaptor adaptor,
386  ConversionPatternRewriter &rewriter) const override {
387  Value srcVector = adaptor.getOperands().front();
388  Value dstVector = adaptor.getOperands().back();
389 
390  uint64_t stride = getFirstIntValue(insertOp.getStrides());
391  if (stride != 1)
392  return failure();
393  uint64_t offset = getFirstIntValue(insertOp.getOffsets());
394 
395  if (isa<spirv::ScalarType>(srcVector.getType())) {
396  assert(!isa<spirv::ScalarType>(dstVector.getType()));
397  rewriter.replaceOpWithNewOp<spirv::CompositeInsertOp>(
398  insertOp, dstVector.getType(), srcVector, dstVector,
399  rewriter.getI32ArrayAttr(offset));
400  return success();
401  }
402 
403  uint64_t totalSize = cast<VectorType>(dstVector.getType()).getNumElements();
404  uint64_t insertSize =
405  cast<VectorType>(srcVector.getType()).getNumElements();
406 
407  SmallVector<int32_t, 2> indices(totalSize);
408  std::iota(indices.begin(), indices.end(), 0);
409  std::iota(indices.begin() + offset, indices.begin() + offset + insertSize,
410  totalSize);
411 
412  rewriter.replaceOpWithNewOp<spirv::VectorShuffleOp>(
413  insertOp, dstVector.getType(), dstVector, srcVector,
414  rewriter.getI32ArrayAttr(indices));
415 
416  return success();
417  }
418 };
419 
420 static SmallVector<Value> extractAllElements(
421  vector::ReductionOp reduceOp, vector::ReductionOp::Adaptor adaptor,
422  VectorType srcVectorType, ConversionPatternRewriter &rewriter) {
423  int numElements = static_cast<int>(srcVectorType.getDimSize(0));
424  SmallVector<Value> values;
425  values.reserve(numElements + (adaptor.getAcc() ? 1 : 0));
426  Location loc = reduceOp.getLoc();
427 
428  for (int i = 0; i < numElements; ++i) {
429  values.push_back(rewriter.create<spirv::CompositeExtractOp>(
430  loc, srcVectorType.getElementType(), adaptor.getVector(),
431  rewriter.getI32ArrayAttr({i})));
432  }
433  if (Value acc = adaptor.getAcc())
434  values.push_back(acc);
435 
436  return values;
437 }
438 
439 struct ReductionRewriteInfo {
440  Type resultType;
441  SmallVector<Value> extractedElements;
442 };
443 
444 FailureOr<ReductionRewriteInfo> static getReductionInfo(
445  vector::ReductionOp op, vector::ReductionOp::Adaptor adaptor,
446  ConversionPatternRewriter &rewriter, const TypeConverter &typeConverter) {
447  Type resultType = typeConverter.convertType(op.getType());
448  if (!resultType)
449  return failure();
450 
451  auto srcVectorType = dyn_cast<VectorType>(adaptor.getVector().getType());
452  if (!srcVectorType || srcVectorType.getRank() != 1)
453  return rewriter.notifyMatchFailure(op, "not a 1-D vector source");
454 
455  SmallVector<Value> extractedElements =
456  extractAllElements(op, adaptor, srcVectorType, rewriter);
457 
458  return ReductionRewriteInfo{resultType, std::move(extractedElements)};
459 }
460 
461 template <typename SPIRVUMaxOp, typename SPIRVUMinOp, typename SPIRVSMaxOp,
462  typename SPIRVSMinOp>
463 struct VectorReductionPattern final : OpConversionPattern<vector::ReductionOp> {
465 
466  LogicalResult
467  matchAndRewrite(vector::ReductionOp reduceOp, OpAdaptor adaptor,
468  ConversionPatternRewriter &rewriter) const override {
469  auto reductionInfo =
470  getReductionInfo(reduceOp, adaptor, rewriter, *getTypeConverter());
471  if (failed(reductionInfo))
472  return failure();
473 
474  auto [resultType, extractedElements] = *reductionInfo;
475  Location loc = reduceOp->getLoc();
476  Value result = extractedElements.front();
477  for (Value next : llvm::drop_begin(extractedElements)) {
478  switch (reduceOp.getKind()) {
479 
480 #define INT_AND_FLOAT_CASE(kind, iop, fop) \
481  case vector::CombiningKind::kind: \
482  if (llvm::isa<IntegerType>(resultType)) { \
483  result = rewriter.create<spirv::iop>(loc, resultType, result, next); \
484  } else { \
485  assert(llvm::isa<FloatType>(resultType)); \
486  result = rewriter.create<spirv::fop>(loc, resultType, result, next); \
487  } \
488  break
489 
490 #define INT_OR_FLOAT_CASE(kind, fop) \
491  case vector::CombiningKind::kind: \
492  result = rewriter.create<fop>(loc, resultType, result, next); \
493  break
494 
495  INT_AND_FLOAT_CASE(ADD, IAddOp, FAddOp);
496  INT_AND_FLOAT_CASE(MUL, IMulOp, FMulOp);
497  INT_OR_FLOAT_CASE(MINUI, SPIRVUMinOp);
498  INT_OR_FLOAT_CASE(MINSI, SPIRVSMinOp);
499  INT_OR_FLOAT_CASE(MAXUI, SPIRVUMaxOp);
500  INT_OR_FLOAT_CASE(MAXSI, SPIRVSMaxOp);
501 
502  case vector::CombiningKind::AND:
503  case vector::CombiningKind::OR:
504  case vector::CombiningKind::XOR:
505  return rewriter.notifyMatchFailure(reduceOp, "unimplemented");
506  default:
507  return rewriter.notifyMatchFailure(reduceOp, "not handled here");
508  }
509 #undef INT_AND_FLOAT_CASE
510 #undef INT_OR_FLOAT_CASE
511  }
512 
513  rewriter.replaceOp(reduceOp, result);
514  return success();
515  }
516 };
517 
518 template <typename SPIRVFMaxOp, typename SPIRVFMinOp>
519 struct VectorReductionFloatMinMax final
520  : OpConversionPattern<vector::ReductionOp> {
522 
523  LogicalResult
524  matchAndRewrite(vector::ReductionOp reduceOp, OpAdaptor adaptor,
525  ConversionPatternRewriter &rewriter) const override {
526  auto reductionInfo =
527  getReductionInfo(reduceOp, adaptor, rewriter, *getTypeConverter());
528  if (failed(reductionInfo))
529  return failure();
530 
531  auto [resultType, extractedElements] = *reductionInfo;
532  Location loc = reduceOp->getLoc();
533  Value result = extractedElements.front();
534  for (Value next : llvm::drop_begin(extractedElements)) {
535  switch (reduceOp.getKind()) {
536 
537 #define INT_OR_FLOAT_CASE(kind, fop) \
538  case vector::CombiningKind::kind: \
539  result = rewriter.create<fop>(loc, resultType, result, next); \
540  break
541 
542  INT_OR_FLOAT_CASE(MAXIMUMF, SPIRVFMaxOp);
543  INT_OR_FLOAT_CASE(MINIMUMF, SPIRVFMinOp);
544  INT_OR_FLOAT_CASE(MAXNUMF, SPIRVFMaxOp);
545  INT_OR_FLOAT_CASE(MINNUMF, SPIRVFMinOp);
546 
547  default:
548  return rewriter.notifyMatchFailure(reduceOp, "not handled here");
549  }
550 #undef INT_OR_FLOAT_CASE
551  }
552 
553  rewriter.replaceOp(reduceOp, result);
554  return success();
555  }
556 };
557 
558 class VectorSplatPattern final : public OpConversionPattern<vector::SplatOp> {
559 public:
561 
562  LogicalResult
563  matchAndRewrite(vector::SplatOp op, OpAdaptor adaptor,
564  ConversionPatternRewriter &rewriter) const override {
565  Type dstType = getTypeConverter()->convertType(op.getType());
566  if (!dstType)
567  return failure();
568  if (isa<spirv::ScalarType>(dstType)) {
569  rewriter.replaceOp(op, adaptor.getInput());
570  } else {
571  auto dstVecType = cast<VectorType>(dstType);
572  SmallVector<Value, 4> source(dstVecType.getNumElements(),
573  adaptor.getInput());
574  rewriter.replaceOpWithNewOp<spirv::CompositeConstructOp>(op, dstType,
575  source);
576  }
577  return success();
578  }
579 };
580 
581 struct VectorShuffleOpConvert final
582  : public OpConversionPattern<vector::ShuffleOp> {
584 
585  LogicalResult
586  matchAndRewrite(vector::ShuffleOp shuffleOp, OpAdaptor adaptor,
587  ConversionPatternRewriter &rewriter) const override {
588  VectorType oldResultType = shuffleOp.getResultVectorType();
589  Type newResultType = getTypeConverter()->convertType(oldResultType);
590  if (!newResultType)
591  return rewriter.notifyMatchFailure(shuffleOp,
592  "unsupported result vector type");
593 
594  auto mask = llvm::to_vector_of<int32_t>(shuffleOp.getMask());
595 
596  VectorType oldV1Type = shuffleOp.getV1VectorType();
597  VectorType oldV2Type = shuffleOp.getV2VectorType();
598 
599  // When both operands and the result are SPIR-V vectors, emit a SPIR-V
600  // shuffle.
601  if (oldV1Type.getNumElements() > 1 && oldV2Type.getNumElements() > 1 &&
602  oldResultType.getNumElements() > 1) {
603  rewriter.replaceOpWithNewOp<spirv::VectorShuffleOp>(
604  shuffleOp, newResultType, adaptor.getV1(), adaptor.getV2(),
605  rewriter.getI32ArrayAttr(mask));
606  return success();
607  }
608 
609  // When at least one of the operands or the result becomes a scalar after
610  // type conversion for SPIR-V, extract all the required elements and
611  // construct the result vector.
612  auto getElementAtIdx = [&rewriter, loc = shuffleOp.getLoc()](
613  Value scalarOrVec, int32_t idx) -> Value {
614  if (auto vecTy = dyn_cast<VectorType>(scalarOrVec.getType()))
615  return rewriter.create<spirv::CompositeExtractOp>(loc, scalarOrVec,
616  idx);
617 
618  assert(idx == 0 && "Invalid scalar element index");
619  return scalarOrVec;
620  };
621 
622  int32_t numV1Elems = oldV1Type.getNumElements();
623  SmallVector<Value> newOperands(mask.size());
624  for (auto [shuffleIdx, newOperand] : llvm::zip_equal(mask, newOperands)) {
625  Value vec = adaptor.getV1();
626  int32_t elementIdx = shuffleIdx;
627  if (elementIdx >= numV1Elems) {
628  vec = adaptor.getV2();
629  elementIdx -= numV1Elems;
630  }
631 
632  newOperand = getElementAtIdx(vec, elementIdx);
633  }
634 
635  // Handle the scalar result corner case.
636  if (newOperands.size() == 1) {
637  rewriter.replaceOp(shuffleOp, newOperands.front());
638  return success();
639  }
640 
641  rewriter.replaceOpWithNewOp<spirv::CompositeConstructOp>(
642  shuffleOp, newResultType, newOperands);
643  return success();
644  }
645 };
646 
647 struct VectorInterleaveOpConvert final
648  : public OpConversionPattern<vector::InterleaveOp> {
650 
651  LogicalResult
652  matchAndRewrite(vector::InterleaveOp interleaveOp, OpAdaptor adaptor,
653  ConversionPatternRewriter &rewriter) const override {
654  // Check the result vector type.
655  VectorType oldResultType = interleaveOp.getResultVectorType();
656  Type newResultType = getTypeConverter()->convertType(oldResultType);
657  if (!newResultType)
658  return rewriter.notifyMatchFailure(interleaveOp,
659  "unsupported result vector type");
660 
661  // Interleave the indices.
662  VectorType sourceType = interleaveOp.getSourceVectorType();
663  int n = sourceType.getNumElements();
664 
665  // Input vectors of size 1 are converted to scalars by the type converter.
666  // We cannot use `spirv::VectorShuffleOp` directly in this case, and need to
667  // use `spirv::CompositeConstructOp`.
668  if (n == 1) {
669  Value newOperands[] = {adaptor.getLhs(), adaptor.getRhs()};
670  rewriter.replaceOpWithNewOp<spirv::CompositeConstructOp>(
671  interleaveOp, newResultType, newOperands);
672  return success();
673  }
674 
675  auto seq = llvm::seq<int64_t>(2 * n);
676  auto indices = llvm::map_to_vector(
677  seq, [n](int i) { return (i % 2 ? n : 0) + i / 2; });
678 
679  // Emit a SPIR-V shuffle.
680  rewriter.replaceOpWithNewOp<spirv::VectorShuffleOp>(
681  interleaveOp, newResultType, adaptor.getLhs(), adaptor.getRhs(),
682  rewriter.getI32ArrayAttr(indices));
683 
684  return success();
685  }
686 };
687 
688 struct VectorDeinterleaveOpConvert final
689  : public OpConversionPattern<vector::DeinterleaveOp> {
691 
692  LogicalResult
693  matchAndRewrite(vector::DeinterleaveOp deinterleaveOp, OpAdaptor adaptor,
694  ConversionPatternRewriter &rewriter) const override {
695 
696  // Check the result vector type.
697  VectorType oldResultType = deinterleaveOp.getResultVectorType();
698  Type newResultType = getTypeConverter()->convertType(oldResultType);
699  if (!newResultType)
700  return rewriter.notifyMatchFailure(deinterleaveOp,
701  "unsupported result vector type");
702 
703  Location loc = deinterleaveOp->getLoc();
704 
705  // Deinterleave the indices.
706  Value sourceVector = adaptor.getSource();
707  VectorType sourceType = deinterleaveOp.getSourceVectorType();
708  int n = sourceType.getNumElements();
709 
710  // Output vectors of size 1 are converted to scalars by the type converter.
711  // We cannot use `spirv::VectorShuffleOp` directly in this case, and need to
712  // use `spirv::CompositeExtractOp`.
713  if (n == 2) {
714  auto elem0 = rewriter.create<spirv::CompositeExtractOp>(
715  loc, newResultType, sourceVector, rewriter.getI32ArrayAttr({0}));
716 
717  auto elem1 = rewriter.create<spirv::CompositeExtractOp>(
718  loc, newResultType, sourceVector, rewriter.getI32ArrayAttr({1}));
719 
720  rewriter.replaceOp(deinterleaveOp, {elem0, elem1});
721  return success();
722  }
723 
724  // Indices for `shuffleEven` (result 0).
725  auto seqEven = llvm::seq<int64_t>(n / 2);
726  auto indicesEven =
727  llvm::map_to_vector(seqEven, [](int i) { return i * 2; });
728 
729  // Indices for `shuffleOdd` (result 1).
730  auto seqOdd = llvm::seq<int64_t>(n / 2);
731  auto indicesOdd =
732  llvm::map_to_vector(seqOdd, [](int i) { return i * 2 + 1; });
733 
734  // Create two SPIR-V shuffles.
735  auto shuffleEven = rewriter.create<spirv::VectorShuffleOp>(
736  loc, newResultType, sourceVector, sourceVector,
737  rewriter.getI32ArrayAttr(indicesEven));
738 
739  auto shuffleOdd = rewriter.create<spirv::VectorShuffleOp>(
740  loc, newResultType, sourceVector, sourceVector,
741  rewriter.getI32ArrayAttr(indicesOdd));
742 
743  rewriter.replaceOp(deinterleaveOp, {shuffleEven, shuffleOdd});
744  return success();
745  }
746 };
747 
748 struct VectorLoadOpConverter final
749  : public OpConversionPattern<vector::LoadOp> {
751 
752  LogicalResult
753  matchAndRewrite(vector::LoadOp loadOp, OpAdaptor adaptor,
754  ConversionPatternRewriter &rewriter) const override {
755  auto memrefType = loadOp.getMemRefType();
756  auto attr =
757  dyn_cast_or_null<spirv::StorageClassAttr>(memrefType.getMemorySpace());
758  if (!attr)
759  return rewriter.notifyMatchFailure(
760  loadOp, "expected spirv.storage_class memory space");
761 
762  const auto &typeConverter = *getTypeConverter<SPIRVTypeConverter>();
763  auto loc = loadOp.getLoc();
764  Value accessChain =
765  spirv::getElementPtr(typeConverter, memrefType, adaptor.getBase(),
766  adaptor.getIndices(), loc, rewriter);
767  if (!accessChain)
768  return rewriter.notifyMatchFailure(
769  loadOp, "failed to get memref element pointer");
770 
771  spirv::StorageClass storageClass = attr.getValue();
772  auto vectorType = loadOp.getVectorType();
773  // Use the converted vector type instead of original (single element vector
774  // would get converted to scalar).
775  auto spirvVectorType = typeConverter.convertType(vectorType);
776  auto vectorPtrType = spirv::PointerType::get(spirvVectorType, storageClass);
777 
778  // For single element vectors, we don't need to bitcast the access chain to
779  // the original vector type. Both is going to be the same, a pointer
780  // to a scalar.
781  Value castedAccessChain = (vectorType.getNumElements() == 1)
782  ? accessChain
783  : rewriter.create<spirv::BitcastOp>(
784  loc, vectorPtrType, accessChain);
785 
786  rewriter.replaceOpWithNewOp<spirv::LoadOp>(loadOp, spirvVectorType,
787  castedAccessChain);
788 
789  return success();
790  }
791 };
792 
793 struct VectorStoreOpConverter final
794  : public OpConversionPattern<vector::StoreOp> {
796 
797  LogicalResult
798  matchAndRewrite(vector::StoreOp storeOp, OpAdaptor adaptor,
799  ConversionPatternRewriter &rewriter) const override {
800  auto memrefType = storeOp.getMemRefType();
801  auto attr =
802  dyn_cast_or_null<spirv::StorageClassAttr>(memrefType.getMemorySpace());
803  if (!attr)
804  return rewriter.notifyMatchFailure(
805  storeOp, "expected spirv.storage_class memory space");
806 
807  const auto &typeConverter = *getTypeConverter<SPIRVTypeConverter>();
808  auto loc = storeOp.getLoc();
809  Value accessChain =
810  spirv::getElementPtr(typeConverter, memrefType, adaptor.getBase(),
811  adaptor.getIndices(), loc, rewriter);
812  if (!accessChain)
813  return rewriter.notifyMatchFailure(
814  storeOp, "failed to get memref element pointer");
815 
816  spirv::StorageClass storageClass = attr.getValue();
817  auto vectorType = storeOp.getVectorType();
818  auto vectorPtrType = spirv::PointerType::get(vectorType, storageClass);
819 
820  // For single element vectors, we don't need to bitcast the access chain to
821  // the original vector type. Both is going to be the same, a pointer
822  // to a scalar.
823  Value castedAccessChain = (vectorType.getNumElements() == 1)
824  ? accessChain
825  : rewriter.create<spirv::BitcastOp>(
826  loc, vectorPtrType, accessChain);
827 
828  rewriter.replaceOpWithNewOp<spirv::StoreOp>(storeOp, castedAccessChain,
829  adaptor.getValueToStore());
830 
831  return success();
832  }
833 };
834 
835 struct VectorReductionToIntDotProd final
836  : OpRewritePattern<vector::ReductionOp> {
838 
839  LogicalResult matchAndRewrite(vector::ReductionOp op,
840  PatternRewriter &rewriter) const override {
841  if (op.getKind() != vector::CombiningKind::ADD)
842  return rewriter.notifyMatchFailure(op, "combining kind is not 'add'");
843 
844  auto resultType = dyn_cast<IntegerType>(op.getType());
845  if (!resultType)
846  return rewriter.notifyMatchFailure(op, "result is not an integer");
847 
848  int64_t resultBitwidth = resultType.getIntOrFloatBitWidth();
849  if (!llvm::is_contained({32, 64}, resultBitwidth))
850  return rewriter.notifyMatchFailure(op, "unsupported integer bitwidth");
851 
852  VectorType inVecTy = op.getSourceVectorType();
853  if (!llvm::is_contained({4, 3}, inVecTy.getNumElements()) ||
854  inVecTy.getShape().size() != 1 || inVecTy.isScalable())
855  return rewriter.notifyMatchFailure(op, "unsupported vector shape");
856 
857  auto mul = op.getVector().getDefiningOp<arith::MulIOp>();
858  if (!mul)
859  return rewriter.notifyMatchFailure(
860  op, "reduction operand is not 'arith.muli'");
861 
862  if (succeeded(handleCase<arith::ExtSIOp, arith::ExtSIOp, spirv::SDotOp,
863  spirv::SDotAccSatOp, false>(op, mul, rewriter)))
864  return success();
865 
866  if (succeeded(handleCase<arith::ExtUIOp, arith::ExtUIOp, spirv::UDotOp,
867  spirv::UDotAccSatOp, false>(op, mul, rewriter)))
868  return success();
869 
870  if (succeeded(handleCase<arith::ExtSIOp, arith::ExtUIOp, spirv::SUDotOp,
871  spirv::SUDotAccSatOp, false>(op, mul, rewriter)))
872  return success();
873 
874  if (succeeded(handleCase<arith::ExtUIOp, arith::ExtSIOp, spirv::SUDotOp,
875  spirv::SUDotAccSatOp, true>(op, mul, rewriter)))
876  return success();
877 
878  return failure();
879  }
880 
881 private:
882  template <typename LhsExtensionOp, typename RhsExtensionOp, typename DotOp,
883  typename DotAccOp, bool SwapOperands>
884  static LogicalResult handleCase(vector::ReductionOp op, arith::MulIOp mul,
885  PatternRewriter &rewriter) {
886  auto lhs = mul.getLhs().getDefiningOp<LhsExtensionOp>();
887  if (!lhs)
888  return failure();
889  Value lhsIn = lhs.getIn();
890  auto lhsInType = cast<VectorType>(lhsIn.getType());
891  if (!lhsInType.getElementType().isInteger(8))
892  return failure();
893 
894  auto rhs = mul.getRhs().getDefiningOp<RhsExtensionOp>();
895  if (!rhs)
896  return failure();
897  Value rhsIn = rhs.getIn();
898  auto rhsInType = cast<VectorType>(rhsIn.getType());
899  if (!rhsInType.getElementType().isInteger(8))
900  return failure();
901 
902  if (op.getSourceVectorType().getNumElements() == 3) {
903  IntegerType i8Type = rewriter.getI8Type();
904  auto v4i8Type = VectorType::get({4}, i8Type);
905  Location loc = op.getLoc();
906  Value zero = spirv::ConstantOp::getZero(i8Type, loc, rewriter);
907  lhsIn = rewriter.create<spirv::CompositeConstructOp>(
908  loc, v4i8Type, ValueRange{lhsIn, zero});
909  rhsIn = rewriter.create<spirv::CompositeConstructOp>(
910  loc, v4i8Type, ValueRange{rhsIn, zero});
911  }
912 
913  // There's no variant of dot prod ops for unsigned LHS and signed RHS, so
914  // we have to swap operands instead in that case.
915  if (SwapOperands)
916  std::swap(lhsIn, rhsIn);
917 
918  if (Value acc = op.getAcc()) {
919  rewriter.replaceOpWithNewOp<DotAccOp>(op, op.getType(), lhsIn, rhsIn, acc,
920  nullptr);
921  } else {
922  rewriter.replaceOpWithNewOp<DotOp>(op, op.getType(), lhsIn, rhsIn,
923  nullptr);
924  }
925 
926  return success();
927  }
928 };
929 
930 struct VectorReductionToFPDotProd final
931  : OpConversionPattern<vector::ReductionOp> {
933 
934  LogicalResult
935  matchAndRewrite(vector::ReductionOp op, OpAdaptor adaptor,
936  ConversionPatternRewriter &rewriter) const override {
937  if (op.getKind() != vector::CombiningKind::ADD)
938  return rewriter.notifyMatchFailure(op, "combining kind is not 'add'");
939 
940  auto resultType = getTypeConverter()->convertType<FloatType>(op.getType());
941  if (!resultType)
942  return rewriter.notifyMatchFailure(op, "result is not a float");
943 
944  Value vec = adaptor.getVector();
945  Value acc = adaptor.getAcc();
946 
947  auto vectorType = dyn_cast<VectorType>(vec.getType());
948  if (!vectorType) {
949  assert(isa<FloatType>(vec.getType()) &&
950  "Expected the vector to be scalarized");
951  if (acc) {
952  rewriter.replaceOpWithNewOp<spirv::FAddOp>(op, acc, vec);
953  return success();
954  }
955 
956  rewriter.replaceOp(op, vec);
957  return success();
958  }
959 
960  Location loc = op.getLoc();
961  Value lhs;
962  Value rhs;
963  if (auto mul = vec.getDefiningOp<arith::MulFOp>()) {
964  lhs = mul.getLhs();
965  rhs = mul.getRhs();
966  } else {
967  // If the operand is not a mul, use a vector of ones for the dot operand
968  // to just sum up all values.
969  lhs = vec;
970  Attribute oneAttr =
971  rewriter.getFloatAttr(vectorType.getElementType(), 1.0);
972  oneAttr = SplatElementsAttr::get(vectorType, oneAttr);
973  rhs = rewriter.create<spirv::ConstantOp>(loc, vectorType, oneAttr);
974  }
975  assert(lhs);
976  assert(rhs);
977 
978  Value res = rewriter.create<spirv::DotOp>(loc, resultType, lhs, rhs);
979  if (acc)
980  res = rewriter.create<spirv::FAddOp>(loc, acc, res);
981 
982  rewriter.replaceOp(op, res);
983  return success();
984  }
985 };
986 
987 struct VectorStepOpConvert final : OpConversionPattern<vector::StepOp> {
989 
990  LogicalResult
991  matchAndRewrite(vector::StepOp stepOp, OpAdaptor adaptor,
992  ConversionPatternRewriter &rewriter) const override {
993  const auto &typeConverter = *getTypeConverter<SPIRVTypeConverter>();
994  Type dstType = typeConverter.convertType(stepOp.getType());
995  if (!dstType)
996  return failure();
997 
998  Location loc = stepOp.getLoc();
999  int64_t numElements = stepOp.getType().getNumElements();
1000  auto intType =
1001  rewriter.getIntegerType(typeConverter.getIndexTypeBitwidth());
1002 
1003  // Input vectors of size 1 are converted to scalars by the type converter.
1004  // We just create a constant in this case.
1005  if (numElements == 1) {
1006  Value zero = spirv::ConstantOp::getZero(intType, loc, rewriter);
1007  rewriter.replaceOp(stepOp, zero);
1008  return success();
1009  }
1010 
1011  SmallVector<Value> source;
1012  source.reserve(numElements);
1013  for (int64_t i = 0; i < numElements; ++i) {
1014  Attribute intAttr = rewriter.getIntegerAttr(intType, i);
1015  Value constOp = rewriter.create<spirv::ConstantOp>(loc, intType, intAttr);
1016  source.push_back(constOp);
1017  }
1018  rewriter.replaceOpWithNewOp<spirv::CompositeConstructOp>(stepOp, dstType,
1019  source);
1020  return success();
1021  }
1022 };
1023 
1024 } // namespace
1025 #define CL_INT_MAX_MIN_OPS \
1026  spirv::CLUMaxOp, spirv::CLUMinOp, spirv::CLSMaxOp, spirv::CLSMinOp
1027 
1028 #define GL_INT_MAX_MIN_OPS \
1029  spirv::GLUMaxOp, spirv::GLUMinOp, spirv::GLSMaxOp, spirv::GLSMinOp
1030 
1031 #define CL_FLOAT_MAX_MIN_OPS spirv::CLFMaxOp, spirv::CLFMinOp
1032 #define GL_FLOAT_MAX_MIN_OPS spirv::GLFMaxOp, spirv::GLFMinOp
1033 
1035  const SPIRVTypeConverter &typeConverter, RewritePatternSet &patterns) {
1036  patterns.add<
1037  VectorBitcastConvert, VectorBroadcastConvert,
1038  VectorExtractElementOpConvert, VectorExtractOpConvert,
1039  VectorExtractStridedSliceOpConvert, VectorFmaOpConvert<spirv::GLFmaOp>,
1040  VectorFmaOpConvert<spirv::CLFmaOp>, VectorFromElementsOpConvert,
1041  VectorInsertElementOpConvert, VectorInsertOpConvert,
1042  VectorReductionPattern<GL_INT_MAX_MIN_OPS>,
1043  VectorReductionPattern<CL_INT_MAX_MIN_OPS>,
1044  VectorReductionFloatMinMax<CL_FLOAT_MAX_MIN_OPS>,
1045  VectorReductionFloatMinMax<GL_FLOAT_MAX_MIN_OPS>, VectorShapeCast,
1046  VectorInsertStridedSliceOpConvert, VectorShuffleOpConvert,
1047  VectorInterleaveOpConvert, VectorDeinterleaveOpConvert,
1048  VectorSplatPattern, VectorLoadOpConverter, VectorStoreOpConverter,
1049  VectorStepOpConvert>(typeConverter, patterns.getContext(),
1050  PatternBenefit(1));
1051 
1052  // Make sure that the more specialized dot product pattern has higher benefit
1053  // than the generic one that extracts all elements.
1054  patterns.add<VectorReductionToFPDotProd>(typeConverter, patterns.getContext(),
1055  PatternBenefit(2));
1056 }
1057 
1060  patterns.add<VectorReductionToIntDotProd>(patterns.getContext());
1061 }
static Value getZero(OpBuilder &b, Location loc, Type elementType)
Get zero value for an element type.
#define MINUI(lhs, rhs)
static uint64_t getFirstIntValue(ArrayAttr attr)
Returns the integer value from the first valid input element, assuming Value inputs are defined by a ...
static int getNumBits(Type type)
Returns the number of bits for the given scalar/vector type.
#define INT_AND_FLOAT_CASE(kind, iop, fop)
#define INT_OR_FLOAT_CASE(kind, fop)
Attributes are known-constant values of operations.
Definition: Attributes.h:25
IntegerAttr getIntegerAttr(Type type, int64_t value)
Definition: Builders.cpp:224
ArrayAttr getI32ArrayAttr(ArrayRef< int32_t > values)
Definition: Builders.cpp:272
FloatAttr getFloatAttr(Type type, double value)
Definition: Builders.cpp:250
IntegerType getIntegerType(unsigned width)
Definition: Builders.cpp:67
IntegerType getI8Type()
Definition: Builders.cpp:59
This class implements a pattern rewriter for use with ConversionPatterns.
void replaceOp(Operation *op, ValueRange newValues) override
Replace the given operation with the new values.
static DenseElementsAttr get(ShapedType type, ArrayRef< Attribute > values)
Constructs a dense elements attribute from an array of element values.
This class defines the main interface for locations in MLIR and acts as a non-nullable wrapper around...
Definition: Location.h:66
Operation * create(const OperationState &state)
Creates an operation given the fields represented as an OperationState.
Definition: Builders.cpp:453
OpConversionPattern is a wrapper around ConversionPattern that allows for matching and rewriting agai...
OpConversionPattern(MLIRContext *context, PatternBenefit benefit=1)
This class implements the operand iterators for the Operation class.
Definition: ValueRange.h:42
This class represents the benefit of a pattern match in a unitless scheme that ranges from 0 (very li...
Definition: PatternMatch.h:34
A special type of RewriterBase that coordinates the application of a rewrite pattern on the current I...
Definition: PatternMatch.h:803
std::enable_if_t<!std::is_convertible< CallbackT, Twine >::value, LogicalResult > notifyMatchFailure(Location loc, CallbackT &&reasonCallback)
Used to notify the listener that the IR failed to be rewritten because of a match failure,...
Definition: PatternMatch.h:736
OpTy replaceOpWithNewOp(Operation *op, Args &&...args)
Replace the results of the given (original) op with a new op that is created without verification (re...
Definition: PatternMatch.h:554
Type conversion from builtin types to SPIR-V types for shader interface.
Type conversion class.
LogicalResult convertType(Type t, SmallVectorImpl< Type > &results) const
Convert the given type.
Instances of the Type class are uniqued, have an immutable identifier and an optional mutable compone...
Definition: Types.h:74
unsigned getIntOrFloatBitWidth() const
Return the bit width of an integer or a float type, assert failure on other types.
Definition: Types.cpp:122
This class provides an abstraction over the different types of ranges over Values.
Definition: ValueRange.h:381
This class represents an instance of an SSA value in the MLIR system, representing a computable value...
Definition: Value.h:96
Type getType() const
Return the type of this value.
Definition: Value.h:129
Operation * getDefiningOp() const
If this value is the result of an operation, return the operation that defines it.
Definition: Value.cpp:20
static PointerType get(Type pointeeType, StorageClass storageClass)
Definition: SPIRVTypes.cpp:406
Value getElementPtr(const SPIRVTypeConverter &typeConverter, MemRefType baseType, Value basePtr, ValueRange indices, Location loc, OpBuilder &builder)
Performs the index computation to get to the element at indices of the memory pointed to by basePtr,...
Include the generated interface declarations.
bool matchPattern(Value value, const Pattern &pattern)
Entry point for matching a pattern over a Value.
Definition: Matchers.h:490
detail::constant_int_value_binder m_ConstantInt(IntegerAttr::ValueType *bind_value)
Matches a constant holding a scalar/vector/tensor integer (splat) and writes the integer value to bin...
Definition: Matchers.h:527
std::optional< int64_t > getConstantIntValue(OpFoldResult ofr)
If ofr is a constant integer or an IntegerAttr, return the integer.
void populateVectorReductionToSPIRVDotProductPatterns(RewritePatternSet &patterns)
Appends patterns to convert vector reduction of the form:
const FrozenRewritePatternSet & patterns
auto get(MLIRContext *context, Ts &&...params)
Helper method that injects context only if needed, this helps unify some of the attribute constructio...
void populateVectorToSPIRVPatterns(const SPIRVTypeConverter &typeConverter, RewritePatternSet &patterns)
Appends to a pattern list additional patterns for translating Vector Ops to SPIR-V ops.
OpRewritePattern is a wrapper around RewritePattern that allows for matching and rewriting against an...
Definition: PatternMatch.h:358
OpRewritePattern(MLIRContext *context, PatternBenefit benefit=1, ArrayRef< StringRef > generatedNames={})
Patterns must specify the root operation name they match against, and can also specify the benefit of...
Definition: PatternMatch.h:368