MLIR  19.0.0git
ConvertVectorToLLVM.cpp
Go to the documentation of this file.
1 //===- VectorToLLVM.cpp - Conversion from Vector to the LLVM dialect ------===//
2 //
3 // Part of the LLVM Project, under the Apache License v2.0 with LLVM Exceptions.
4 // See https://llvm.org/LICENSE.txt for license information.
5 // SPDX-License-Identifier: Apache-2.0 WITH LLVM-exception
6 //
7 //===----------------------------------------------------------------------===//
8 
10 
26 #include "mlir/IR/BuiltinTypes.h"
27 #include "mlir/IR/TypeUtilities.h"
30 #include "llvm/ADT/APFloat.h"
31 #include "llvm/Support/Casting.h"
32 #include <optional>
33 
34 using namespace mlir;
35 using namespace mlir::vector;
36 
37 // Helper to reduce vector type by *all* but one rank at back.
38 static VectorType reducedVectorTypeBack(VectorType tp) {
39  assert((tp.getRank() > 1) && "unlowerable vector type");
40  return VectorType::get(tp.getShape().take_back(), tp.getElementType(),
41  tp.getScalableDims().take_back());
42 }
43 
44 // Helper that picks the proper sequence for inserting.
46  const LLVMTypeConverter &typeConverter, Location loc,
47  Value val1, Value val2, Type llvmType, int64_t rank,
48  int64_t pos) {
49  assert(rank > 0 && "0-D vector corner case should have been handled already");
50  if (rank == 1) {
51  auto idxType = rewriter.getIndexType();
52  auto constant = rewriter.create<LLVM::ConstantOp>(
53  loc, typeConverter.convertType(idxType),
54  rewriter.getIntegerAttr(idxType, pos));
55  return rewriter.create<LLVM::InsertElementOp>(loc, llvmType, val1, val2,
56  constant);
57  }
58  return rewriter.create<LLVM::InsertValueOp>(loc, val1, val2, pos);
59 }
60 
61 // Helper that picks the proper sequence for extracting.
63  const LLVMTypeConverter &typeConverter, Location loc,
64  Value val, Type llvmType, int64_t rank, int64_t pos) {
65  if (rank <= 1) {
66  auto idxType = rewriter.getIndexType();
67  auto constant = rewriter.create<LLVM::ConstantOp>(
68  loc, typeConverter.convertType(idxType),
69  rewriter.getIntegerAttr(idxType, pos));
70  return rewriter.create<LLVM::ExtractElementOp>(loc, llvmType, val,
71  constant);
72  }
73  return rewriter.create<LLVM::ExtractValueOp>(loc, val, pos);
74 }
75 
76 // Helper that returns data layout alignment of a memref.
78  MemRefType memrefType, unsigned &align) {
79  Type elementTy = typeConverter.convertType(memrefType.getElementType());
80  if (!elementTy)
81  return failure();
82 
83  // TODO: this should use the MLIR data layout when it becomes available and
84  // stop depending on translation.
85  llvm::LLVMContext llvmContext;
86  align = LLVM::TypeToLLVMIRTranslator(llvmContext)
87  .getPreferredAlignment(elementTy, typeConverter.getDataLayout());
88  return success();
89 }
90 
91 // Check if the last stride is non-unit and has a valid memory space.
92 static LogicalResult isMemRefTypeSupported(MemRefType memRefType,
93  const LLVMTypeConverter &converter) {
94  if (!isLastMemrefDimUnitStride(memRefType))
95  return failure();
96  if (failed(converter.getMemRefAddressSpace(memRefType)))
97  return failure();
98  return success();
99 }
100 
101 // Add an index vector component to a base pointer.
103  const LLVMTypeConverter &typeConverter,
104  MemRefType memRefType, Value llvmMemref, Value base,
105  Value index, uint64_t vLen) {
106  assert(succeeded(isMemRefTypeSupported(memRefType, typeConverter)) &&
107  "unsupported memref type");
108  auto pType = MemRefDescriptor(llvmMemref).getElementPtrType();
109  auto ptrsType = LLVM::getFixedVectorType(pType, vLen);
110  return rewriter.create<LLVM::GEPOp>(
111  loc, ptrsType, typeConverter.convertType(memRefType.getElementType()),
112  base, index);
113 }
114 
115 /// Convert `foldResult` into a Value. Integer attribute is converted to
116 /// an LLVM constant op.
117 static Value getAsLLVMValue(OpBuilder &builder, Location loc,
118  OpFoldResult foldResult) {
119  if (auto attr = foldResult.dyn_cast<Attribute>()) {
120  auto intAttr = cast<IntegerAttr>(attr);
121  return builder.create<LLVM::ConstantOp>(loc, intAttr).getResult();
122  }
123 
124  return foldResult.get<Value>();
125 }
126 
127 namespace {
128 
129 /// Trivial Vector to LLVM conversions
130 using VectorScaleOpConversion =
132 
133 /// Conversion pattern for a vector.bitcast.
134 class VectorBitCastOpConversion
135  : public ConvertOpToLLVMPattern<vector::BitCastOp> {
136 public:
138 
140  matchAndRewrite(vector::BitCastOp bitCastOp, OpAdaptor adaptor,
141  ConversionPatternRewriter &rewriter) const override {
142  // Only 0-D and 1-D vectors can be lowered to LLVM.
143  VectorType resultTy = bitCastOp.getResultVectorType();
144  if (resultTy.getRank() > 1)
145  return failure();
146  Type newResultTy = typeConverter->convertType(resultTy);
147  rewriter.replaceOpWithNewOp<LLVM::BitcastOp>(bitCastOp, newResultTy,
148  adaptor.getOperands()[0]);
149  return success();
150  }
151 };
152 
153 /// Conversion pattern for a vector.matrix_multiply.
154 /// This is lowered directly to the proper llvm.intr.matrix.multiply.
155 class VectorMatmulOpConversion
156  : public ConvertOpToLLVMPattern<vector::MatmulOp> {
157 public:
159 
161  matchAndRewrite(vector::MatmulOp matmulOp, OpAdaptor adaptor,
162  ConversionPatternRewriter &rewriter) const override {
163  rewriter.replaceOpWithNewOp<LLVM::MatrixMultiplyOp>(
164  matmulOp, typeConverter->convertType(matmulOp.getRes().getType()),
165  adaptor.getLhs(), adaptor.getRhs(), matmulOp.getLhsRows(),
166  matmulOp.getLhsColumns(), matmulOp.getRhsColumns());
167  return success();
168  }
169 };
170 
171 /// Conversion pattern for a vector.flat_transpose.
172 /// This is lowered directly to the proper llvm.intr.matrix.transpose.
173 class VectorFlatTransposeOpConversion
174  : public ConvertOpToLLVMPattern<vector::FlatTransposeOp> {
175 public:
177 
179  matchAndRewrite(vector::FlatTransposeOp transOp, OpAdaptor adaptor,
180  ConversionPatternRewriter &rewriter) const override {
181  rewriter.replaceOpWithNewOp<LLVM::MatrixTransposeOp>(
182  transOp, typeConverter->convertType(transOp.getRes().getType()),
183  adaptor.getMatrix(), transOp.getRows(), transOp.getColumns());
184  return success();
185  }
186 };
187 
188 /// Overloaded utility that replaces a vector.load, vector.store,
189 /// vector.maskedload and vector.maskedstore with their respective LLVM
190 /// couterparts.
191 static void replaceLoadOrStoreOp(vector::LoadOp loadOp,
192  vector::LoadOpAdaptor adaptor,
193  VectorType vectorTy, Value ptr, unsigned align,
194  ConversionPatternRewriter &rewriter) {
195  rewriter.replaceOpWithNewOp<LLVM::LoadOp>(loadOp, vectorTy, ptr, align,
196  /*volatile_=*/false,
197  loadOp.getNontemporal());
198 }
199 
200 static void replaceLoadOrStoreOp(vector::MaskedLoadOp loadOp,
201  vector::MaskedLoadOpAdaptor adaptor,
202  VectorType vectorTy, Value ptr, unsigned align,
203  ConversionPatternRewriter &rewriter) {
204  rewriter.replaceOpWithNewOp<LLVM::MaskedLoadOp>(
205  loadOp, vectorTy, ptr, adaptor.getMask(), adaptor.getPassThru(), align);
206 }
207 
208 static void replaceLoadOrStoreOp(vector::StoreOp storeOp,
209  vector::StoreOpAdaptor adaptor,
210  VectorType vectorTy, Value ptr, unsigned align,
211  ConversionPatternRewriter &rewriter) {
212  rewriter.replaceOpWithNewOp<LLVM::StoreOp>(storeOp, adaptor.getValueToStore(),
213  ptr, align, /*volatile_=*/false,
214  storeOp.getNontemporal());
215 }
216 
217 static void replaceLoadOrStoreOp(vector::MaskedStoreOp storeOp,
218  vector::MaskedStoreOpAdaptor adaptor,
219  VectorType vectorTy, Value ptr, unsigned align,
220  ConversionPatternRewriter &rewriter) {
221  rewriter.replaceOpWithNewOp<LLVM::MaskedStoreOp>(
222  storeOp, adaptor.getValueToStore(), ptr, adaptor.getMask(), align);
223 }
224 
225 /// Conversion pattern for a vector.load, vector.store, vector.maskedload, and
226 /// vector.maskedstore.
227 template <class LoadOrStoreOp>
228 class VectorLoadStoreConversion : public ConvertOpToLLVMPattern<LoadOrStoreOp> {
229 public:
231 
233  matchAndRewrite(LoadOrStoreOp loadOrStoreOp,
234  typename LoadOrStoreOp::Adaptor adaptor,
235  ConversionPatternRewriter &rewriter) const override {
236  // Only 1-D vectors can be lowered to LLVM.
237  VectorType vectorTy = loadOrStoreOp.getVectorType();
238  if (vectorTy.getRank() > 1)
239  return failure();
240 
241  auto loc = loadOrStoreOp->getLoc();
242  MemRefType memRefTy = loadOrStoreOp.getMemRefType();
243 
244  // Resolve alignment.
245  unsigned align;
246  if (failed(getMemRefAlignment(*this->getTypeConverter(), memRefTy, align)))
247  return failure();
248 
249  // Resolve address.
250  auto vtype = cast<VectorType>(
251  this->typeConverter->convertType(loadOrStoreOp.getVectorType()));
252  Value dataPtr = this->getStridedElementPtr(loc, memRefTy, adaptor.getBase(),
253  adaptor.getIndices(), rewriter);
254  replaceLoadOrStoreOp(loadOrStoreOp, adaptor, vtype, dataPtr, align,
255  rewriter);
256  return success();
257  }
258 };
259 
260 /// Conversion pattern for a vector.gather.
261 class VectorGatherOpConversion
262  : public ConvertOpToLLVMPattern<vector::GatherOp> {
263 public:
265 
267  matchAndRewrite(vector::GatherOp gather, OpAdaptor adaptor,
268  ConversionPatternRewriter &rewriter) const override {
269  MemRefType memRefType = dyn_cast<MemRefType>(gather.getBaseType());
270  assert(memRefType && "The base should be bufferized");
271 
272  if (failed(isMemRefTypeSupported(memRefType, *this->getTypeConverter())))
273  return failure();
274 
275  auto loc = gather->getLoc();
276 
277  // Resolve alignment.
278  unsigned align;
279  if (failed(getMemRefAlignment(*getTypeConverter(), memRefType, align)))
280  return failure();
281 
282  Value ptr = getStridedElementPtr(loc, memRefType, adaptor.getBase(),
283  adaptor.getIndices(), rewriter);
284  Value base = adaptor.getBase();
285 
286  auto llvmNDVectorTy = adaptor.getIndexVec().getType();
287  // Handle the simple case of 1-D vector.
288  if (!isa<LLVM::LLVMArrayType>(llvmNDVectorTy)) {
289  auto vType = gather.getVectorType();
290  // Resolve address.
291  Value ptrs = getIndexedPtrs(rewriter, loc, *this->getTypeConverter(),
292  memRefType, base, ptr, adaptor.getIndexVec(),
293  /*vLen=*/vType.getDimSize(0));
294  // Replace with the gather intrinsic.
295  rewriter.replaceOpWithNewOp<LLVM::masked_gather>(
296  gather, typeConverter->convertType(vType), ptrs, adaptor.getMask(),
297  adaptor.getPassThru(), rewriter.getI32IntegerAttr(align));
298  return success();
299  }
300 
301  const LLVMTypeConverter &typeConverter = *this->getTypeConverter();
302  auto callback = [align, memRefType, base, ptr, loc, &rewriter,
303  &typeConverter](Type llvm1DVectorTy,
304  ValueRange vectorOperands) {
305  // Resolve address.
306  Value ptrs = getIndexedPtrs(
307  rewriter, loc, typeConverter, memRefType, base, ptr,
308  /*index=*/vectorOperands[0],
309  LLVM::getVectorNumElements(llvm1DVectorTy).getFixedValue());
310  // Create the gather intrinsic.
311  return rewriter.create<LLVM::masked_gather>(
312  loc, llvm1DVectorTy, ptrs, /*mask=*/vectorOperands[1],
313  /*passThru=*/vectorOperands[2], rewriter.getI32IntegerAttr(align));
314  };
315  SmallVector<Value> vectorOperands = {
316  adaptor.getIndexVec(), adaptor.getMask(), adaptor.getPassThru()};
318  gather, vectorOperands, *getTypeConverter(), callback, rewriter);
319  }
320 };
321 
322 /// Conversion pattern for a vector.scatter.
323 class VectorScatterOpConversion
324  : public ConvertOpToLLVMPattern<vector::ScatterOp> {
325 public:
327 
329  matchAndRewrite(vector::ScatterOp scatter, OpAdaptor adaptor,
330  ConversionPatternRewriter &rewriter) const override {
331  auto loc = scatter->getLoc();
332  MemRefType memRefType = scatter.getMemRefType();
333 
334  if (failed(isMemRefTypeSupported(memRefType, *this->getTypeConverter())))
335  return failure();
336 
337  // Resolve alignment.
338  unsigned align;
339  if (failed(getMemRefAlignment(*getTypeConverter(), memRefType, align)))
340  return failure();
341 
342  // Resolve address.
343  VectorType vType = scatter.getVectorType();
344  Value ptr = getStridedElementPtr(loc, memRefType, adaptor.getBase(),
345  adaptor.getIndices(), rewriter);
346  Value ptrs = getIndexedPtrs(
347  rewriter, loc, *this->getTypeConverter(), memRefType, adaptor.getBase(),
348  ptr, adaptor.getIndexVec(), /*vLen=*/vType.getDimSize(0));
349 
350  // Replace with the scatter intrinsic.
351  rewriter.replaceOpWithNewOp<LLVM::masked_scatter>(
352  scatter, adaptor.getValueToStore(), ptrs, adaptor.getMask(),
353  rewriter.getI32IntegerAttr(align));
354  return success();
355  }
356 };
357 
358 /// Conversion pattern for a vector.expandload.
359 class VectorExpandLoadOpConversion
360  : public ConvertOpToLLVMPattern<vector::ExpandLoadOp> {
361 public:
363 
365  matchAndRewrite(vector::ExpandLoadOp expand, OpAdaptor adaptor,
366  ConversionPatternRewriter &rewriter) const override {
367  auto loc = expand->getLoc();
368  MemRefType memRefType = expand.getMemRefType();
369 
370  // Resolve address.
371  auto vtype = typeConverter->convertType(expand.getVectorType());
372  Value ptr = getStridedElementPtr(loc, memRefType, adaptor.getBase(),
373  adaptor.getIndices(), rewriter);
374 
375  rewriter.replaceOpWithNewOp<LLVM::masked_expandload>(
376  expand, vtype, ptr, adaptor.getMask(), adaptor.getPassThru());
377  return success();
378  }
379 };
380 
381 /// Conversion pattern for a vector.compressstore.
382 class VectorCompressStoreOpConversion
383  : public ConvertOpToLLVMPattern<vector::CompressStoreOp> {
384 public:
386 
388  matchAndRewrite(vector::CompressStoreOp compress, OpAdaptor adaptor,
389  ConversionPatternRewriter &rewriter) const override {
390  auto loc = compress->getLoc();
391  MemRefType memRefType = compress.getMemRefType();
392 
393  // Resolve address.
394  Value ptr = getStridedElementPtr(loc, memRefType, adaptor.getBase(),
395  adaptor.getIndices(), rewriter);
396 
397  rewriter.replaceOpWithNewOp<LLVM::masked_compressstore>(
398  compress, adaptor.getValueToStore(), ptr, adaptor.getMask());
399  return success();
400  }
401 };
402 
403 /// Reduction neutral classes for overloading.
404 class ReductionNeutralZero {};
405 class ReductionNeutralIntOne {};
406 class ReductionNeutralFPOne {};
407 class ReductionNeutralAllOnes {};
408 class ReductionNeutralSIntMin {};
409 class ReductionNeutralUIntMin {};
410 class ReductionNeutralSIntMax {};
411 class ReductionNeutralUIntMax {};
412 class ReductionNeutralFPMin {};
413 class ReductionNeutralFPMax {};
414 
415 /// Create the reduction neutral zero value.
416 static Value createReductionNeutralValue(ReductionNeutralZero neutral,
417  ConversionPatternRewriter &rewriter,
418  Location loc, Type llvmType) {
419  return rewriter.create<LLVM::ConstantOp>(loc, llvmType,
420  rewriter.getZeroAttr(llvmType));
421 }
422 
423 /// Create the reduction neutral integer one value.
424 static Value createReductionNeutralValue(ReductionNeutralIntOne neutral,
425  ConversionPatternRewriter &rewriter,
426  Location loc, Type llvmType) {
427  return rewriter.create<LLVM::ConstantOp>(
428  loc, llvmType, rewriter.getIntegerAttr(llvmType, 1));
429 }
430 
431 /// Create the reduction neutral fp one value.
432 static Value createReductionNeutralValue(ReductionNeutralFPOne neutral,
433  ConversionPatternRewriter &rewriter,
434  Location loc, Type llvmType) {
435  return rewriter.create<LLVM::ConstantOp>(
436  loc, llvmType, rewriter.getFloatAttr(llvmType, 1.0));
437 }
438 
439 /// Create the reduction neutral all-ones value.
440 static Value createReductionNeutralValue(ReductionNeutralAllOnes neutral,
441  ConversionPatternRewriter &rewriter,
442  Location loc, Type llvmType) {
443  return rewriter.create<LLVM::ConstantOp>(
444  loc, llvmType,
445  rewriter.getIntegerAttr(
446  llvmType, llvm::APInt::getAllOnes(llvmType.getIntOrFloatBitWidth())));
447 }
448 
449 /// Create the reduction neutral signed int minimum value.
450 static Value createReductionNeutralValue(ReductionNeutralSIntMin neutral,
451  ConversionPatternRewriter &rewriter,
452  Location loc, Type llvmType) {
453  return rewriter.create<LLVM::ConstantOp>(
454  loc, llvmType,
455  rewriter.getIntegerAttr(llvmType, llvm::APInt::getSignedMinValue(
456  llvmType.getIntOrFloatBitWidth())));
457 }
458 
459 /// Create the reduction neutral unsigned int minimum value.
460 static Value createReductionNeutralValue(ReductionNeutralUIntMin neutral,
461  ConversionPatternRewriter &rewriter,
462  Location loc, Type llvmType) {
463  return rewriter.create<LLVM::ConstantOp>(
464  loc, llvmType,
465  rewriter.getIntegerAttr(llvmType, llvm::APInt::getMinValue(
466  llvmType.getIntOrFloatBitWidth())));
467 }
468 
469 /// Create the reduction neutral signed int maximum value.
470 static Value createReductionNeutralValue(ReductionNeutralSIntMax neutral,
471  ConversionPatternRewriter &rewriter,
472  Location loc, Type llvmType) {
473  return rewriter.create<LLVM::ConstantOp>(
474  loc, llvmType,
475  rewriter.getIntegerAttr(llvmType, llvm::APInt::getSignedMaxValue(
476  llvmType.getIntOrFloatBitWidth())));
477 }
478 
479 /// Create the reduction neutral unsigned int maximum value.
480 static Value createReductionNeutralValue(ReductionNeutralUIntMax neutral,
481  ConversionPatternRewriter &rewriter,
482  Location loc, Type llvmType) {
483  return rewriter.create<LLVM::ConstantOp>(
484  loc, llvmType,
485  rewriter.getIntegerAttr(llvmType, llvm::APInt::getMaxValue(
486  llvmType.getIntOrFloatBitWidth())));
487 }
488 
489 /// Create the reduction neutral fp minimum value.
490 static Value createReductionNeutralValue(ReductionNeutralFPMin neutral,
491  ConversionPatternRewriter &rewriter,
492  Location loc, Type llvmType) {
493  auto floatType = cast<FloatType>(llvmType);
494  return rewriter.create<LLVM::ConstantOp>(
495  loc, llvmType,
496  rewriter.getFloatAttr(
497  llvmType, llvm::APFloat::getQNaN(floatType.getFloatSemantics(),
498  /*Negative=*/false)));
499 }
500 
501 /// Create the reduction neutral fp maximum value.
502 static Value createReductionNeutralValue(ReductionNeutralFPMax neutral,
503  ConversionPatternRewriter &rewriter,
504  Location loc, Type llvmType) {
505  auto floatType = cast<FloatType>(llvmType);
506  return rewriter.create<LLVM::ConstantOp>(
507  loc, llvmType,
508  rewriter.getFloatAttr(
509  llvmType, llvm::APFloat::getQNaN(floatType.getFloatSemantics(),
510  /*Negative=*/true)));
511 }
512 
513 /// Returns `accumulator` if it has a valid value. Otherwise, creates and
514 /// returns a new accumulator value using `ReductionNeutral`.
515 template <class ReductionNeutral>
516 static Value getOrCreateAccumulator(ConversionPatternRewriter &rewriter,
517  Location loc, Type llvmType,
518  Value accumulator) {
519  if (accumulator)
520  return accumulator;
521 
522  return createReductionNeutralValue(ReductionNeutral(), rewriter, loc,
523  llvmType);
524 }
525 
526 /// Creates a constant value with the 1-D vector shape provided in `llvmType`.
527 /// This is used as effective vector length by some intrinsics supporting
528 /// dynamic vector lengths at runtime.
529 static Value createVectorLengthValue(ConversionPatternRewriter &rewriter,
530  Location loc, Type llvmType) {
531  VectorType vType = cast<VectorType>(llvmType);
532  auto vShape = vType.getShape();
533  assert(vShape.size() == 1 && "Unexpected multi-dim vector type");
534 
535  return rewriter.create<LLVM::ConstantOp>(
536  loc, rewriter.getI32Type(),
537  rewriter.getIntegerAttr(rewriter.getI32Type(), vShape[0]));
538 }
539 
540 /// Helper method to lower a `vector.reduction` op that performs an arithmetic
541 /// operation like add,mul, etc.. `VectorOp` is the LLVM vector intrinsic to use
542 /// and `ScalarOp` is the scalar operation used to add the accumulation value if
543 /// non-null.
544 template <class LLVMRedIntrinOp, class ScalarOp>
545 static Value createIntegerReductionArithmeticOpLowering(
546  ConversionPatternRewriter &rewriter, Location loc, Type llvmType,
547  Value vectorOperand, Value accumulator) {
548 
549  Value result = rewriter.create<LLVMRedIntrinOp>(loc, llvmType, vectorOperand);
550 
551  if (accumulator)
552  result = rewriter.create<ScalarOp>(loc, accumulator, result);
553  return result;
554 }
555 
556 /// Helper method to lower a `vector.reduction` operation that performs
557 /// a comparison operation like `min`/`max`. `VectorOp` is the LLVM vector
558 /// intrinsic to use and `predicate` is the predicate to use to compare+combine
559 /// the accumulator value if non-null.
560 template <class LLVMRedIntrinOp>
561 static Value createIntegerReductionComparisonOpLowering(
562  ConversionPatternRewriter &rewriter, Location loc, Type llvmType,
563  Value vectorOperand, Value accumulator, LLVM::ICmpPredicate predicate) {
564  Value result = rewriter.create<LLVMRedIntrinOp>(loc, llvmType, vectorOperand);
565  if (accumulator) {
566  Value cmp =
567  rewriter.create<LLVM::ICmpOp>(loc, predicate, accumulator, result);
568  result = rewriter.create<LLVM::SelectOp>(loc, cmp, accumulator, result);
569  }
570  return result;
571 }
572 
573 namespace {
574 template <typename Source>
575 struct VectorToScalarMapper;
576 template <>
577 struct VectorToScalarMapper<LLVM::vector_reduce_fmaximum> {
578  using Type = LLVM::MaximumOp;
579 };
580 template <>
581 struct VectorToScalarMapper<LLVM::vector_reduce_fminimum> {
582  using Type = LLVM::MinimumOp;
583 };
584 template <>
585 struct VectorToScalarMapper<LLVM::vector_reduce_fmax> {
586  using Type = LLVM::MaxNumOp;
587 };
588 template <>
589 struct VectorToScalarMapper<LLVM::vector_reduce_fmin> {
590  using Type = LLVM::MinNumOp;
591 };
592 } // namespace
593 
594 template <class LLVMRedIntrinOp>
595 static Value createFPReductionComparisonOpLowering(
596  ConversionPatternRewriter &rewriter, Location loc, Type llvmType,
597  Value vectorOperand, Value accumulator, LLVM::FastmathFlagsAttr fmf) {
598  Value result =
599  rewriter.create<LLVMRedIntrinOp>(loc, llvmType, vectorOperand, fmf);
600 
601  if (accumulator) {
602  result =
603  rewriter.create<typename VectorToScalarMapper<LLVMRedIntrinOp>::Type>(
604  loc, result, accumulator);
605  }
606 
607  return result;
608 }
609 
610 /// Reduction neutral classes for overloading
611 class MaskNeutralFMaximum {};
612 class MaskNeutralFMinimum {};
613 
614 /// Get the mask neutral floating point maximum value
615 static llvm::APFloat
616 getMaskNeutralValue(MaskNeutralFMaximum,
617  const llvm::fltSemantics &floatSemantics) {
618  return llvm::APFloat::getSmallest(floatSemantics, /*Negative=*/true);
619 }
620 /// Get the mask neutral floating point minimum value
621 static llvm::APFloat
622 getMaskNeutralValue(MaskNeutralFMinimum,
623  const llvm::fltSemantics &floatSemantics) {
624  return llvm::APFloat::getLargest(floatSemantics, /*Negative=*/false);
625 }
626 
627 /// Create the mask neutral floating point MLIR vector constant
628 template <typename MaskNeutral>
629 static Value createMaskNeutralValue(ConversionPatternRewriter &rewriter,
630  Location loc, Type llvmType,
631  Type vectorType) {
632  const auto &floatSemantics = cast<FloatType>(llvmType).getFloatSemantics();
633  auto value = getMaskNeutralValue(MaskNeutral{}, floatSemantics);
634  auto denseValue = DenseElementsAttr::get(cast<ShapedType>(vectorType), value);
635  return rewriter.create<LLVM::ConstantOp>(loc, vectorType, denseValue);
636 }
637 
638 /// Lowers masked `fmaximum` and `fminimum` reductions using the non-masked
639 /// intrinsics. It is a workaround to overcome the lack of masked intrinsics for
640 /// `fmaximum`/`fminimum`.
641 /// More information: https://github.com/llvm/llvm-project/issues/64940
642 template <class LLVMRedIntrinOp, class MaskNeutral>
643 static Value
644 lowerMaskedReductionWithRegular(ConversionPatternRewriter &rewriter,
645  Location loc, Type llvmType,
646  Value vectorOperand, Value accumulator,
647  Value mask, LLVM::FastmathFlagsAttr fmf) {
648  const Value vectorMaskNeutral = createMaskNeutralValue<MaskNeutral>(
649  rewriter, loc, llvmType, vectorOperand.getType());
650  const Value selectedVectorByMask = rewriter.create<LLVM::SelectOp>(
651  loc, mask, vectorOperand, vectorMaskNeutral);
652  return createFPReductionComparisonOpLowering<LLVMRedIntrinOp>(
653  rewriter, loc, llvmType, selectedVectorByMask, accumulator, fmf);
654 }
655 
656 template <class LLVMRedIntrinOp, class ReductionNeutral>
657 static Value
658 lowerReductionWithStartValue(ConversionPatternRewriter &rewriter, Location loc,
659  Type llvmType, Value vectorOperand,
660  Value accumulator, LLVM::FastmathFlagsAttr fmf) {
661  accumulator = getOrCreateAccumulator<ReductionNeutral>(rewriter, loc,
662  llvmType, accumulator);
663  return rewriter.create<LLVMRedIntrinOp>(loc, llvmType,
664  /*startValue=*/accumulator,
665  vectorOperand, fmf);
666 }
667 
668 /// Overloaded methods to lower a *predicated* reduction to an llvm instrinsic
669 /// that requires a start value. This start value format spans across fp
670 /// reductions without mask and all the masked reduction intrinsics.
671 template <class LLVMVPRedIntrinOp, class ReductionNeutral>
672 static Value
673 lowerPredicatedReductionWithStartValue(ConversionPatternRewriter &rewriter,
674  Location loc, Type llvmType,
675  Value vectorOperand, Value accumulator) {
676  accumulator = getOrCreateAccumulator<ReductionNeutral>(rewriter, loc,
677  llvmType, accumulator);
678  return rewriter.create<LLVMVPRedIntrinOp>(loc, llvmType,
679  /*startValue=*/accumulator,
680  vectorOperand);
681 }
682 
683 template <class LLVMVPRedIntrinOp, class ReductionNeutral>
684 static Value lowerPredicatedReductionWithStartValue(
685  ConversionPatternRewriter &rewriter, Location loc, Type llvmType,
686  Value vectorOperand, Value accumulator, Value mask) {
687  accumulator = getOrCreateAccumulator<ReductionNeutral>(rewriter, loc,
688  llvmType, accumulator);
689  Value vectorLength =
690  createVectorLengthValue(rewriter, loc, vectorOperand.getType());
691  return rewriter.create<LLVMVPRedIntrinOp>(loc, llvmType,
692  /*startValue=*/accumulator,
693  vectorOperand, mask, vectorLength);
694 }
695 
696 template <class LLVMIntVPRedIntrinOp, class IntReductionNeutral,
697  class LLVMFPVPRedIntrinOp, class FPReductionNeutral>
698 static Value lowerPredicatedReductionWithStartValue(
699  ConversionPatternRewriter &rewriter, Location loc, Type llvmType,
700  Value vectorOperand, Value accumulator, Value mask) {
701  if (llvmType.isIntOrIndex())
702  return lowerPredicatedReductionWithStartValue<LLVMIntVPRedIntrinOp,
703  IntReductionNeutral>(
704  rewriter, loc, llvmType, vectorOperand, accumulator, mask);
705 
706  // FP dispatch.
707  return lowerPredicatedReductionWithStartValue<LLVMFPVPRedIntrinOp,
708  FPReductionNeutral>(
709  rewriter, loc, llvmType, vectorOperand, accumulator, mask);
710 }
711 
712 /// Conversion pattern for all vector reductions.
713 class VectorReductionOpConversion
714  : public ConvertOpToLLVMPattern<vector::ReductionOp> {
715 public:
716  explicit VectorReductionOpConversion(const LLVMTypeConverter &typeConv,
717  bool reassociateFPRed)
718  : ConvertOpToLLVMPattern<vector::ReductionOp>(typeConv),
719  reassociateFPReductions(reassociateFPRed) {}
720 
722  matchAndRewrite(vector::ReductionOp reductionOp, OpAdaptor adaptor,
723  ConversionPatternRewriter &rewriter) const override {
724  auto kind = reductionOp.getKind();
725  Type eltType = reductionOp.getDest().getType();
726  Type llvmType = typeConverter->convertType(eltType);
727  Value operand = adaptor.getVector();
728  Value acc = adaptor.getAcc();
729  Location loc = reductionOp.getLoc();
730 
731  if (eltType.isIntOrIndex()) {
732  // Integer reductions: add/mul/min/max/and/or/xor.
733  Value result;
734  switch (kind) {
735  case vector::CombiningKind::ADD:
736  result =
737  createIntegerReductionArithmeticOpLowering<LLVM::vector_reduce_add,
738  LLVM::AddOp>(
739  rewriter, loc, llvmType, operand, acc);
740  break;
741  case vector::CombiningKind::MUL:
742  result =
743  createIntegerReductionArithmeticOpLowering<LLVM::vector_reduce_mul,
744  LLVM::MulOp>(
745  rewriter, loc, llvmType, operand, acc);
746  break;
748  result = createIntegerReductionComparisonOpLowering<
749  LLVM::vector_reduce_umin>(rewriter, loc, llvmType, operand, acc,
750  LLVM::ICmpPredicate::ule);
751  break;
752  case vector::CombiningKind::MINSI:
753  result = createIntegerReductionComparisonOpLowering<
754  LLVM::vector_reduce_smin>(rewriter, loc, llvmType, operand, acc,
755  LLVM::ICmpPredicate::sle);
756  break;
757  case vector::CombiningKind::MAXUI:
758  result = createIntegerReductionComparisonOpLowering<
759  LLVM::vector_reduce_umax>(rewriter, loc, llvmType, operand, acc,
760  LLVM::ICmpPredicate::uge);
761  break;
762  case vector::CombiningKind::MAXSI:
763  result = createIntegerReductionComparisonOpLowering<
764  LLVM::vector_reduce_smax>(rewriter, loc, llvmType, operand, acc,
765  LLVM::ICmpPredicate::sge);
766  break;
767  case vector::CombiningKind::AND:
768  result =
769  createIntegerReductionArithmeticOpLowering<LLVM::vector_reduce_and,
770  LLVM::AndOp>(
771  rewriter, loc, llvmType, operand, acc);
772  break;
773  case vector::CombiningKind::OR:
774  result =
775  createIntegerReductionArithmeticOpLowering<LLVM::vector_reduce_or,
776  LLVM::OrOp>(
777  rewriter, loc, llvmType, operand, acc);
778  break;
779  case vector::CombiningKind::XOR:
780  result =
781  createIntegerReductionArithmeticOpLowering<LLVM::vector_reduce_xor,
782  LLVM::XOrOp>(
783  rewriter, loc, llvmType, operand, acc);
784  break;
785  default:
786  return failure();
787  }
788  rewriter.replaceOp(reductionOp, result);
789 
790  return success();
791  }
792 
793  if (!isa<FloatType>(eltType))
794  return failure();
795 
796  arith::FastMathFlagsAttr fMFAttr = reductionOp.getFastMathFlagsAttr();
797  LLVM::FastmathFlagsAttr fmf = LLVM::FastmathFlagsAttr::get(
798  reductionOp.getContext(),
799  convertArithFastMathFlagsToLLVM(fMFAttr.getValue()));
801  reductionOp.getContext(),
802  fmf.getValue() | (reassociateFPReductions ? LLVM::FastmathFlags::reassoc
803  : LLVM::FastmathFlags::none));
804 
805  // Floating-point reductions: add/mul/min/max
806  Value result;
807  if (kind == vector::CombiningKind::ADD) {
808  result = lowerReductionWithStartValue<LLVM::vector_reduce_fadd,
809  ReductionNeutralZero>(
810  rewriter, loc, llvmType, operand, acc, fmf);
811  } else if (kind == vector::CombiningKind::MUL) {
812  result = lowerReductionWithStartValue<LLVM::vector_reduce_fmul,
813  ReductionNeutralFPOne>(
814  rewriter, loc, llvmType, operand, acc, fmf);
815  } else if (kind == vector::CombiningKind::MINIMUMF) {
816  result =
817  createFPReductionComparisonOpLowering<LLVM::vector_reduce_fminimum>(
818  rewriter, loc, llvmType, operand, acc, fmf);
819  } else if (kind == vector::CombiningKind::MAXIMUMF) {
820  result =
821  createFPReductionComparisonOpLowering<LLVM::vector_reduce_fmaximum>(
822  rewriter, loc, llvmType, operand, acc, fmf);
823  } else if (kind == vector::CombiningKind::MINNUMF) {
824  result = createFPReductionComparisonOpLowering<LLVM::vector_reduce_fmin>(
825  rewriter, loc, llvmType, operand, acc, fmf);
826  } else if (kind == vector::CombiningKind::MAXNUMF) {
827  result = createFPReductionComparisonOpLowering<LLVM::vector_reduce_fmax>(
828  rewriter, loc, llvmType, operand, acc, fmf);
829  } else
830  return failure();
831 
832  rewriter.replaceOp(reductionOp, result);
833  return success();
834  }
835 
836 private:
837  const bool reassociateFPReductions;
838 };
839 
840 /// Base class to convert a `vector.mask` operation while matching traits
841 /// of the maskable operation nested inside. A `VectorMaskOpConversionBase`
842 /// instance matches against a `vector.mask` operation. The `matchAndRewrite`
843 /// method performs a second match against the maskable operation `MaskedOp`.
844 /// Finally, it invokes the virtual method `matchAndRewriteMaskableOp` to be
845 /// implemented by the concrete conversion classes. This method can match
846 /// against specific traits of the `vector.mask` and the maskable operation. It
847 /// must replace the `vector.mask` operation.
848 template <class MaskedOp>
849 class VectorMaskOpConversionBase
850  : public ConvertOpToLLVMPattern<vector::MaskOp> {
851 public:
853 
855  matchAndRewrite(vector::MaskOp maskOp, OpAdaptor adaptor,
856  ConversionPatternRewriter &rewriter) const final {
857  // Match against the maskable operation kind.
858  auto maskedOp = llvm::dyn_cast_or_null<MaskedOp>(maskOp.getMaskableOp());
859  if (!maskedOp)
860  return failure();
861  return matchAndRewriteMaskableOp(maskOp, maskedOp, rewriter);
862  }
863 
864 protected:
865  virtual LogicalResult
866  matchAndRewriteMaskableOp(vector::MaskOp maskOp,
867  vector::MaskableOpInterface maskableOp,
868  ConversionPatternRewriter &rewriter) const = 0;
869 };
870 
871 class MaskedReductionOpConversion
872  : public VectorMaskOpConversionBase<vector::ReductionOp> {
873 
874 public:
875  using VectorMaskOpConversionBase<
876  vector::ReductionOp>::VectorMaskOpConversionBase;
877 
878  LogicalResult matchAndRewriteMaskableOp(
879  vector::MaskOp maskOp, MaskableOpInterface maskableOp,
880  ConversionPatternRewriter &rewriter) const override {
881  auto reductionOp = cast<ReductionOp>(maskableOp.getOperation());
882  auto kind = reductionOp.getKind();
883  Type eltType = reductionOp.getDest().getType();
884  Type llvmType = typeConverter->convertType(eltType);
885  Value operand = reductionOp.getVector();
886  Value acc = reductionOp.getAcc();
887  Location loc = reductionOp.getLoc();
888 
889  arith::FastMathFlagsAttr fMFAttr = reductionOp.getFastMathFlagsAttr();
890  LLVM::FastmathFlagsAttr fmf = LLVM::FastmathFlagsAttr::get(
891  reductionOp.getContext(),
892  convertArithFastMathFlagsToLLVM(fMFAttr.getValue()));
893 
894  Value result;
895  switch (kind) {
896  case vector::CombiningKind::ADD:
897  result = lowerPredicatedReductionWithStartValue<
898  LLVM::VPReduceAddOp, ReductionNeutralZero, LLVM::VPReduceFAddOp,
899  ReductionNeutralZero>(rewriter, loc, llvmType, operand, acc,
900  maskOp.getMask());
901  break;
902  case vector::CombiningKind::MUL:
903  result = lowerPredicatedReductionWithStartValue<
904  LLVM::VPReduceMulOp, ReductionNeutralIntOne, LLVM::VPReduceFMulOp,
905  ReductionNeutralFPOne>(rewriter, loc, llvmType, operand, acc,
906  maskOp.getMask());
907  break;
909  result = lowerPredicatedReductionWithStartValue<LLVM::VPReduceUMinOp,
910  ReductionNeutralUIntMax>(
911  rewriter, loc, llvmType, operand, acc, maskOp.getMask());
912  break;
913  case vector::CombiningKind::MINSI:
914  result = lowerPredicatedReductionWithStartValue<LLVM::VPReduceSMinOp,
915  ReductionNeutralSIntMax>(
916  rewriter, loc, llvmType, operand, acc, maskOp.getMask());
917  break;
918  case vector::CombiningKind::MAXUI:
919  result = lowerPredicatedReductionWithStartValue<LLVM::VPReduceUMaxOp,
920  ReductionNeutralUIntMin>(
921  rewriter, loc, llvmType, operand, acc, maskOp.getMask());
922  break;
923  case vector::CombiningKind::MAXSI:
924  result = lowerPredicatedReductionWithStartValue<LLVM::VPReduceSMaxOp,
925  ReductionNeutralSIntMin>(
926  rewriter, loc, llvmType, operand, acc, maskOp.getMask());
927  break;
928  case vector::CombiningKind::AND:
929  result = lowerPredicatedReductionWithStartValue<LLVM::VPReduceAndOp,
930  ReductionNeutralAllOnes>(
931  rewriter, loc, llvmType, operand, acc, maskOp.getMask());
932  break;
933  case vector::CombiningKind::OR:
934  result = lowerPredicatedReductionWithStartValue<LLVM::VPReduceOrOp,
935  ReductionNeutralZero>(
936  rewriter, loc, llvmType, operand, acc, maskOp.getMask());
937  break;
938  case vector::CombiningKind::XOR:
939  result = lowerPredicatedReductionWithStartValue<LLVM::VPReduceXorOp,
940  ReductionNeutralZero>(
941  rewriter, loc, llvmType, operand, acc, maskOp.getMask());
942  break;
943  case vector::CombiningKind::MINNUMF:
944  result = lowerPredicatedReductionWithStartValue<LLVM::VPReduceFMinOp,
945  ReductionNeutralFPMax>(
946  rewriter, loc, llvmType, operand, acc, maskOp.getMask());
947  break;
948  case vector::CombiningKind::MAXNUMF:
949  result = lowerPredicatedReductionWithStartValue<LLVM::VPReduceFMaxOp,
950  ReductionNeutralFPMin>(
951  rewriter, loc, llvmType, operand, acc, maskOp.getMask());
952  break;
953  case CombiningKind::MAXIMUMF:
954  result = lowerMaskedReductionWithRegular<LLVM::vector_reduce_fmaximum,
955  MaskNeutralFMaximum>(
956  rewriter, loc, llvmType, operand, acc, maskOp.getMask(), fmf);
957  break;
958  case CombiningKind::MINIMUMF:
959  result = lowerMaskedReductionWithRegular<LLVM::vector_reduce_fminimum,
960  MaskNeutralFMinimum>(
961  rewriter, loc, llvmType, operand, acc, maskOp.getMask(), fmf);
962  break;
963  }
964 
965  // Replace `vector.mask` operation altogether.
966  rewriter.replaceOp(maskOp, result);
967  return success();
968  }
969 };
970 
971 class VectorShuffleOpConversion
972  : public ConvertOpToLLVMPattern<vector::ShuffleOp> {
973 public:
975 
977  matchAndRewrite(vector::ShuffleOp shuffleOp, OpAdaptor adaptor,
978  ConversionPatternRewriter &rewriter) const override {
979  auto loc = shuffleOp->getLoc();
980  auto v1Type = shuffleOp.getV1VectorType();
981  auto v2Type = shuffleOp.getV2VectorType();
982  auto vectorType = shuffleOp.getResultVectorType();
983  Type llvmType = typeConverter->convertType(vectorType);
984  auto maskArrayAttr = shuffleOp.getMask();
985 
986  // Bail if result type cannot be lowered.
987  if (!llvmType)
988  return failure();
989 
990  // Get rank and dimension sizes.
991  int64_t rank = vectorType.getRank();
992 #ifndef NDEBUG
993  bool wellFormed0DCase =
994  v1Type.getRank() == 0 && v2Type.getRank() == 0 && rank == 1;
995  bool wellFormedNDCase =
996  v1Type.getRank() == rank && v2Type.getRank() == rank;
997  assert((wellFormed0DCase || wellFormedNDCase) && "op is not well-formed");
998 #endif
999 
1000  // For rank 0 and 1, where both operands have *exactly* the same vector
1001  // type, there is direct shuffle support in LLVM. Use it!
1002  if (rank <= 1 && v1Type == v2Type) {
1003  Value llvmShuffleOp = rewriter.create<LLVM::ShuffleVectorOp>(
1004  loc, adaptor.getV1(), adaptor.getV2(),
1005  LLVM::convertArrayToIndices<int32_t>(maskArrayAttr));
1006  rewriter.replaceOp(shuffleOp, llvmShuffleOp);
1007  return success();
1008  }
1009 
1010  // For all other cases, insert the individual values individually.
1011  int64_t v1Dim = v1Type.getDimSize(0);
1012  Type eltType;
1013  if (auto arrayType = dyn_cast<LLVM::LLVMArrayType>(llvmType))
1014  eltType = arrayType.getElementType();
1015  else
1016  eltType = cast<VectorType>(llvmType).getElementType();
1017  Value insert = rewriter.create<LLVM::UndefOp>(loc, llvmType);
1018  int64_t insPos = 0;
1019  for (const auto &en : llvm::enumerate(maskArrayAttr)) {
1020  int64_t extPos = cast<IntegerAttr>(en.value()).getInt();
1021  Value value = adaptor.getV1();
1022  if (extPos >= v1Dim) {
1023  extPos -= v1Dim;
1024  value = adaptor.getV2();
1025  }
1026  Value extract = extractOne(rewriter, *getTypeConverter(), loc, value,
1027  eltType, rank, extPos);
1028  insert = insertOne(rewriter, *getTypeConverter(), loc, insert, extract,
1029  llvmType, rank, insPos++);
1030  }
1031  rewriter.replaceOp(shuffleOp, insert);
1032  return success();
1033  }
1034 };
1035 
1036 class VectorExtractElementOpConversion
1037  : public ConvertOpToLLVMPattern<vector::ExtractElementOp> {
1038 public:
1039  using ConvertOpToLLVMPattern<
1040  vector::ExtractElementOp>::ConvertOpToLLVMPattern;
1041 
1043  matchAndRewrite(vector::ExtractElementOp extractEltOp, OpAdaptor adaptor,
1044  ConversionPatternRewriter &rewriter) const override {
1045  auto vectorType = extractEltOp.getSourceVectorType();
1046  auto llvmType = typeConverter->convertType(vectorType.getElementType());
1047 
1048  // Bail if result type cannot be lowered.
1049  if (!llvmType)
1050  return failure();
1051 
1052  if (vectorType.getRank() == 0) {
1053  Location loc = extractEltOp.getLoc();
1054  auto idxType = rewriter.getIndexType();
1055  auto zero = rewriter.create<LLVM::ConstantOp>(
1056  loc, typeConverter->convertType(idxType),
1057  rewriter.getIntegerAttr(idxType, 0));
1058  rewriter.replaceOpWithNewOp<LLVM::ExtractElementOp>(
1059  extractEltOp, llvmType, adaptor.getVector(), zero);
1060  return success();
1061  }
1062 
1063  rewriter.replaceOpWithNewOp<LLVM::ExtractElementOp>(
1064  extractEltOp, llvmType, adaptor.getVector(), adaptor.getPosition());
1065  return success();
1066  }
1067 };
1068 
1069 class VectorExtractOpConversion
1070  : public ConvertOpToLLVMPattern<vector::ExtractOp> {
1071 public:
1073 
1075  matchAndRewrite(vector::ExtractOp extractOp, OpAdaptor adaptor,
1076  ConversionPatternRewriter &rewriter) const override {
1077  auto loc = extractOp->getLoc();
1078  auto resultType = extractOp.getResult().getType();
1079  auto llvmResultType = typeConverter->convertType(resultType);
1080  // Bail if result type cannot be lowered.
1081  if (!llvmResultType)
1082  return failure();
1083 
1085  adaptor.getStaticPosition(), adaptor.getDynamicPosition(), rewriter);
1086 
1087  // Extract entire vector. Should be handled by folder, but just to be safe.
1088  ArrayRef<OpFoldResult> position(positionVec);
1089  if (position.empty()) {
1090  rewriter.replaceOp(extractOp, adaptor.getVector());
1091  return success();
1092  }
1093 
1094  // One-shot extraction of vector from array (only requires extractvalue).
1095  if (isa<VectorType>(resultType)) {
1096  if (extractOp.hasDynamicPosition())
1097  return failure();
1098 
1099  Value extracted = rewriter.create<LLVM::ExtractValueOp>(
1100  loc, adaptor.getVector(), getAsIntegers(position));
1101  rewriter.replaceOp(extractOp, extracted);
1102  return success();
1103  }
1104 
1105  // Potential extraction of 1-D vector from array.
1106  Value extracted = adaptor.getVector();
1107  if (position.size() > 1) {
1108  if (extractOp.hasDynamicPosition())
1109  return failure();
1110 
1111  SmallVector<int64_t> nMinusOnePosition =
1112  getAsIntegers(position.drop_back());
1113  extracted = rewriter.create<LLVM::ExtractValueOp>(loc, extracted,
1114  nMinusOnePosition);
1115  }
1116 
1117  Value lastPosition = getAsLLVMValue(rewriter, loc, position.back());
1118  // Remaining extraction of element from 1-D LLVM vector.
1119  rewriter.replaceOpWithNewOp<LLVM::ExtractElementOp>(extractOp, extracted,
1120  lastPosition);
1121  return success();
1122  }
1123 };
1124 
1125 /// Conversion pattern that turns a vector.fma on a 1-D vector
1126 /// into an llvm.intr.fmuladd. This is a trivial 1-1 conversion.
1127 /// This does not match vectors of n >= 2 rank.
1128 ///
1129 /// Example:
1130 /// ```
1131 /// vector.fma %a, %a, %a : vector<8xf32>
1132 /// ```
1133 /// is converted to:
1134 /// ```
1135 /// llvm.intr.fmuladd %va, %va, %va:
1136 /// (!llvm."<8 x f32>">, !llvm<"<8 x f32>">, !llvm<"<8 x f32>">)
1137 /// -> !llvm."<8 x f32>">
1138 /// ```
1139 class VectorFMAOp1DConversion : public ConvertOpToLLVMPattern<vector::FMAOp> {
1140 public:
1142 
1144  matchAndRewrite(vector::FMAOp fmaOp, OpAdaptor adaptor,
1145  ConversionPatternRewriter &rewriter) const override {
1146  VectorType vType = fmaOp.getVectorType();
1147  if (vType.getRank() > 1)
1148  return failure();
1149 
1150  rewriter.replaceOpWithNewOp<LLVM::FMulAddOp>(
1151  fmaOp, adaptor.getLhs(), adaptor.getRhs(), adaptor.getAcc());
1152  return success();
1153  }
1154 };
1155 
1156 class VectorInsertElementOpConversion
1157  : public ConvertOpToLLVMPattern<vector::InsertElementOp> {
1158 public:
1160 
1162  matchAndRewrite(vector::InsertElementOp insertEltOp, OpAdaptor adaptor,
1163  ConversionPatternRewriter &rewriter) const override {
1164  auto vectorType = insertEltOp.getDestVectorType();
1165  auto llvmType = typeConverter->convertType(vectorType);
1166 
1167  // Bail if result type cannot be lowered.
1168  if (!llvmType)
1169  return failure();
1170 
1171  if (vectorType.getRank() == 0) {
1172  Location loc = insertEltOp.getLoc();
1173  auto idxType = rewriter.getIndexType();
1174  auto zero = rewriter.create<LLVM::ConstantOp>(
1175  loc, typeConverter->convertType(idxType),
1176  rewriter.getIntegerAttr(idxType, 0));
1177  rewriter.replaceOpWithNewOp<LLVM::InsertElementOp>(
1178  insertEltOp, llvmType, adaptor.getDest(), adaptor.getSource(), zero);
1179  return success();
1180  }
1181 
1182  rewriter.replaceOpWithNewOp<LLVM::InsertElementOp>(
1183  insertEltOp, llvmType, adaptor.getDest(), adaptor.getSource(),
1184  adaptor.getPosition());
1185  return success();
1186  }
1187 };
1188 
1189 class VectorInsertOpConversion
1190  : public ConvertOpToLLVMPattern<vector::InsertOp> {
1191 public:
1193 
1195  matchAndRewrite(vector::InsertOp insertOp, OpAdaptor adaptor,
1196  ConversionPatternRewriter &rewriter) const override {
1197  auto loc = insertOp->getLoc();
1198  auto sourceType = insertOp.getSourceType();
1199  auto destVectorType = insertOp.getDestVectorType();
1200  auto llvmResultType = typeConverter->convertType(destVectorType);
1201  // Bail if result type cannot be lowered.
1202  if (!llvmResultType)
1203  return failure();
1204 
1206  adaptor.getStaticPosition(), adaptor.getDynamicPosition(), rewriter);
1207 
1208  // Overwrite entire vector with value. Should be handled by folder, but
1209  // just to be safe.
1210  ArrayRef<OpFoldResult> position(positionVec);
1211  if (position.empty()) {
1212  rewriter.replaceOp(insertOp, adaptor.getSource());
1213  return success();
1214  }
1215 
1216  // One-shot insertion of a vector into an array (only requires insertvalue).
1217  if (isa<VectorType>(sourceType)) {
1218  if (insertOp.hasDynamicPosition())
1219  return failure();
1220 
1221  Value inserted = rewriter.create<LLVM::InsertValueOp>(
1222  loc, adaptor.getDest(), adaptor.getSource(), getAsIntegers(position));
1223  rewriter.replaceOp(insertOp, inserted);
1224  return success();
1225  }
1226 
1227  // Potential extraction of 1-D vector from array.
1228  Value extracted = adaptor.getDest();
1229  auto oneDVectorType = destVectorType;
1230  if (position.size() > 1) {
1231  if (insertOp.hasDynamicPosition())
1232  return failure();
1233 
1234  oneDVectorType = reducedVectorTypeBack(destVectorType);
1235  extracted = rewriter.create<LLVM::ExtractValueOp>(
1236  loc, extracted, getAsIntegers(position.drop_back()));
1237  }
1238 
1239  // Insertion of an element into a 1-D LLVM vector.
1240  Value inserted = rewriter.create<LLVM::InsertElementOp>(
1241  loc, typeConverter->convertType(oneDVectorType), extracted,
1242  adaptor.getSource(), getAsLLVMValue(rewriter, loc, position.back()));
1243 
1244  // Potential insertion of resulting 1-D vector into array.
1245  if (position.size() > 1) {
1246  if (insertOp.hasDynamicPosition())
1247  return failure();
1248 
1249  inserted = rewriter.create<LLVM::InsertValueOp>(
1250  loc, adaptor.getDest(), inserted,
1251  getAsIntegers(position.drop_back()));
1252  }
1253 
1254  rewriter.replaceOp(insertOp, inserted);
1255  return success();
1256  }
1257 };
1258 
1259 /// Lower vector.scalable.insert ops to LLVM vector.insert
1260 struct VectorScalableInsertOpLowering
1261  : public ConvertOpToLLVMPattern<vector::ScalableInsertOp> {
1262  using ConvertOpToLLVMPattern<
1263  vector::ScalableInsertOp>::ConvertOpToLLVMPattern;
1264 
1266  matchAndRewrite(vector::ScalableInsertOp insOp, OpAdaptor adaptor,
1267  ConversionPatternRewriter &rewriter) const override {
1268  rewriter.replaceOpWithNewOp<LLVM::vector_insert>(
1269  insOp, adaptor.getDest(), adaptor.getSource(), adaptor.getPos());
1270  return success();
1271  }
1272 };
1273 
1274 /// Lower vector.scalable.extract ops to LLVM vector.extract
1275 struct VectorScalableExtractOpLowering
1276  : public ConvertOpToLLVMPattern<vector::ScalableExtractOp> {
1277  using ConvertOpToLLVMPattern<
1278  vector::ScalableExtractOp>::ConvertOpToLLVMPattern;
1279 
1281  matchAndRewrite(vector::ScalableExtractOp extOp, OpAdaptor adaptor,
1282  ConversionPatternRewriter &rewriter) const override {
1283  rewriter.replaceOpWithNewOp<LLVM::vector_extract>(
1284  extOp, typeConverter->convertType(extOp.getResultVectorType()),
1285  adaptor.getSource(), adaptor.getPos());
1286  return success();
1287  }
1288 };
1289 
1290 /// Rank reducing rewrite for n-D FMA into (n-1)-D FMA where n > 1.
1291 ///
1292 /// Example:
1293 /// ```
1294 /// %d = vector.fma %a, %b, %c : vector<2x4xf32>
1295 /// ```
1296 /// is rewritten into:
1297 /// ```
1298 /// %r = splat %f0: vector<2x4xf32>
1299 /// %va = vector.extractvalue %a[0] : vector<2x4xf32>
1300 /// %vb = vector.extractvalue %b[0] : vector<2x4xf32>
1301 /// %vc = vector.extractvalue %c[0] : vector<2x4xf32>
1302 /// %vd = vector.fma %va, %vb, %vc : vector<4xf32>
1303 /// %r2 = vector.insertvalue %vd, %r[0] : vector<4xf32> into vector<2x4xf32>
1304 /// %va2 = vector.extractvalue %a2[1] : vector<2x4xf32>
1305 /// %vb2 = vector.extractvalue %b2[1] : vector<2x4xf32>
1306 /// %vc2 = vector.extractvalue %c2[1] : vector<2x4xf32>
1307 /// %vd2 = vector.fma %va2, %vb2, %vc2 : vector<4xf32>
1308 /// %r3 = vector.insertvalue %vd2, %r2[1] : vector<4xf32> into vector<2x4xf32>
1309 /// // %r3 holds the final value.
1310 /// ```
1311 class VectorFMAOpNDRewritePattern : public OpRewritePattern<FMAOp> {
1312 public:
1314 
1315  void initialize() {
1316  // This pattern recursively unpacks one dimension at a time. The recursion
1317  // bounded as the rank is strictly decreasing.
1318  setHasBoundedRewriteRecursion();
1319  }
1320 
1321  LogicalResult matchAndRewrite(FMAOp op,
1322  PatternRewriter &rewriter) const override {
1323  auto vType = op.getVectorType();
1324  if (vType.getRank() < 2)
1325  return failure();
1326 
1327  auto loc = op.getLoc();
1328  auto elemType = vType.getElementType();
1329  Value zero = rewriter.create<arith::ConstantOp>(
1330  loc, elemType, rewriter.getZeroAttr(elemType));
1331  Value desc = rewriter.create<vector::SplatOp>(loc, vType, zero);
1332  for (int64_t i = 0, e = vType.getShape().front(); i != e; ++i) {
1333  Value extrLHS = rewriter.create<ExtractOp>(loc, op.getLhs(), i);
1334  Value extrRHS = rewriter.create<ExtractOp>(loc, op.getRhs(), i);
1335  Value extrACC = rewriter.create<ExtractOp>(loc, op.getAcc(), i);
1336  Value fma = rewriter.create<FMAOp>(loc, extrLHS, extrRHS, extrACC);
1337  desc = rewriter.create<InsertOp>(loc, fma, desc, i);
1338  }
1339  rewriter.replaceOp(op, desc);
1340  return success();
1341  }
1342 };
1343 
1344 /// Returns the strides if the memory underlying `memRefType` has a contiguous
1345 /// static layout.
1346 static std::optional<SmallVector<int64_t, 4>>
1347 computeContiguousStrides(MemRefType memRefType) {
1348  int64_t offset;
1349  SmallVector<int64_t, 4> strides;
1350  if (failed(getStridesAndOffset(memRefType, strides, offset)))
1351  return std::nullopt;
1352  if (!strides.empty() && strides.back() != 1)
1353  return std::nullopt;
1354  // If no layout or identity layout, this is contiguous by definition.
1355  if (memRefType.getLayout().isIdentity())
1356  return strides;
1357 
1358  // Otherwise, we must determine contiguity form shapes. This can only ever
1359  // work in static cases because MemRefType is underspecified to represent
1360  // contiguous dynamic shapes in other ways than with just empty/identity
1361  // layout.
1362  auto sizes = memRefType.getShape();
1363  for (int index = 0, e = strides.size() - 1; index < e; ++index) {
1364  if (ShapedType::isDynamic(sizes[index + 1]) ||
1365  ShapedType::isDynamic(strides[index]) ||
1366  ShapedType::isDynamic(strides[index + 1]))
1367  return std::nullopt;
1368  if (strides[index] != strides[index + 1] * sizes[index + 1])
1369  return std::nullopt;
1370  }
1371  return strides;
1372 }
1373 
1374 class VectorTypeCastOpConversion
1375  : public ConvertOpToLLVMPattern<vector::TypeCastOp> {
1376 public:
1378 
1380  matchAndRewrite(vector::TypeCastOp castOp, OpAdaptor adaptor,
1381  ConversionPatternRewriter &rewriter) const override {
1382  auto loc = castOp->getLoc();
1383  MemRefType sourceMemRefType =
1384  cast<MemRefType>(castOp.getOperand().getType());
1385  MemRefType targetMemRefType = castOp.getType();
1386 
1387  // Only static shape casts supported atm.
1388  if (!sourceMemRefType.hasStaticShape() ||
1389  !targetMemRefType.hasStaticShape())
1390  return failure();
1391 
1392  auto llvmSourceDescriptorTy =
1393  dyn_cast<LLVM::LLVMStructType>(adaptor.getOperands()[0].getType());
1394  if (!llvmSourceDescriptorTy)
1395  return failure();
1396  MemRefDescriptor sourceMemRef(adaptor.getOperands()[0]);
1397 
1398  auto llvmTargetDescriptorTy = dyn_cast_or_null<LLVM::LLVMStructType>(
1399  typeConverter->convertType(targetMemRefType));
1400  if (!llvmTargetDescriptorTy)
1401  return failure();
1402 
1403  // Only contiguous source buffers supported atm.
1404  auto sourceStrides = computeContiguousStrides(sourceMemRefType);
1405  if (!sourceStrides)
1406  return failure();
1407  auto targetStrides = computeContiguousStrides(targetMemRefType);
1408  if (!targetStrides)
1409  return failure();
1410  // Only support static strides for now, regardless of contiguity.
1411  if (llvm::any_of(*targetStrides, ShapedType::isDynamic))
1412  return failure();
1413 
1414  auto int64Ty = IntegerType::get(rewriter.getContext(), 64);
1415 
1416  // Create descriptor.
1417  auto desc = MemRefDescriptor::undef(rewriter, loc, llvmTargetDescriptorTy);
1418  // Set allocated ptr.
1419  Value allocated = sourceMemRef.allocatedPtr(rewriter, loc);
1420  desc.setAllocatedPtr(rewriter, loc, allocated);
1421 
1422  // Set aligned ptr.
1423  Value ptr = sourceMemRef.alignedPtr(rewriter, loc);
1424  desc.setAlignedPtr(rewriter, loc, ptr);
1425  // Fill offset 0.
1426  auto attr = rewriter.getIntegerAttr(rewriter.getIndexType(), 0);
1427  auto zero = rewriter.create<LLVM::ConstantOp>(loc, int64Ty, attr);
1428  desc.setOffset(rewriter, loc, zero);
1429 
1430  // Fill size and stride descriptors in memref.
1431  for (const auto &indexedSize :
1432  llvm::enumerate(targetMemRefType.getShape())) {
1433  int64_t index = indexedSize.index();
1434  auto sizeAttr =
1435  rewriter.getIntegerAttr(rewriter.getIndexType(), indexedSize.value());
1436  auto size = rewriter.create<LLVM::ConstantOp>(loc, int64Ty, sizeAttr);
1437  desc.setSize(rewriter, loc, index, size);
1438  auto strideAttr = rewriter.getIntegerAttr(rewriter.getIndexType(),
1439  (*targetStrides)[index]);
1440  auto stride = rewriter.create<LLVM::ConstantOp>(loc, int64Ty, strideAttr);
1441  desc.setStride(rewriter, loc, index, stride);
1442  }
1443 
1444  rewriter.replaceOp(castOp, {desc});
1445  return success();
1446  }
1447 };
1448 
1449 /// Conversion pattern for a `vector.create_mask` (1-D scalable vectors only).
1450 /// Non-scalable versions of this operation are handled in Vector Transforms.
1451 class VectorCreateMaskOpRewritePattern
1452  : public OpRewritePattern<vector::CreateMaskOp> {
1453 public:
1454  explicit VectorCreateMaskOpRewritePattern(MLIRContext *context,
1455  bool enableIndexOpt)
1456  : OpRewritePattern<vector::CreateMaskOp>(context),
1457  force32BitVectorIndices(enableIndexOpt) {}
1458 
1459  LogicalResult matchAndRewrite(vector::CreateMaskOp op,
1460  PatternRewriter &rewriter) const override {
1461  auto dstType = op.getType();
1462  if (dstType.getRank() != 1 || !cast<VectorType>(dstType).isScalable())
1463  return failure();
1464  IntegerType idxType =
1465  force32BitVectorIndices ? rewriter.getI32Type() : rewriter.getI64Type();
1466  auto loc = op->getLoc();
1467  Value indices = rewriter.create<LLVM::StepVectorOp>(
1468  loc, LLVM::getVectorType(idxType, dstType.getShape()[0],
1469  /*isScalable=*/true));
1470  auto bound = getValueOrCreateCastToIndexLike(rewriter, loc, idxType,
1471  op.getOperand(0));
1472  Value bounds = rewriter.create<SplatOp>(loc, indices.getType(), bound);
1473  Value comp = rewriter.create<arith::CmpIOp>(loc, arith::CmpIPredicate::slt,
1474  indices, bounds);
1475  rewriter.replaceOp(op, comp);
1476  return success();
1477  }
1478 
1479 private:
1480  const bool force32BitVectorIndices;
1481 };
1482 
1483 class VectorPrintOpConversion : public ConvertOpToLLVMPattern<vector::PrintOp> {
1484 public:
1486 
1487  // Lowering implementation that relies on a small runtime support library,
1488  // which only needs to provide a few printing methods (single value for all
1489  // data types, opening/closing bracket, comma, newline). The lowering splits
1490  // the vector into elementary printing operations. The advantage of this
1491  // approach is that the library can remain unaware of all low-level
1492  // implementation details of vectors while still supporting output of any
1493  // shaped and dimensioned vector.
1494  //
1495  // Note: This lowering only handles scalars, n-D vectors are broken into
1496  // printing scalars in loops in VectorToSCF.
1497  //
1498  // TODO: rely solely on libc in future? something else?
1499  //
1501  matchAndRewrite(vector::PrintOp printOp, OpAdaptor adaptor,
1502  ConversionPatternRewriter &rewriter) const override {
1503  auto parent = printOp->getParentOfType<ModuleOp>();
1504  if (!parent)
1505  return failure();
1506 
1507  auto loc = printOp->getLoc();
1508 
1509  if (auto value = adaptor.getSource()) {
1510  Type printType = printOp.getPrintType();
1511  if (isa<VectorType>(printType)) {
1512  // Vectors should be broken into elementary print ops in VectorToSCF.
1513  return failure();
1514  }
1515  if (failed(emitScalarPrint(rewriter, parent, loc, printType, value)))
1516  return failure();
1517  }
1518 
1519  auto punct = printOp.getPunctuation();
1520  if (auto stringLiteral = printOp.getStringLiteral()) {
1521  LLVM::createPrintStrCall(rewriter, loc, parent, "vector_print_str",
1522  *stringLiteral, *getTypeConverter(),
1523  /*addNewline=*/false);
1524  } else if (punct != PrintPunctuation::NoPunctuation) {
1525  emitCall(rewriter, printOp->getLoc(), [&] {
1526  switch (punct) {
1527  case PrintPunctuation::Close:
1528  return LLVM::lookupOrCreatePrintCloseFn(parent);
1529  case PrintPunctuation::Open:
1530  return LLVM::lookupOrCreatePrintOpenFn(parent);
1531  case PrintPunctuation::Comma:
1532  return LLVM::lookupOrCreatePrintCommaFn(parent);
1533  case PrintPunctuation::NewLine:
1534  return LLVM::lookupOrCreatePrintNewlineFn(parent);
1535  default:
1536  llvm_unreachable("unexpected punctuation");
1537  }
1538  }());
1539  }
1540 
1541  rewriter.eraseOp(printOp);
1542  return success();
1543  }
1544 
1545 private:
1546  enum class PrintConversion {
1547  // clang-format off
1548  None,
1549  ZeroExt64,
1550  SignExt64,
1551  Bitcast16
1552  // clang-format on
1553  };
1554 
1555  LogicalResult emitScalarPrint(ConversionPatternRewriter &rewriter,
1556  ModuleOp parent, Location loc, Type printType,
1557  Value value) const {
1558  if (typeConverter->convertType(printType) == nullptr)
1559  return failure();
1560 
1561  // Make sure element type has runtime support.
1562  PrintConversion conversion = PrintConversion::None;
1563  Operation *printer;
1564  if (printType.isF32()) {
1565  printer = LLVM::lookupOrCreatePrintF32Fn(parent);
1566  } else if (printType.isF64()) {
1567  printer = LLVM::lookupOrCreatePrintF64Fn(parent);
1568  } else if (printType.isF16()) {
1569  conversion = PrintConversion::Bitcast16; // bits!
1570  printer = LLVM::lookupOrCreatePrintF16Fn(parent);
1571  } else if (printType.isBF16()) {
1572  conversion = PrintConversion::Bitcast16; // bits!
1573  printer = LLVM::lookupOrCreatePrintBF16Fn(parent);
1574  } else if (printType.isIndex()) {
1575  printer = LLVM::lookupOrCreatePrintU64Fn(parent);
1576  } else if (auto intTy = dyn_cast<IntegerType>(printType)) {
1577  // Integers need a zero or sign extension on the operand
1578  // (depending on the source type) as well as a signed or
1579  // unsigned print method. Up to 64-bit is supported.
1580  unsigned width = intTy.getWidth();
1581  if (intTy.isUnsigned()) {
1582  if (width <= 64) {
1583  if (width < 64)
1584  conversion = PrintConversion::ZeroExt64;
1585  printer = LLVM::lookupOrCreatePrintU64Fn(parent);
1586  } else {
1587  return failure();
1588  }
1589  } else {
1590  assert(intTy.isSignless() || intTy.isSigned());
1591  if (width <= 64) {
1592  // Note that we *always* zero extend booleans (1-bit integers),
1593  // so that true/false is printed as 1/0 rather than -1/0.
1594  if (width == 1)
1595  conversion = PrintConversion::ZeroExt64;
1596  else if (width < 64)
1597  conversion = PrintConversion::SignExt64;
1598  printer = LLVM::lookupOrCreatePrintI64Fn(parent);
1599  } else {
1600  return failure();
1601  }
1602  }
1603  } else {
1604  return failure();
1605  }
1606 
1607  switch (conversion) {
1608  case PrintConversion::ZeroExt64:
1609  value = rewriter.create<arith::ExtUIOp>(
1610  loc, IntegerType::get(rewriter.getContext(), 64), value);
1611  break;
1612  case PrintConversion::SignExt64:
1613  value = rewriter.create<arith::ExtSIOp>(
1614  loc, IntegerType::get(rewriter.getContext(), 64), value);
1615  break;
1616  case PrintConversion::Bitcast16:
1617  value = rewriter.create<LLVM::BitcastOp>(
1618  loc, IntegerType::get(rewriter.getContext(), 16), value);
1619  break;
1620  case PrintConversion::None:
1621  break;
1622  }
1623  emitCall(rewriter, loc, printer, value);
1624  return success();
1625  }
1626 
1627  // Helper to emit a call.
1628  static void emitCall(ConversionPatternRewriter &rewriter, Location loc,
1629  Operation *ref, ValueRange params = ValueRange()) {
1630  rewriter.create<LLVM::CallOp>(loc, TypeRange(), SymbolRefAttr::get(ref),
1631  params);
1632  }
1633 };
1634 
1635 /// The Splat operation is lowered to an insertelement + a shufflevector
1636 /// operation. Splat to only 0-d and 1-d vector result types are lowered.
1637 struct VectorSplatOpLowering : public ConvertOpToLLVMPattern<vector::SplatOp> {
1639 
1641  matchAndRewrite(vector::SplatOp splatOp, OpAdaptor adaptor,
1642  ConversionPatternRewriter &rewriter) const override {
1643  VectorType resultType = cast<VectorType>(splatOp.getType());
1644  if (resultType.getRank() > 1)
1645  return failure();
1646 
1647  // First insert it into an undef vector so we can shuffle it.
1648  auto vectorType = typeConverter->convertType(splatOp.getType());
1649  Value undef = rewriter.create<LLVM::UndefOp>(splatOp.getLoc(), vectorType);
1650  auto zero = rewriter.create<LLVM::ConstantOp>(
1651  splatOp.getLoc(),
1652  typeConverter->convertType(rewriter.getIntegerType(32)),
1653  rewriter.getZeroAttr(rewriter.getIntegerType(32)));
1654 
1655  // For 0-d vector, we simply do `insertelement`.
1656  if (resultType.getRank() == 0) {
1657  rewriter.replaceOpWithNewOp<LLVM::InsertElementOp>(
1658  splatOp, vectorType, undef, adaptor.getInput(), zero);
1659  return success();
1660  }
1661 
1662  // For 1-d vector, we additionally do a `vectorshuffle`.
1663  auto v = rewriter.create<LLVM::InsertElementOp>(
1664  splatOp.getLoc(), vectorType, undef, adaptor.getInput(), zero);
1665 
1666  int64_t width = cast<VectorType>(splatOp.getType()).getDimSize(0);
1667  SmallVector<int32_t> zeroValues(width, 0);
1668 
1669  // Shuffle the value across the desired number of elements.
1670  rewriter.replaceOpWithNewOp<LLVM::ShuffleVectorOp>(splatOp, v, undef,
1671  zeroValues);
1672  return success();
1673  }
1674 };
1675 
1676 /// The Splat operation is lowered to an insertelement + a shufflevector
1677 /// operation. Splat to only 2+-d vector result types are lowered by the
1678 /// SplatNdOpLowering, the 1-d case is handled by SplatOpLowering.
1679 struct VectorSplatNdOpLowering : public ConvertOpToLLVMPattern<SplatOp> {
1681 
1683  matchAndRewrite(SplatOp splatOp, OpAdaptor adaptor,
1684  ConversionPatternRewriter &rewriter) const override {
1685  VectorType resultType = splatOp.getType();
1686  if (resultType.getRank() <= 1)
1687  return failure();
1688 
1689  // First insert it into an undef vector so we can shuffle it.
1690  auto loc = splatOp.getLoc();
1691  auto vectorTypeInfo =
1692  LLVM::detail::extractNDVectorTypeInfo(resultType, *getTypeConverter());
1693  auto llvmNDVectorTy = vectorTypeInfo.llvmNDVectorTy;
1694  auto llvm1DVectorTy = vectorTypeInfo.llvm1DVectorTy;
1695  if (!llvmNDVectorTy || !llvm1DVectorTy)
1696  return failure();
1697 
1698  // Construct returned value.
1699  Value desc = rewriter.create<LLVM::UndefOp>(loc, llvmNDVectorTy);
1700 
1701  // Construct a 1-D vector with the splatted value that we insert in all the
1702  // places within the returned descriptor.
1703  Value vdesc = rewriter.create<LLVM::UndefOp>(loc, llvm1DVectorTy);
1704  auto zero = rewriter.create<LLVM::ConstantOp>(
1705  loc, typeConverter->convertType(rewriter.getIntegerType(32)),
1706  rewriter.getZeroAttr(rewriter.getIntegerType(32)));
1707  Value v = rewriter.create<LLVM::InsertElementOp>(loc, llvm1DVectorTy, vdesc,
1708  adaptor.getInput(), zero);
1709 
1710  // Shuffle the value across the desired number of elements.
1711  int64_t width = resultType.getDimSize(resultType.getRank() - 1);
1712  SmallVector<int32_t> zeroValues(width, 0);
1713  v = rewriter.create<LLVM::ShuffleVectorOp>(loc, v, v, zeroValues);
1714 
1715  // Iterate of linear index, convert to coords space and insert splatted 1-D
1716  // vector in each position.
1717  nDVectorIterate(vectorTypeInfo, rewriter, [&](ArrayRef<int64_t> position) {
1718  desc = rewriter.create<LLVM::InsertValueOp>(loc, desc, v, position);
1719  });
1720  rewriter.replaceOp(splatOp, desc);
1721  return success();
1722  }
1723 };
1724 
1725 /// Conversion pattern for a `vector.interleave`.
1726 /// This supports fixed-sized vectors and scalable vectors.
1727 struct VectorInterleaveOpLowering
1728  : public ConvertOpToLLVMPattern<vector::InterleaveOp> {
1730 
1732  matchAndRewrite(vector::InterleaveOp interleaveOp, OpAdaptor adaptor,
1733  ConversionPatternRewriter &rewriter) const override {
1734  VectorType resultType = interleaveOp.getResultVectorType();
1735  // n-D interleaves should have been lowered already.
1736  if (resultType.getRank() != 1)
1737  return rewriter.notifyMatchFailure(interleaveOp,
1738  "InterleaveOp not rank 1");
1739  // If the result is rank 1, then this directly maps to LLVM.
1740  if (resultType.isScalable()) {
1741  rewriter.replaceOpWithNewOp<LLVM::vector_interleave2>(
1742  interleaveOp, typeConverter->convertType(resultType),
1743  adaptor.getLhs(), adaptor.getRhs());
1744  return success();
1745  }
1746  // Lower fixed-size interleaves to a shufflevector. While the
1747  // vector.interleave2 intrinsic supports fixed and scalable vectors, the
1748  // langref still recommends fixed-vectors use shufflevector, see:
1749  // https://llvm.org/docs/LangRef.html#id876.
1750  int64_t resultVectorSize = resultType.getNumElements();
1751  SmallVector<int32_t> interleaveShuffleMask;
1752  interleaveShuffleMask.reserve(resultVectorSize);
1753  for (int i = 0, end = resultVectorSize / 2; i < end; ++i) {
1754  interleaveShuffleMask.push_back(i);
1755  interleaveShuffleMask.push_back((resultVectorSize / 2) + i);
1756  }
1757  rewriter.replaceOpWithNewOp<LLVM::ShuffleVectorOp>(
1758  interleaveOp, adaptor.getLhs(), adaptor.getRhs(),
1759  interleaveShuffleMask);
1760  return success();
1761  }
1762 };
1763 
1764 } // namespace
1765 
1766 /// Populate the given list with patterns that convert from Vector to LLVM.
1768  LLVMTypeConverter &converter, RewritePatternSet &patterns,
1769  bool reassociateFPReductions, bool force32BitVectorIndices) {
1770  MLIRContext *ctx = converter.getDialect()->getContext();
1771  patterns.add<VectorFMAOpNDRewritePattern>(ctx);
1773  patterns.add<VectorReductionOpConversion>(converter, reassociateFPReductions);
1774  patterns.add<VectorCreateMaskOpRewritePattern>(ctx, force32BitVectorIndices);
1775  patterns.add<VectorBitCastOpConversion, VectorShuffleOpConversion,
1776  VectorExtractElementOpConversion, VectorExtractOpConversion,
1777  VectorFMAOp1DConversion, VectorInsertElementOpConversion,
1778  VectorInsertOpConversion, VectorPrintOpConversion,
1779  VectorTypeCastOpConversion, VectorScaleOpConversion,
1780  VectorLoadStoreConversion<vector::LoadOp>,
1781  VectorLoadStoreConversion<vector::MaskedLoadOp>,
1782  VectorLoadStoreConversion<vector::StoreOp>,
1783  VectorLoadStoreConversion<vector::MaskedStoreOp>,
1784  VectorGatherOpConversion, VectorScatterOpConversion,
1785  VectorExpandLoadOpConversion, VectorCompressStoreOpConversion,
1786  VectorSplatOpLowering, VectorSplatNdOpLowering,
1787  VectorScalableInsertOpLowering, VectorScalableExtractOpLowering,
1788  MaskedReductionOpConversion, VectorInterleaveOpLowering>(
1789  converter);
1790  // Transfer ops with rank > 1 are handled by VectorToSCF.
1791  populateVectorTransferLoweringPatterns(patterns, /*maxTransferRank=*/1);
1792 }
1793 
1795  LLVMTypeConverter &converter, RewritePatternSet &patterns) {
1796  patterns.add<VectorMatmulOpConversion>(converter);
1797  patterns.add<VectorFlatTransposeOpConversion>(converter);
1798 }
static Value getIndexedPtrs(ConversionPatternRewriter &rewriter, Location loc, const LLVMTypeConverter &typeConverter, MemRefType memRefType, Value llvmMemref, Value base, Value index, uint64_t vLen)
LogicalResult getMemRefAlignment(const LLVMTypeConverter &typeConverter, MemRefType memrefType, unsigned &align)
static Value extractOne(ConversionPatternRewriter &rewriter, const LLVMTypeConverter &typeConverter, Location loc, Value val, Type llvmType, int64_t rank, int64_t pos)
static Value insertOne(ConversionPatternRewriter &rewriter, const LLVMTypeConverter &typeConverter, Location loc, Value val1, Value val2, Type llvmType, int64_t rank, int64_t pos)
static Value getAsLLVMValue(OpBuilder &builder, Location loc, OpFoldResult foldResult)
Convert foldResult into a Value.
static LogicalResult isMemRefTypeSupported(MemRefType memRefType, const LLVMTypeConverter &converter)
static VectorType reducedVectorTypeBack(VectorType tp)
@ None
#define MINUI(lhs, rhs)
static void printOp(llvm::raw_ostream &os, Operation *op, OpPrintingFlags &flags)
Definition: Unit.cpp:19
Attributes are known-constant values of operations.
Definition: Attributes.h:25
IntegerAttr getI32IntegerAttr(int32_t value)
Definition: Builders.cpp:216
IntegerAttr getIntegerAttr(Type type, int64_t value)
Definition: Builders.cpp:238
FloatAttr getFloatAttr(Type type, double value)
Definition: Builders.cpp:261
IntegerType getI64Type()
Definition: Builders.cpp:85
IntegerType getI32Type()
Definition: Builders.cpp:83
IntegerType getIntegerType(unsigned width)
Definition: Builders.cpp:87
TypedAttr getZeroAttr(Type type)
Definition: Builders.cpp:331
MLIRContext * getContext() const
Definition: Builders.h:55
IndexType getIndexType()
Definition: Builders.cpp:71
This class implements a pattern rewriter for use with ConversionPatterns.
void replaceOp(Operation *op, ValueRange newValues) override
PatternRewriter hook for replacing an operation.
void eraseOp(Operation *op) override
PatternRewriter hook for erasing a dead operation.
Utility class for operation conversions targeting the LLVM dialect that match exactly one source oper...
Definition: Pattern.h:143
ConvertOpToLLVMPattern(const LLVMTypeConverter &typeConverter, PatternBenefit benefit=1)
Definition: Pattern.h:147
static DenseElementsAttr get(ShapedType type, ArrayRef< Attribute > values)
Constructs a dense elements attribute from an array of element values.
Conversion from types to the LLVM IR dialect.
Definition: TypeConverter.h:34
LLVM::LLVMDialect * getDialect() const
Returns the LLVM dialect.
Definition: TypeConverter.h:92
LogicalResult convertType(Type t, SmallVectorImpl< Type > &results) const
Convert the given type.
FailureOr< unsigned > getMemRefAddressSpace(BaseMemRefType type) const
Return the LLVM address space corresponding to the memory space of the memref type type or failure if...
const llvm::DataLayout & getDataLayout() const
Returns the data layout to use during and after conversion.
Utility class to translate MLIR LLVM dialect types to LLVM IR.
Definition: TypeToLLVM.h:39
unsigned getPreferredAlignment(Type type, const llvm::DataLayout &layout)
Returns the preferred alignment for the type given the data layout.
Definition: TypeToLLVM.cpp:196
This class defines the main interface for locations in MLIR and acts as a non-nullable wrapper around...
Definition: Location.h:63
MLIRContext is the top-level object for a collection of MLIR operations.
Definition: MLIRContext.h:60
Helper class to produce LLVM dialect operations extracting or inserting elements of a MemRef descript...
Definition: MemRefBuilder.h:33
static MemRefDescriptor undef(OpBuilder &builder, Location loc, Type descriptorType)
Builds IR creating an undef value of the descriptor type.
LLVM::LLVMPointerType getElementPtrType()
Returns the (LLVM) pointer type this descriptor contains.
Generic implementation of one-to-one conversion from "SourceOp" to "TargetOp" where the latter belong...
Definition: Pattern.h:198
This class helps build Operations.
Definition: Builders.h:209
Operation * create(const OperationState &state)
Creates an operation given the fields represented as an OperationState.
Definition: Builders.cpp:464
This class represents a single result from folding an operation.
Definition: OpDefinition.h:268
Operation is the basic unit of execution within MLIR.
Definition: Operation.h:88
Value getOperand(unsigned idx)
Definition: Operation.h:345
Location getLoc()
The source location the operation was defined or derived from.
Definition: Operation.h:223
A special type of RewriterBase that coordinates the application of a rewrite pattern on the current I...
Definition: PatternMatch.h:785
RewritePatternSet & add(ConstructorArg &&arg, ConstructorArgs &&...args)
Add an instance of each of the pattern types 'Ts' to the pattern list with the given arguments.
Definition: PatternMatch.h:846
std::enable_if_t<!std::is_convertible< CallbackT, Twine >::value, LogicalResult > notifyMatchFailure(Location loc, CallbackT &&reasonCallback)
Used to notify the listener that the IR failed to be rewritten because of a match failure,...
Definition: PatternMatch.h:718
virtual void replaceOp(Operation *op, ValueRange newValues)
Replace the results of the given (original) operation with the specified list of values (replacements...
OpTy replaceOpWithNewOp(Operation *op, Args &&...args)
Replace the results of the given (original) op with a new op that is created without verification (re...
Definition: PatternMatch.h:536
This class provides an abstraction over the various different ranges of value types.
Definition: TypeRange.h:36
Instances of the Type class are uniqued, have an immutable identifier and an optional mutable compone...
Definition: Types.h:74
bool isIntOrIndex() const
Return true if this is an integer (of any signedness) or an index type.
Definition: Types.cpp:115
unsigned getIntOrFloatBitWidth() const
Return the bit width of an integer or a float type, assert failure on other types.
Definition: Types.cpp:125
This class provides an abstraction over the different types of ranges over Values.
Definition: ValueRange.h:381
This class represents an instance of an SSA value in the MLIR system, representing a computable value...
Definition: Value.h:96
Type getType() const
Return the type of this value.
Definition: Value.h:129
LogicalResult handleMultidimensionalVectors(Operation *op, ValueRange operands, const LLVMTypeConverter &typeConverter, std::function< Value(Type, ValueRange)> createOperand, ConversionPatternRewriter &rewriter)
void printType(Type type, AsmPrinter &printer)
Prints an LLVM Dialect type.
void nDVectorIterate(const NDVectorTypeInfo &info, OpBuilder &builder, function_ref< void(ArrayRef< int64_t >)> fun)
NDVectorTypeInfo extractNDVectorTypeInfo(VectorType vectorType, const LLVMTypeConverter &converter)
Type getVectorType(Type elementType, unsigned numElements, bool isScalable=false)
Creates an LLVM dialect-compatible vector type with the given element type and length.
Definition: LLVMTypes.cpp:928
LLVM::LLVMFuncOp lookupOrCreatePrintF16Fn(ModuleOp moduleOp)
Type getFixedVectorType(Type elementType, unsigned numElements)
Creates an LLVM dialect-compatible type with the given element type and length.
Definition: LLVMTypes.cpp:955
LLVM::LLVMFuncOp lookupOrCreatePrintU64Fn(ModuleOp moduleOp)
void createPrintStrCall(OpBuilder &builder, Location loc, ModuleOp moduleOp, StringRef symbolName, StringRef string, const LLVMTypeConverter &typeConverter, bool addNewline=true, std::optional< StringRef > runtimeFunctionName={})
Generate IR that prints the given string to stdout.
LLVM::LLVMFuncOp lookupOrCreatePrintF64Fn(ModuleOp moduleOp)
LLVM::LLVMFuncOp lookupOrCreatePrintBF16Fn(ModuleOp moduleOp)
LLVM::LLVMFuncOp lookupOrCreatePrintF32Fn(ModuleOp moduleOp)
llvm::ElementCount getVectorNumElements(Type type)
Returns the element count of any LLVM-compatible vector type.
Definition: LLVMTypes.cpp:901
LLVM::LLVMFuncOp lookupOrCreatePrintI64Fn(ModuleOp moduleOp)
Helper functions to lookup or create the declaration for commonly used external C function calls.
LLVM::FastmathFlags convertArithFastMathFlagsToLLVM(arith::FastMathFlags arithFMF)
Maps arithmetic fastmath enum values to LLVM enum values.
constexpr void enumerate(std::tuple< Tys... > &tuple, CallbackT &&callback)
Definition: Matchers.h:285
void populateVectorTransferLoweringPatterns(RewritePatternSet &patterns, std::optional< unsigned > maxTransferRank=std::nullopt, PatternBenefit benefit=1)
Populate the pattern set with the following patterns:
SmallVector< int64_t > getAsIntegers(ArrayRef< Value > values)
Returns the integer numbers in values.
Definition: VectorOps.cpp:315
void populateVectorInsertExtractStridedSliceTransforms(RewritePatternSet &patterns, PatternBenefit benefit=1)
Populate patterns with the following patterns.
Include the generated interface declarations.
LogicalResult failure(bool isFailure=true)
Utility function to generate a LogicalResult.
Definition: LogicalResult.h:62
bool isLastMemrefDimUnitStride(MemRefType type)
Return "true" if the last dimension of the given type has a static unit stride.
LogicalResult getStridesAndOffset(MemRefType t, SmallVectorImpl< int64_t > &strides, int64_t &offset)
Returns the strides of the MemRef if the layout map is in strided form.
bool succeeded(LogicalResult result)
Utility function that returns true if the provided LogicalResult corresponds to a success value.
Definition: LogicalResult.h:68
LogicalResult success(bool isSuccess=true)
Utility function to generate a LogicalResult.
Definition: LogicalResult.h:56
void populateVectorToLLVMConversionPatterns(LLVMTypeConverter &converter, RewritePatternSet &patterns, bool reassociateFPReductions=false, bool force32BitVectorIndices=false)
Collect a set of patterns to convert from the Vector dialect to LLVM.
void populateVectorToLLVMMatrixConversionPatterns(LLVMTypeConverter &converter, RewritePatternSet &patterns)
Collect a set of patterns to convert from Vector contractions to LLVM Matrix Intrinsics.
Value getValueOrCreateCastToIndexLike(OpBuilder &b, Location loc, Type targetType, Value value)
Create a cast from an index-like value (index or integer) to another index-like value.
Definition: Utils.cpp:112
auto get(MLIRContext *context, Ts &&...params)
Helper method that injects context only if needed, this helps unify some of the attribute constructio...
SmallVector< OpFoldResult > getMixedValues(ArrayRef< int64_t > staticValues, ValueRange dynamicValues, Builder &b)
Return a vector of OpFoldResults with the same size a staticValues, but all elements for which Shaped...
bool failed(LogicalResult result)
Utility function that returns true if the provided LogicalResult corresponds to a failure value.
Definition: LogicalResult.h:72
This class represents an efficient way to signal success or failure.
Definition: LogicalResult.h:26
OpRewritePattern is a wrapper around RewritePattern that allows for matching and rewriting against an...
Definition: PatternMatch.h:358