MLIR 24.0.0git
ArithToLLVM.cpp
Go to the documentation of this file.
1//===- ArithToLLVM.cpp - Arithmetic to LLVM dialect conversion -------===//
2//
3// Part of the LLVM Project, under the Apache License v2.0 with LLVM Exceptions.
4// See https://llvm.org/LICENSE.txt for license information.
5// SPDX-License-Identifier: Apache-2.0 WITH LLVM-exception
6//
7//===----------------------------------------------------------------------===//
8
9#include "mlir/Conversion/ArithToLLVM/ArithToLLVM.h"
10
11#include "mlir/Conversion/ArithCommon/AttrToLLVMConverter.h"
12#include "mlir/Conversion/ConvertToLLVM/ToLLVMInterface.h"
13#include "mlir/Conversion/LLVMCommon/ConversionTarget.h"
14#include "mlir/Conversion/LLVMCommon/VectorPattern.h"
15#include "mlir/Dialect/Arith/IR/Arith.h"
16#include "mlir/Dialect/Arith/Transforms/Passes.h"
17#include "mlir/Dialect/LLVMIR/FunctionCallUtils.h"
18#include "mlir/Dialect/LLVMIR/LLVMAttrs.h"
19#include "mlir/Dialect/LLVMIR/LLVMDialect.h"
20#include "mlir/IR/DialectResourceBlobManager.h"
21#include "mlir/IR/TypeUtilities.h"
22#include <type_traits>
23
24namespace mlir {
25#define GEN_PASS_DEF_ARITHTOLLVMCONVERSIONPASS
26#include "mlir/Conversion/Passes.h.inc"
27} // namespace mlir
28
29using namespace mlir;
30
31namespace {
32
33/// Lowering pattern that matches only when the source op's rounding mode
34/// presence agrees with `HasRoundingMode`. This allows registering two
35/// instances of the same pattern for one source op: one that handles the
36/// unconstrained case (no rounding mode, lowering to a regular LLVM op) and
37/// one that handles the constrained case (rounding mode present, lowering to
38/// a constrained LLVM intrinsic).
39///
40/// * `HasRoundingMode`: the pattern matches if and only if the source op has
41/// a rounding mode attribute.
42/// * `AttrConvert`: attribute converter to translate source attributes to
43/// target attributes.
44/// * `FailOnUnsupportedFP`: whether to fail if the source op has unsupported
45/// floating point types.
46template <typename SourceOp, typename TargetOp, bool HasRoundingMode,
47 template <typename, typename> typename AttrConvert =
48 AttrConvertPassThrough,
49 bool FailOnUnsupportedFP = false>
50struct ConstrainedVectorConvertToLLVMPattern
51 : public VectorConvertToLLVMPattern<SourceOp, TargetOp, AttrConvert,
52 FailOnUnsupportedFP> {
53 using VectorConvertToLLVMPattern<
54 SourceOp, TargetOp, AttrConvert,
55 FailOnUnsupportedFP>::VectorConvertToLLVMPattern;
56
57 LogicalResult
58 matchAndRewrite(SourceOp op, typename SourceOp::Adaptor adaptor,
59 ConversionPatternRewriter &rewriter) const override {
60 if (HasRoundingMode != static_cast<bool>(op.getRoundingModeAttr()))
61 return failure();
62 return VectorConvertToLLVMPattern<
63 SourceOp, TargetOp, AttrConvert,
64 FailOnUnsupportedFP>::matchAndRewrite(op, adaptor, rewriter);
65 }
66};
67
68/// No-op bitcast. Propagate type input arg if converted source and dest types
69/// are the same.
70struct IdentityBitcastLowering final
71 : public OpConversionPattern<arith::BitcastOp> {
72 using Base::Base;
73
74 LogicalResult
75 matchAndRewrite(arith::BitcastOp op, OpAdaptor adaptor,
76 ConversionPatternRewriter &rewriter) const final {
77 Value src = adaptor.getIn();
78 Type resultType = getTypeConverter()->convertType(op.getType());
79 if (src.getType() != resultType)
80 return rewriter.notifyMatchFailure(op, "Types are different");
81
82 rewriter.replaceOp(op, src);
83 return success();
84 }
85};
86
87//===----------------------------------------------------------------------===//
88// Straightforward Op Lowerings
89//===----------------------------------------------------------------------===//
90
91using AddFOpLowering =
92 ConstrainedVectorConvertToLLVMPattern<arith::AddFOp, LLVM::FAddOp,
93 /*HasRoundingMode=*/false,
94 arith::AttrConvertFastMathToLLVM,
95 /*FailOnUnsupportedFP=*/true>;
96using ConstrainedAddFOpLowering = ConstrainedVectorConvertToLLVMPattern<
97 arith::AddFOp, LLVM::ConstrainedFAddIntr, /*HasRoundingMode=*/true,
98 arith::AttrConverterConstrainedFPToLLVM, /*FailOnUnsupportedFP=*/true>;
99using AddIOpLowering =
100 VectorConvertToLLVMPattern<arith::AddIOp, LLVM::AddOp,
101 arith::AttrConvertOverflowToLLVM>;
102using AndIOpLowering = VectorConvertToLLVMPattern<arith::AndIOp, LLVM::AndOp>;
103using BitcastOpLowering =
104 VectorConvertToLLVMPattern<arith::BitcastOp, LLVM::BitcastOp>;
105using DivFOpLowering =
106 ConstrainedVectorConvertToLLVMPattern<arith::DivFOp, LLVM::FDivOp,
107 /*HasRoundingMode=*/false,
108 arith::AttrConvertFastMathToLLVM,
109 /*FailOnUnsupportedFP=*/true>;
110using ConstrainedDivFOpLowering = ConstrainedVectorConvertToLLVMPattern<
111 arith::DivFOp, LLVM::ConstrainedFDivIntr, /*HasRoundingMode=*/true,
112 arith::AttrConverterConstrainedFPToLLVM, /*FailOnUnsupportedFP=*/true>;
113using DivSIOpLowering =
114 VectorConvertToLLVMPattern<arith::DivSIOp, LLVM::SDivOp>;
115using DivUIOpLowering =
116 VectorConvertToLLVMPattern<arith::DivUIOp, LLVM::UDivOp>;
117using ExtFOpLowering =
118 VectorConvertToLLVMPattern<arith::ExtFOp, LLVM::FPExtOp,
119 arith::AttrConvertFastMathToLLVM,
120 /*FailOnUnsupportedFP=*/true>;
121using ExtSIOpLowering =
122 VectorConvertToLLVMPattern<arith::ExtSIOp, LLVM::SExtOp>;
123using ExtUIOpLowering =
124 VectorConvertToLLVMPattern<arith::ExtUIOp, LLVM::ZExtOp,
125 arith::AttrConvertNonNegToLLVM>;
126using FPToSIOpLowering =
127 VectorConvertToLLVMPattern<arith::FPToSIOp, LLVM::FPToSIOp,
128 AttrConvertPassThrough,
129 /*FailOnUnsupportedFP=*/true>;
130using FPToUIOpLowering =
131 VectorConvertToLLVMPattern<arith::FPToUIOp, LLVM::FPToUIOp,
132 AttrConvertPassThrough,
133 /*FailOnUnsupportedFP=*/true>;
134using MaximumFOpLowering =
135 VectorConvertToLLVMPattern<arith::MaximumFOp, LLVM::MaximumOp,
136 arith::AttrConvertFastMathToLLVM,
137 /*FailOnUnsupportedFP=*/true>;
138using MaxNumFOpLowering =
139 VectorConvertToLLVMPattern<arith::MaxNumFOp, LLVM::MaxNumOp,
140 arith::AttrConvertFastMathToLLVM,
141 /*FailOnUnsupportedFP=*/true>;
142using MaxSIOpLowering =
143 VectorConvertToLLVMPattern<arith::MaxSIOp, LLVM::SMaxOp>;
144using MaxUIOpLowering =
145 VectorConvertToLLVMPattern<arith::MaxUIOp, LLVM::UMaxOp>;
146using MinimumFOpLowering =
147 VectorConvertToLLVMPattern<arith::MinimumFOp, LLVM::MinimumOp,
148 arith::AttrConvertFastMathToLLVM,
149 /*FailOnUnsupportedFP=*/true>;
150using MinNumFOpLowering =
151 VectorConvertToLLVMPattern<arith::MinNumFOp, LLVM::MinNumOp,
152 arith::AttrConvertFastMathToLLVM,
153 /*FailOnUnsupportedFP=*/true>;
154using MinSIOpLowering =
155 VectorConvertToLLVMPattern<arith::MinSIOp, LLVM::SMinOp>;
156using MinUIOpLowering =
157 VectorConvertToLLVMPattern<arith::MinUIOp, LLVM::UMinOp>;
158using MulFOpLowering =
159 ConstrainedVectorConvertToLLVMPattern<arith::MulFOp, LLVM::FMulOp,
160 /*HasRoundingMode=*/false,
161 arith::AttrConvertFastMathToLLVM,
162 /*FailOnUnsupportedFP=*/true>;
163using ConstrainedMulFOpLowering = ConstrainedVectorConvertToLLVMPattern<
164 arith::MulFOp, LLVM::ConstrainedFMulIntr, /*HasRoundingMode=*/true,
165 arith::AttrConverterConstrainedFPToLLVM, /*FailOnUnsupportedFP=*/true>;
166using MulIOpLowering =
167 VectorConvertToLLVMPattern<arith::MulIOp, LLVM::MulOp,
168 arith::AttrConvertOverflowToLLVM>;
169using NegFOpLowering =
170 VectorConvertToLLVMPattern<arith::NegFOp, LLVM::FNegOp,
171 arith::AttrConvertFastMathToLLVM,
172 /*FailOnUnsupportedFP=*/true>;
173using OrIOpLowering = VectorConvertToLLVMPattern<arith::OrIOp, LLVM::OrOp>;
174using RemFOpLowering =
175 VectorConvertToLLVMPattern<arith::RemFOp, LLVM::FRemOp,
176 arith::AttrConvertFastMathToLLVM,
177 /*FailOnUnsupportedFP=*/true>;
178using RemSIOpLowering =
179 VectorConvertToLLVMPattern<arith::RemSIOp, LLVM::SRemOp>;
180using RemUIOpLowering =
181 VectorConvertToLLVMPattern<arith::RemUIOp, LLVM::URemOp>;
182using SelectOpLowering =
183 VectorConvertToLLVMPattern<arith::SelectOp, LLVM::SelectOp>;
184using ShLIOpLowering =
185 VectorConvertToLLVMPattern<arith::ShLIOp, LLVM::ShlOp,
186 arith::AttrConvertOverflowToLLVM>;
187using ShRSIOpLowering =
188 VectorConvertToLLVMPattern<arith::ShRSIOp, LLVM::AShrOp>;
189using ShRUIOpLowering =
190 VectorConvertToLLVMPattern<arith::ShRUIOp, LLVM::LShrOp>;
191using SIToFPOpLowering =
192 VectorConvertToLLVMPattern<arith::SIToFPOp, LLVM::SIToFPOp>;
193using SubFOpLowering =
194 ConstrainedVectorConvertToLLVMPattern<arith::SubFOp, LLVM::FSubOp,
195 /*HasRoundingMode=*/false,
196 arith::AttrConvertFastMathToLLVM,
197 /*FailOnUnsupportedFP=*/true>;
198using ConstrainedSubFOpLowering = ConstrainedVectorConvertToLLVMPattern<
199 arith::SubFOp, LLVM::ConstrainedFSubIntr, /*HasRoundingMode=*/true,
200 arith::AttrConverterConstrainedFPToLLVM, /*FailOnUnsupportedFP=*/true>;
201using SubIOpLowering =
202 VectorConvertToLLVMPattern<arith::SubIOp, LLVM::SubOp,
203 arith::AttrConvertOverflowToLLVM>;
204using TruncFOpLowering =
205 ConstrainedVectorConvertToLLVMPattern<arith::TruncFOp, LLVM::FPTruncOp,
206 /*HasRoundingMode=*/false,
207 arith::AttrConvertFastMathToLLVM,
208 /*FailOnUnsupportedFP=*/true>;
209using ConstrainedTruncFOpLowering = ConstrainedVectorConvertToLLVMPattern<
210 arith::TruncFOp, LLVM::ConstrainedFPTruncIntr, /*HasRoundingMode=*/true,
211 arith::AttrConverterConstrainedFPToLLVM, /*FailOnUnsupportedFP=*/true>;
212using TruncIOpLowering =
213 VectorConvertToLLVMPattern<arith::TruncIOp, LLVM::TruncOp,
214 arith::AttrConvertOverflowToLLVM>;
215using UIToFPOpLowering =
216 VectorConvertToLLVMPattern<arith::UIToFPOp, LLVM::UIToFPOp,
217 arith::AttrConvertNonNegToLLVM,
218 /*FailOnUnsupportedFP=*/true>;
219using XOrIOpLowering = VectorConvertToLLVMPattern<arith::XOrIOp, LLVM::XOrOp>;
220
221//===----------------------------------------------------------------------===//
222// Op Lowering Patterns
223//===----------------------------------------------------------------------===//
224
225/// Directly lower to LLVM op.
226struct ConstantOpLowering : public ConvertOpToLLVMPattern<arith::ConstantOp> {
227 using ConvertOpToLLVMPattern::ConvertOpToLLVMPattern;
228
229 LogicalResult
230 matchAndRewrite(arith::ConstantOp op, OpAdaptor adaptor,
231 ConversionPatternRewriter &rewriter) const override;
232};
233
234/// The lowering of index_cast becomes an integer conversion since index
235/// becomes an integer. If the bit width of the source and target integer
236/// types is the same, just erase the cast. If the target type is wider,
237/// sign-extend the value, otherwise truncate it.
238template <typename OpTy, typename ExtCastTy>
239struct IndexCastOpLowering : public ConvertOpToLLVMPattern<OpTy> {
240 using ConvertOpToLLVMPattern<OpTy>::ConvertOpToLLVMPattern;
241
242 LogicalResult
243 matchAndRewrite(OpTy op, typename OpTy::Adaptor adaptor,
244 ConversionPatternRewriter &rewriter) const override;
245};
246
247using IndexCastOpSILowering =
248 IndexCastOpLowering<arith::IndexCastOp, LLVM::SExtOp>;
249using IndexCastOpUILowering =
250 IndexCastOpLowering<arith::IndexCastUIOp, LLVM::ZExtOp>;
251
252struct AddUIExtendedOpLowering
253 : public ConvertOpToLLVMPattern<arith::AddUIExtendedOp> {
254 using ConvertOpToLLVMPattern::ConvertOpToLLVMPattern;
255
256 LogicalResult
257 matchAndRewrite(arith::AddUIExtendedOp op, OpAdaptor adaptor,
258 ConversionPatternRewriter &rewriter) const override;
259};
260
261struct SubUIExtendedOpLowering
262 : public ConvertOpToLLVMPattern<arith::SubUIExtendedOp> {
263 using ConvertOpToLLVMPattern::ConvertOpToLLVMPattern;
264
265 LogicalResult
266 matchAndRewrite(arith::SubUIExtendedOp op, OpAdaptor adaptor,
267 ConversionPatternRewriter &rewriter) const override;
268};
269
270template <typename ArithMulOp, bool IsSigned>
271struct MulIExtendedOpLowering : public ConvertOpToLLVMPattern<ArithMulOp> {
272 using ConvertOpToLLVMPattern<ArithMulOp>::ConvertOpToLLVMPattern;
273
274 LogicalResult
275 matchAndRewrite(ArithMulOp op, typename ArithMulOp::Adaptor adaptor,
276 ConversionPatternRewriter &rewriter) const override;
277};
278
279using MulSIExtendedOpLowering =
280 MulIExtendedOpLowering<arith::MulSIExtendedOp, true>;
281using MulUIExtendedOpLowering =
282 MulIExtendedOpLowering<arith::MulUIExtendedOp, false>;
283
284struct CmpIOpLowering : public ConvertOpToLLVMPattern<arith::CmpIOp> {
285 using ConvertOpToLLVMPattern::ConvertOpToLLVMPattern;
286
287 LogicalResult
288 matchAndRewrite(arith::CmpIOp op, OpAdaptor adaptor,
289 ConversionPatternRewriter &rewriter) const override;
290};
291
292struct CmpFOpLowering : public ConvertOpToLLVMPattern<arith::CmpFOp> {
293 using ConvertOpToLLVMPattern::ConvertOpToLLVMPattern;
294
295 LogicalResult
296 matchAndRewrite(arith::CmpFOp op, OpAdaptor adaptor,
297 ConversionPatternRewriter &rewriter) const override;
298};
299
300/// Lower arith.convertf (same-bitwidth FP cast) to LLVM.
301///
302/// Extends to f32 via llvm.fpext, then truncates to the target type via
303/// llvm.fptrunc. This handles bf16 <-> f16, which is the only same-bitwidth
304/// pair of LLVM-supported FP types.
305struct ConvertFOpLowering : public ConvertOpToLLVMPattern<arith::ConvertFOp> {
306 using ConvertOpToLLVMPattern::ConvertOpToLLVMPattern;
307
308 LogicalResult
309 matchAndRewrite(arith::ConvertFOp op, OpAdaptor adaptor,
310 ConversionPatternRewriter &rewriter) const override {
311 if (LLVM::detail::opHasUnsupportedFloatingPointTypes(op,
312 *getTypeConverter()))
313 return rewriter.notifyMatchFailure(op, "unsupported floating point type");
314
315 // Only bf16 <-> f16 conversions are supported. There is currently no other
316 // pair of FP types that are valid LLVM types.
317 [[maybe_unused]] auto srcType = getElementTypeOrSelf(op.getIn().getType());
318 [[maybe_unused]] auto dstType = getElementTypeOrSelf(op.getType());
319 assert(((srcType.isBF16() && dstType.isF16()) ||
320 (srcType.isF16() && dstType.isBF16())) &&
321 "only bf16 <-> f16 conversions are supported");
322
323 Type convertedType = getTypeConverter()->convertType(op.getType());
324 if (!convertedType)
325 return rewriter.notifyMatchFailure(op, "failed to convert result type");
326
327 Value input = adaptor.getIn();
328 Location loc = op.getLoc();
329
330 if (!isa<LLVM::LLVMArrayType>(input.getType())) {
331 rewriter.replaceOp(op,
332 emitConversion(rewriter, loc, input, convertedType));
333 return success();
334 }
335
336 if (!isa<VectorType>(op.getType()))
337 return rewriter.notifyMatchFailure(op, "expected vector result type");
338
339 return LLVM::detail::handleMultidimensionalVectors(
340 op.getOperation(), adaptor.getOperands(), *getTypeConverter(),
341 [&](Type llvm1DVectorTy, ValueRange operands) -> Value {
342 return emitConversion(rewriter, loc, operands.front(),
343 llvm1DVectorTy);
344 },
345 rewriter);
346 }
347
348private:
349 static Value emitConversion(ConversionPatternRewriter &rewriter, Location loc,
350 Value input, Type targetType) {
351 Type f32Scalar = Float32Type::get(rewriter.getContext());
352 Type f32Ty = f32Scalar;
353 if (auto vecTy = dyn_cast<VectorType>(targetType))
354 f32Ty = VectorType::get(vecTy.getShape(), f32Scalar);
355
356 Value ext = LLVM::FPExtOp::create(rewriter, loc, f32Ty, input);
357 return LLVM::FPTruncOp::create(rewriter, loc, targetType, ext);
358 }
359};
360
361struct SelectOpOneToNLowering : public ConvertOpToLLVMPattern<arith::SelectOp> {
362 using ConvertOpToLLVMPattern::ConvertOpToLLVMPattern;
363 using Adaptor = ConvertOpToLLVMPattern<arith::SelectOp>::OneToNOpAdaptor;
364
365 LogicalResult
366 matchAndRewrite(arith::SelectOp op, Adaptor adaptor,
367 ConversionPatternRewriter &rewriter) const override;
368};
369
370} // namespace
371
372//===----------------------------------------------------------------------===//
373// ConstantOpLowering
374//===----------------------------------------------------------------------===//
375
376/// Retypes `attr` for a `llvm.mlir.constant` of `resultType`. `arith.constant`
377/// requires the value attribute and the result to have the same type, but the
378/// type converter may map the element type to a different one, e.g. `index` to
379/// `i32` or `i64` depending on the configured index bitwidth. Build the
380/// attribute from the converted type so that the two agree wherever that is
381/// representable. Returns a null attribute if `attr` cannot be used for
382/// `resultType`.
383static TypedAttr convertConstantValue(TypedAttr attr, Type resultType) {
384 // Compare the element types, but explicitly ignore the non-scalar portions of
385 // types. This relaxation is required to support multi-dimensional vector
386 // constant that result in nested LLVM array values.
387 Type sourceElementType = LLVM::getConstantElementType(attr.getType());
388 Type targetElementType = LLVM::getConstantElementType(resultType);
389 if (sourceElementType == targetElementType)
390 return attr;
391
392 auto targetIntType = dyn_cast<IntegerType>(targetElementType);
393 if (!targetIntType)
394 return {};
395
396 // The converter maps the low-precision float types that have no LLVM
397 // equivalent to an integer of the same width. The attribute stays a float
398 // attribute in that case.
399 if (auto sourceFloatType = dyn_cast<FloatType>(sourceElementType)) {
400 if (sourceFloatType.getWidth() != targetIntType.getWidth())
401 return {};
402 return attr;
403 }
404
405 // Apart from those floats, `index` is the only element type the converter
406 // rewrites, so anything else is a malformed `arith.constant`. Bail out rather
407 // than reinterpret its value, which would silently change the constant (e.g.
408 // sign-extending an `i1` `true` to -1).
409 if (!isa<IndexType>(sourceElementType))
410 return {};
411 // `index` is signless but holds signed values, so narrowing to a smaller
412 // index bitwidth truncates and widening sign-extends.
413 unsigned width = targetIntType.getWidth();
414
415 if (auto intAttr = dyn_cast<IntegerAttr>(attr))
416 return IntegerAttr::get(targetIntType,
417 intAttr.getValue().sextOrTrunc(width));
418
419 auto retypeValues = [&](DenseIntElementsAttr values) {
420 return values.mapValues(targetIntType, [&](const APInt &value) {
421 return value.sextOrTrunc(width);
422 });
423 };
424
425 if (auto denseAttr = dyn_cast<DenseIntElementsAttr>(attr))
426 return retypeValues(denseAttr);
427
428 if (auto sparseAttr = dyn_cast<SparseElementsAttr>(attr))
429 return SparseElementsAttr::get(
430 cast<ShapedType>(attr.getType()).clone(targetIntType),
431 sparseAttr.getIndices(),
432 retypeValues(cast<DenseIntElementsAttr>(sparseAttr.getValues())));
433
434 // A resource-backed elements attribute refers to a blob laid out for its own
435 // element type. The blob cannot be rewritten here, only reinterpreted, which
436 // is correct exactly when the target type has the same width as the storage
437 // `index` uses in a blob.
438 if (auto resourceAttr = dyn_cast<DenseResourceElementsAttr>(attr)) {
439 if (width != IndexType::kInternalStorageBitWidth)
440 return {};
441 return DenseResourceElementsAttr::get(
442 cast<ShapedType>(attr.getType()).clone(targetIntType),
443 resourceAttr.getRawHandle());
444 }
445
446 return {};
447}
448
449LogicalResult
450ConstantOpLowering::matchAndRewrite(arith::ConstantOp op, OpAdaptor adaptor,
451 ConversionPatternRewriter &rewriter) const {
452 Type resultType = getTypeConverter()->convertType(op.getType());
453 if (!resultType)
454 return rewriter.notifyMatchFailure(op, "failed to convert result type");
455
456 TypedAttr value = convertConstantValue(op.getValue(), resultType);
457 if (!value)
458 return rewriter.notifyMatchFailure(
459 op, "failed to convert value attribute to the converted result type");
460
461 // `arith.constant` has no operands and a single result, so there is nothing
462 // for `oneToOneRewrite` to do here beyond converting `resultType` a second
463 // time.
464 DictionaryAttr discardableAttrs = op->getDiscardableAttrDictionary();
465 auto constantOp =
466 LLVM::ConstantOp::create(rewriter, op.getLoc(), resultType, value);
467 constantOp->setDiscardableAttrs(discardableAttrs);
468 rewriter.replaceOp(op, constantOp);
469 return success();
470}
471
472//===----------------------------------------------------------------------===//
473// IndexCastOpLowering
474//===----------------------------------------------------------------------===//
475
476template <typename OpTy, typename ExtCastTy>
477LogicalResult IndexCastOpLowering<OpTy, ExtCastTy>::matchAndRewrite(
478 OpTy op, typename OpTy::Adaptor adaptor,
479 ConversionPatternRewriter &rewriter) const {
480 Type resultType = op.getResult().getType();
481 Type targetElementType =
482 this->typeConverter->convertType(getElementTypeOrSelf(resultType));
483 Type sourceElementType =
484 this->typeConverter->convertType(getElementTypeOrSelf(op.getIn()));
485 unsigned targetBits = targetElementType.getIntOrFloatBitWidth();
486 unsigned sourceBits = sourceElementType.getIntOrFloatBitWidth();
487
488 if (targetBits == sourceBits) {
489 rewriter.replaceOp(op, adaptor.getIn());
490 return success();
491 }
492
493 // Memref index_cast is a no-op at the LLVM level since LLVM uses opaque
494 // pointers and memrefs of different integer/index element types all convert
495 // to the same LLVM struct type.
496 if (isa<MemRefType>(op.getIn().getType())) {
497 rewriter.replaceOp(op, adaptor.getIn());
498 return success();
499 }
500
501 bool isNonNeg = false;
502 if constexpr (std::is_same_v<ExtCastTy, LLVM::ZExtOp>)
503 isNonNeg = op.getNonNeg();
504
505 // Handle the scalar and 1D vector cases.
506 Type operandType = adaptor.getIn().getType();
507 if (!isa<LLVM::LLVMArrayType>(operandType)) {
508 Type targetType = this->typeConverter->convertType(resultType);
509 if (targetBits < sourceBits) {
510 rewriter.replaceOpWithNewOp<LLVM::TruncOp>(op, targetType,
511 adaptor.getIn());
512 } else {
513 auto extOp = rewriter.replaceOpWithNewOp<ExtCastTy>(op, targetType,
514 adaptor.getIn());
515 if constexpr (std::is_same_v<ExtCastTy, LLVM::ZExtOp>)
516 extOp.setNonNeg(isNonNeg);
517 }
518 return success();
519 }
520
521 if (!isa<VectorType>(resultType))
522 return rewriter.notifyMatchFailure(op, "expected vector result type");
523
524 return LLVM::detail::handleMultidimensionalVectors(
525 op.getOperation(), adaptor.getOperands(), *(this->getTypeConverter()),
526 [&](Type llvm1DVectorTy, ValueRange operands) -> Value {
527 typename OpTy::Adaptor adaptor(operands);
528 if (targetBits < sourceBits) {
529 return LLVM::TruncOp::create(rewriter, op.getLoc(), llvm1DVectorTy,
530 adaptor.getIn());
531 }
532 auto extOp = ExtCastTy::create(rewriter, op.getLoc(), llvm1DVectorTy,
533 adaptor.getIn());
534 if constexpr (std::is_same_v<ExtCastTy, LLVM::ZExtOp>) {
535 if (isNonNeg)
536 extOp.setNonNeg(true);
537 }
538 return extOp;
539 },
540 rewriter);
541}
542
543//===----------------------------------------------------------------------===//
544// AddUIExtendedOpLowering
545//===----------------------------------------------------------------------===//
546
547LogicalResult AddUIExtendedOpLowering::matchAndRewrite(
548 arith::AddUIExtendedOp op, OpAdaptor adaptor,
549 ConversionPatternRewriter &rewriter) const {
550 Type operandType = adaptor.getLhs().getType();
551 Type sumResultType = op.getSum().getType();
552 Type overflowResultType = op.getOverflow().getType();
553
554 if (!LLVM::isCompatibleType(operandType))
555 return failure();
556
557 MLIRContext *ctx = rewriter.getContext();
558 Location loc = op.getLoc();
559
560 // Handle the scalar and 1D vector cases.
561 if (!isa<LLVM::LLVMArrayType>(operandType)) {
562 Type newOverflowType = typeConverter->convertType(overflowResultType);
563 Type structType =
564 LLVM::LLVMStructType::getLiteral(ctx, {sumResultType, newOverflowType});
565 Value addOverflow = LLVM::UAddWithOverflowOp::create(
566 rewriter, loc, structType, adaptor.getLhs(), adaptor.getRhs());
567 Value sumExtracted =
568 LLVM::ExtractValueOp::create(rewriter, loc, addOverflow, 0);
569 Value overflowExtracted =
570 LLVM::ExtractValueOp::create(rewriter, loc, addOverflow, 1);
571 rewriter.replaceOp(op, {sumExtracted, overflowExtracted});
572 return success();
573 }
574
575 if (!isa<VectorType>(sumResultType))
576 return rewriter.notifyMatchFailure(loc, "expected vector result types");
577
578 return rewriter.notifyMatchFailure(loc,
579 "ND vector types are not supported yet");
580}
581
582//===----------------------------------------------------------------------===//
583// SubUIExtendedOpLowering
584//===----------------------------------------------------------------------===//
585
586LogicalResult SubUIExtendedOpLowering::matchAndRewrite(
587 arith::SubUIExtendedOp op, OpAdaptor adaptor,
588 ConversionPatternRewriter &rewriter) const {
589 Type operandType = adaptor.getLhs().getType();
590 Type diffResultType = op.getDiff().getType();
591 Type borrowResultType = op.getBorrow().getType();
592
593 if (!LLVM::isCompatibleType(operandType))
594 return failure();
595
596 MLIRContext *ctx = rewriter.getContext();
597 Location loc = op.getLoc();
598
599 // Handle the scalar and 1D vector cases.
600 if (!isa<LLVM::LLVMArrayType>(operandType)) {
601 Type newBorrowType = typeConverter->convertType(borrowResultType);
602 Type structType =
603 LLVM::LLVMStructType::getLiteral(ctx, {diffResultType, newBorrowType});
604 Value subOverflow = LLVM::USubWithOverflowOp::create(
605 rewriter, loc, structType, adaptor.getLhs(), adaptor.getRhs());
606 Value diffExtracted =
607 LLVM::ExtractValueOp::create(rewriter, loc, subOverflow, 0);
608 Value borrowExtracted =
609 LLVM::ExtractValueOp::create(rewriter, loc, subOverflow, 1);
610 rewriter.replaceOp(op, {diffExtracted, borrowExtracted});
611 return success();
612 }
613
614 if (!isa<VectorType>(diffResultType))
615 return rewriter.notifyMatchFailure(loc, "expected vector result types");
616
617 return rewriter.notifyMatchFailure(loc,
618 "ND vector types are not supported yet");
619}
620
621//===----------------------------------------------------------------------===//
622// MulIExtendedOpLowering
623//===----------------------------------------------------------------------===//
624
625template <typename ArithMulOp, bool IsSigned>
626LogicalResult MulIExtendedOpLowering<ArithMulOp, IsSigned>::matchAndRewrite(
627 ArithMulOp op, typename ArithMulOp::Adaptor adaptor,
628 ConversionPatternRewriter &rewriter) const {
629 Type resultType = adaptor.getLhs().getType();
630
631 if (!LLVM::isCompatibleType(resultType))
632 return failure();
633
634 Location loc = op.getLoc();
635
636 // Handle the scalar and 1D vector cases. Because LLVM does not have a
637 // matching extended multiplication intrinsic, perform regular multiplication
638 // on operands zero-extended to i(2*N) bits, and truncate the results back to
639 // iN types.
640 if (!isa<LLVM::LLVMArrayType>(resultType)) {
641 // Shift amount necessary to extract the high bits from widened result.
642 TypedAttr shiftValAttr;
643
644 if (auto intTy = dyn_cast<IntegerType>(resultType)) {
645 unsigned resultBitwidth = intTy.getWidth();
646 auto attrTy = rewriter.getIntegerType(resultBitwidth * 2);
647 shiftValAttr = rewriter.getIntegerAttr(attrTy, resultBitwidth);
648 } else {
649 auto vecTy = cast<VectorType>(resultType);
650 unsigned resultBitwidth = vecTy.getElementTypeBitWidth();
651 auto attrTy = VectorType::get(
652 vecTy.getShape(), rewriter.getIntegerType(resultBitwidth * 2));
653 shiftValAttr = SplatElementsAttr::get(
654 attrTy, APInt(resultBitwidth * 2, resultBitwidth));
655 }
656 Type wideType = shiftValAttr.getType();
657 assert(LLVM::isCompatibleType(wideType) &&
658 "LLVM dialect should support all signless integer types");
659
660 using LLVMExtOp = std::conditional_t<IsSigned, LLVM::SExtOp, LLVM::ZExtOp>;
661 Value lhsExt = LLVMExtOp::create(rewriter, loc, wideType, adaptor.getLhs());
662 Value rhsExt = LLVMExtOp::create(rewriter, loc, wideType, adaptor.getRhs());
663 Value mulExt = LLVM::MulOp::create(rewriter, loc, wideType, lhsExt, rhsExt);
664
665 // Split the 2*N-bit wide result into two N-bit values.
666 Value low = LLVM::TruncOp::create(rewriter, loc, resultType, mulExt);
667 Value shiftVal = LLVM::ConstantOp::create(rewriter, loc, shiftValAttr);
668 Value highExt = LLVM::LShrOp::create(rewriter, loc, mulExt, shiftVal);
669 Value high = LLVM::TruncOp::create(rewriter, loc, resultType, highExt);
670
671 rewriter.replaceOp(op, {low, high});
672 return success();
673 }
674
675 if (!isa<VectorType>(resultType))
676 return rewriter.notifyMatchFailure(op, "expected vector result type");
677
678 return rewriter.notifyMatchFailure(op,
679 "ND vector types are not supported yet");
680}
681
682//===----------------------------------------------------------------------===//
683// CmpIOpLowering
684//===----------------------------------------------------------------------===//
685
686// Convert arith.cmp predicate into the LLVM dialect CmpPredicate. The two enums
687// share numerical values so just cast.
688template <typename LLVMPredType, typename PredType>
689static LLVMPredType convertCmpPredicate(PredType pred) {
690 return static_cast<LLVMPredType>(pred);
691}
692
693LogicalResult
694CmpIOpLowering::matchAndRewrite(arith::CmpIOp op, OpAdaptor adaptor,
695 ConversionPatternRewriter &rewriter) const {
696 Type operandType = adaptor.getLhs().getType();
697 Type resultType = op.getResult().getType();
698
699 // Handle the scalar and 1D vector cases.
700 if (!isa<LLVM::LLVMArrayType>(operandType)) {
701 rewriter.replaceOpWithNewOp<LLVM::ICmpOp>(
702 op, typeConverter->convertType(resultType),
703 convertCmpPredicate<LLVM::ICmpPredicate>(op.getPredicate()),
704 adaptor.getLhs(), adaptor.getRhs());
705 return success();
706 }
707
708 if (!isa<VectorType>(resultType))
709 return rewriter.notifyMatchFailure(op, "expected vector result type");
710
711 return LLVM::detail::handleMultidimensionalVectors(
712 op.getOperation(), adaptor.getOperands(), *getTypeConverter(),
713 [&](Type llvm1DVectorTy, ValueRange operands) {
714 OpAdaptor adaptor(operands);
715 return LLVM::ICmpOp::create(
716 rewriter, op.getLoc(), llvm1DVectorTy,
717 convertCmpPredicate<LLVM::ICmpPredicate>(op.getPredicate()),
718 adaptor.getLhs(), adaptor.getRhs());
719 },
720 rewriter);
721}
722
723//===----------------------------------------------------------------------===//
724// CmpFOpLowering
725//===----------------------------------------------------------------------===//
726
727LogicalResult
728CmpFOpLowering::matchAndRewrite(arith::CmpFOp op, OpAdaptor adaptor,
729 ConversionPatternRewriter &rewriter) const {
730 if (LLVM::detail::isUnsupportedFloatingPointType(*this->getTypeConverter(),
731 op.getLhs().getType()))
732 return rewriter.notifyMatchFailure(op, "unsupported floating point type");
733
734 Type operandType = adaptor.getLhs().getType();
735 Type resultType = op.getResult().getType();
736 LLVM::FastmathFlags fmf =
737 arith::convertArithFastMathFlagsToLLVM(op.getFastmath());
738
739 // Handle the scalar and 1D vector cases.
740 if (!isa<LLVM::LLVMArrayType>(operandType)) {
741 rewriter.replaceOpWithNewOp<LLVM::FCmpOp>(
742 op, typeConverter->convertType(resultType),
743 convertCmpPredicate<LLVM::FCmpPredicate>(op.getPredicate()),
744 adaptor.getLhs(), adaptor.getRhs(), fmf);
745 return success();
746 }
747
748 if (!isa<VectorType>(resultType))
749 return rewriter.notifyMatchFailure(op, "expected vector result type");
750
751 return LLVM::detail::handleMultidimensionalVectors(
752 op.getOperation(), adaptor.getOperands(), *getTypeConverter(),
753 [&](Type llvm1DVectorTy, ValueRange operands) {
754 OpAdaptor adaptor(operands);
755 return LLVM::FCmpOp::create(
756 rewriter, op.getLoc(), llvm1DVectorTy,
757 convertCmpPredicate<LLVM::FCmpPredicate>(op.getPredicate()),
758 adaptor.getLhs(), adaptor.getRhs(), fmf);
759 },
760 rewriter);
761}
762
763//===----------------------------------------------------------------------===//
764// SelectOpOneToNLowering
765//===----------------------------------------------------------------------===//
766
767/// Pattern for arith.select where the true/false values lower to multiple
768/// SSA values (1:N conversion). This pattern generates multiple arith.select
769/// than can be lowered by the 1:1 arith.select pattern.
770LogicalResult SelectOpOneToNLowering::matchAndRewrite(
771 arith::SelectOp op, Adaptor adaptor,
772 ConversionPatternRewriter &rewriter) const {
773 // In case of a 1:1 conversion, the 1:1 pattern will match.
774 if (llvm::hasSingleElement(adaptor.getTrueValue()))
775 return rewriter.notifyMatchFailure(
776 op, "not a 1:N conversion, 1:1 pattern will match");
777 if (!op.getCondition().getType().isInteger(1))
778 return rewriter.notifyMatchFailure(op,
779 "non-i1 conditions are not supported");
780 SmallVector<Value> results;
781 for (auto [trueValue, falseValue] :
782 llvm::zip_equal(adaptor.getTrueValue(), adaptor.getFalseValue()))
783 results.push_back(arith::SelectOp::create(
784 rewriter, op.getLoc(), op.getCondition(), trueValue, falseValue));
785 rewriter.replaceOpWithMultiple(op, {results});
786 return success();
787}
788
789//===----------------------------------------------------------------------===//
790// Pass Definition
791//===----------------------------------------------------------------------===//
792
793namespace {
794struct ArithToLLVMConversionPass
795 : public impl::ArithToLLVMConversionPassBase<ArithToLLVMConversionPass> {
796 using Base::Base;
797
798 void runOnOperation() override {
799 LLVMConversionTarget target(getContext());
800 RewritePatternSet patterns(&getContext());
801
802 LowerToLLVMOptions options(&getContext());
803 if (indexBitwidth != kDeriveIndexBitwidthFromDataLayout)
804 options.overrideIndexBitwidth(indexBitwidth);
805
806 LLVMTypeConverter converter(&getContext(), options);
807 arith::populateCeilFloorDivExpandOpsPatterns(patterns);
808 arith::populateArithToLLVMConversionPatterns(converter, patterns);
809
810 if (failed(applyPartialConversion(getOperation(), target,
811 std::move(patterns))))
812 signalPassFailure();
813 }
814};
815} // namespace
816
817//===----------------------------------------------------------------------===//
818// ConvertToLLVMPatternInterface implementation
819//===----------------------------------------------------------------------===//
820
821namespace {
822/// Implement the interface to convert MemRef to LLVM.
823struct ArithToLLVMDialectInterface : public ConvertToLLVMPatternInterface {
824 ArithToLLVMDialectInterface(Dialect *dialect)
825 : ConvertToLLVMPatternInterface(dialect) {}
826
827 void loadDependentDialects(MLIRContext *context) const final {
828 context->loadDialect<LLVM::LLVMDialect>();
829 }
830
831 /// Hook for derived dialect interface to provide conversion patterns
832 /// and mark dialect legal for the conversion target.
833 void populateConvertToLLVMConversionPatterns(
834 ConversionTarget &target, LLVMTypeConverter &typeConverter,
835 RewritePatternSet &patterns) const final {
836 arith::populateCeilFloorDivExpandOpsPatterns(patterns);
837 arith::populateArithToLLVMConversionPatterns(typeConverter, patterns);
838 }
839};
840} // namespace
841
842void mlir::arith::registerConvertArithToLLVMInterface(
843 DialectRegistry &registry) {
844 registry.addExtension(+[](MLIRContext *ctx, arith::ArithDialect *dialect) {
845 dialect->addInterfaces<ArithToLLVMDialectInterface>();
846 });
847}
848
849//===----------------------------------------------------------------------===//
850// Pattern Population
851//===----------------------------------------------------------------------===//
852
853void mlir::arith::populateArithToLLVMConversionPatterns(
854 const LLVMTypeConverter &converter, RewritePatternSet &patterns) {
855
856 // Set a higher pattern benefit for IdentityBitcastLowering so it will run
857 // before BitcastOpLowering.
858 patterns.add<IdentityBitcastLowering>(converter, patterns.getContext(),
859 /*patternBenefit*/ 10);
860
861 // clang-format off
862 patterns.add<
863 AddFOpLowering,
864 ConstrainedAddFOpLowering,
865 AddIOpLowering,
866 AndIOpLowering,
867 AddUIExtendedOpLowering,
868 SubUIExtendedOpLowering,
869 BitcastOpLowering,
870 ConstantOpLowering,
871 CmpFOpLowering,
872 CmpIOpLowering,
873 DivFOpLowering,
874 ConstrainedDivFOpLowering,
875 DivSIOpLowering,
876 DivUIOpLowering,
877 ExtFOpLowering,
878 ExtSIOpLowering,
879 ExtUIOpLowering,
880 ConvertFOpLowering,
881 FPToSIOpLowering,
882 FPToUIOpLowering,
883 IndexCastOpSILowering,
884 IndexCastOpUILowering,
885 MaximumFOpLowering,
886 MaxNumFOpLowering,
887 MaxSIOpLowering,
888 MaxUIOpLowering,
889 MinimumFOpLowering,
890 MinNumFOpLowering,
891 MinSIOpLowering,
892 MinUIOpLowering,
893 MulFOpLowering,
894 ConstrainedMulFOpLowering,
895 MulIOpLowering,
896 MulSIExtendedOpLowering,
897 MulUIExtendedOpLowering,
898 NegFOpLowering,
899 OrIOpLowering,
900 RemFOpLowering,
901 RemSIOpLowering,
902 RemUIOpLowering,
903 SelectOpLowering,
904 SelectOpOneToNLowering,
905 ShLIOpLowering,
906 ShRSIOpLowering,
907 ShRUIOpLowering,
908 SIToFPOpLowering,
909 SubFOpLowering,
910 ConstrainedSubFOpLowering,
911 SubIOpLowering,
912 TruncFOpLowering,
913 ConstrainedTruncFOpLowering,
914 TruncIOpLowering,
915 UIToFPOpLowering,
916 XOrIOpLowering
917 >(converter);
918 // clang-format on
919}
success
return success()
convertCmpPredicate
static LLVMPredType convertCmpPredicate(PredType pred)
Definition ArithToLLVM.cpp:689
convertConstantValue
static TypedAttr convertConstantValue(TypedAttr attr, Type resultType)
Retypes attr for a llvm.mlir.constant of resultType.
Definition ArithToLLVM.cpp:383
getContext
b getContext())
options
static llvm::ManagedStatic< PassManagerOptions > options
Definition PassManagerOptions.cpp:89
mlir::ConvertOpToLLVMPattern
Utility class for operation conversions targeting the LLVM dialect that match exactly one source oper...
Definition Pattern.h:233
mlir::ConvertOpToLLVMPattern::ConvertOpToLLVMPattern
ConvertOpToLLVMPattern(const LLVMTypeConverter &typeConverter, PatternBenefit benefit=1)
Definition Pattern.h:239
mlir::ConvertOpToLLVMPattern::OneToNOpAdaptor
typename SourceOp::template GenericAdaptor< ArrayRef< ValueRange > > OneToNOpAdaptor
Definition Pattern.h:236
mlir::DenseIntElementsAttr
An attribute that represents a reference to a dense integer vector or tensor object.
Definition BuiltinAttributes.h:947
mlir::DialectRegistry
The DialectRegistry maps a dialect namespace to a constructor for the matching dialect.
Definition DialectRegistry.h:150
mlir::DialectRegistry::addExtension
bool addExtension(TypeID extensionID, std::unique_ptr< DialectExtensionBase > extension)
Add the given extension to the registry.
Definition DialectRegistry.h:255
mlir::LLVMTypeConverter
Conversion from types to the LLVM IR dialect.
Definition TypeConverter.h:35
mlir::MLIRContext
MLIRContext is the top-level object for a collection of MLIR operations.
Definition MLIRContext.h:63
mlir::RewritePatternSet::getContext
MLIRContext * getContext() const
Definition PatternMatch.h:837
mlir::RewritePatternSet::add
RewritePatternSet & add(ConstructorArg &&arg, ConstructorArgs &&...args)
Add an instance of each of the pattern types 'Ts' to the pattern list with the given arguments.
Definition PatternMatch.h:861
mlir::Type
Instances of the Type class are uniqued, have an immutable identifier and an optional mutable compone...
Definition Types.h:74
mlir::Type::getIntOrFloatBitWidth
unsigned getIntOrFloatBitWidth() const
Return the bit width of an integer or a float type, assert failure on other types.
Definition Types.cpp:124
mlir::Value::getType
Type getType() const
Return the type of this value.
Definition Value.h:105
mlir::VectorConvertToLLVMPattern
Basic lowering implementation to rewrite Ops with just one result to the LLVM Dialect.
Definition VectorPattern.h:99
mlir::LLVM::detail::handleMultidimensionalVectors
LogicalResult handleMultidimensionalVectors(Operation *op, ValueRange operands, const LLVMTypeConverter &typeConverter, std::function< Value(Type, ValueRange)> createOperand, ConversionPatternRewriter &rewriter)
Definition VectorPattern.cpp:80
mlir::LLVM::detail::isUnsupportedFloatingPointType
bool isUnsupportedFloatingPointType(const TypeConverter &typeConverter, Type type)
Return "true" if the given type is an unsupported floating point type.
Definition Pattern.cpp:678
mlir::LLVM::detail::opHasUnsupportedFloatingPointTypes
bool opHasUnsupportedFloatingPointTypes(Operation *op, const TypeConverter &typeConverter)
Return "true" if the given op has any unsupported floating point types (either operands or results).
Definition Pattern.cpp:689
mlir::LLVM::getConstantElementType
Type getConstantElementType(Type type)
Determines the element type of type the way the llvm.mlir.constant verifier does, i....
Definition LLVMDialect.cpp:3504
mlir::arith::populateArithToLLVMConversionPatterns
void populateArithToLLVMConversionPatterns(const LLVMTypeConverter &converter, RewritePatternSet &patterns)
Definition ArithToLLVM.cpp:853
mlir::arith::registerConvertArithToLLVMInterface
void registerConvertArithToLLVMInterface(DialectRegistry &registry)
Definition ArithToLLVM.cpp:842
mlir::remark::failed
detail::InFlightRemark failed(Location loc, RemarkOpts opts)
Report an optimization remark that failed.
Definition Remarks.h:732
mlir
Include the generated interface declarations.
Definition ABIRewriteContext.h:32
mlir::kDeriveIndexBitwidthFromDataLayout
static constexpr unsigned kDeriveIndexBitwidthFromDataLayout
Value to pass as bitwidth for the index type when the converter is expected to derive the bitwidth fr...
Definition LoweringOptions.h:26
mlir::getElementTypeOrSelf
Type getElementTypeOrSelf(Type type)
Return the element type or return the type itself.
Definition TypeUtilities.cpp:23
Generated on for MLIR by doxygen 1.14.0