MLIR 24.0.0git
ArithToLLVM.cpp
Go to the documentation of this file.
1//===- ArithToLLVM.cpp - Arithmetic to LLVM dialect conversion -------===//
2//
3// Part of the LLVM Project, under the Apache License v2.0 with LLVM Exceptions.
4// See https://llvm.org/LICENSE.txt for license information.
5// SPDX-License-Identifier: Apache-2.0 WITH LLVM-exception
6//
7//===----------------------------------------------------------------------===//
8
9#include "mlir/Conversion/ArithToLLVM/ArithToLLVM.h"
10
11#include "mlir/Analysis/DataLayoutAnalysis.h"
12#include "mlir/Conversion/ArithAttrToLLVM/AttrToLLVMConverter.h"
13#include "mlir/Conversion/ArithCommon/CeilFloorDivExpansion.h"
14#include "mlir/Conversion/ConvertToLLVM/ToLLVMInterface.h"
15#include "mlir/Conversion/LLVMCommon/ConversionTarget.h"
16#include "mlir/Conversion/LLVMCommon/VectorPattern.h"
17#include "mlir/Dialect/Arith/IR/Arith.h"
18#include "mlir/Dialect/LLVMIR/FunctionCallUtils.h"
19#include "mlir/Dialect/LLVMIR/LLVMAttrs.h"
20#include "mlir/Dialect/LLVMIR/LLVMDialect.h"
21#include "mlir/IR/DialectResourceBlobManager.h"
22#include "mlir/IR/TypeUtilities.h"
23#include "mlir/Pass/Pass.h"
24#include <type_traits>
25
26namespace mlir {
27#define GEN_PASS_DEF_ARITHTOLLVMCONVERSIONPASS
28#include "mlir/Conversion/Passes.h.inc"
29} // namespace mlir
30
31using namespace mlir;
32
33namespace {
34
35/// Lowering pattern that matches only when the source op's rounding mode
36/// presence agrees with `HasRoundingMode`. This allows registering two
37/// instances of the same pattern for one source op: one that handles the
38/// unconstrained case (no rounding mode, lowering to a regular LLVM op) and
39/// one that handles the constrained case (rounding mode present, lowering to
40/// a constrained LLVM intrinsic).
41///
42/// * `HasRoundingMode`: the pattern matches if and only if the source op has
43/// a rounding mode attribute.
44/// * `AttrConvert`: attribute converter to translate source attributes to
45/// target attributes.
46/// * `FailOnUnsupportedFP`: whether to fail if the source op has unsupported
47/// floating point types.
48template <typename SourceOp, typename TargetOp, bool HasRoundingMode,
49 template <typename, typename> typename AttrConvert =
50 AttrConvertPassThrough,
51 bool FailOnUnsupportedFP = false>
52struct ConstrainedVectorConvertToLLVMPattern
53 : public VectorConvertToLLVMPattern<SourceOp, TargetOp, AttrConvert,
54 FailOnUnsupportedFP> {
55 using VectorConvertToLLVMPattern<
56 SourceOp, TargetOp, AttrConvert,
57 FailOnUnsupportedFP>::VectorConvertToLLVMPattern;
58
59 LogicalResult
60 matchAndRewrite(SourceOp op, typename SourceOp::Adaptor adaptor,
61 ConversionPatternRewriter &rewriter) const override {
62 if (HasRoundingMode != static_cast<bool>(op.getRoundingModeAttr()))
63 return failure();
64 return VectorConvertToLLVMPattern<
65 SourceOp, TargetOp, AttrConvert,
66 FailOnUnsupportedFP>::matchAndRewrite(op, adaptor, rewriter);
67 }
68};
69
70/// No-op bitcast. Propagate type input arg if converted source and dest types
71/// are the same.
72struct IdentityBitcastLowering final
73 : public OpConversionPattern<arith::BitcastOp> {
74 using Base::Base;
75
76 LogicalResult
77 matchAndRewrite(arith::BitcastOp op, OpAdaptor adaptor,
78 ConversionPatternRewriter &rewriter) const final {
79 Value src = adaptor.getIn();
80 Type resultType = getTypeConverter()->convertType(op.getType());
81 if (src.getType() != resultType)
82 return rewriter.notifyMatchFailure(op, "Types are different");
83
84 rewriter.replaceOp(op, src);
85 return success();
86 }
87};
88
89//===----------------------------------------------------------------------===//
90// Straightforward Op Lowerings
91//===----------------------------------------------------------------------===//
92
93using AddFOpLowering =
94 ConstrainedVectorConvertToLLVMPattern<arith::AddFOp, LLVM::FAddOp,
95 /*HasRoundingMode=*/false,
96 arith::AttrConvertFastMathToLLVM,
97 /*FailOnUnsupportedFP=*/true>;
98using ConstrainedAddFOpLowering = ConstrainedVectorConvertToLLVMPattern<
99 arith::AddFOp, LLVM::ConstrainedFAddIntr, /*HasRoundingMode=*/true,
100 arith::AttrConverterConstrainedFPToLLVM, /*FailOnUnsupportedFP=*/true>;
101using AddIOpLowering =
102 VectorConvertToLLVMPattern<arith::AddIOp, LLVM::AddOp,
103 arith::AttrConvertOverflowToLLVM>;
104using AndIOpLowering = VectorConvertToLLVMPattern<arith::AndIOp, LLVM::AndOp>;
105using BitcastOpLowering =
106 VectorConvertToLLVMPattern<arith::BitcastOp, LLVM::BitcastOp>;
107using DivFOpLowering =
108 ConstrainedVectorConvertToLLVMPattern<arith::DivFOp, LLVM::FDivOp,
109 /*HasRoundingMode=*/false,
110 arith::AttrConvertFastMathToLLVM,
111 /*FailOnUnsupportedFP=*/true>;
112using ConstrainedDivFOpLowering = ConstrainedVectorConvertToLLVMPattern<
113 arith::DivFOp, LLVM::ConstrainedFDivIntr, /*HasRoundingMode=*/true,
114 arith::AttrConverterConstrainedFPToLLVM, /*FailOnUnsupportedFP=*/true>;
115using DivSIOpLowering =
116 VectorConvertToLLVMPattern<arith::DivSIOp, LLVM::SDivOp>;
117using DivUIOpLowering =
118 VectorConvertToLLVMPattern<arith::DivUIOp, LLVM::UDivOp>;
119using ExtFOpLowering =
120 VectorConvertToLLVMPattern<arith::ExtFOp, LLVM::FPExtOp,
121 arith::AttrConvertFastMathToLLVM,
122 /*FailOnUnsupportedFP=*/true>;
123using ExtSIOpLowering =
124 VectorConvertToLLVMPattern<arith::ExtSIOp, LLVM::SExtOp>;
125using ExtUIOpLowering =
126 VectorConvertToLLVMPattern<arith::ExtUIOp, LLVM::ZExtOp,
127 arith::AttrConvertNonNegToLLVM>;
128using FPToSIOpLowering =
129 VectorConvertToLLVMPattern<arith::FPToSIOp, LLVM::FPToSIOp,
130 AttrConvertPassThrough,
131 /*FailOnUnsupportedFP=*/true>;
132using FPToUIOpLowering =
133 VectorConvertToLLVMPattern<arith::FPToUIOp, LLVM::FPToUIOp,
134 AttrConvertPassThrough,
135 /*FailOnUnsupportedFP=*/true>;
136using MaximumFOpLowering =
137 VectorConvertToLLVMPattern<arith::MaximumFOp, LLVM::MaximumOp,
138 arith::AttrConvertFastMathToLLVM,
139 /*FailOnUnsupportedFP=*/true>;
140using MaxNumFOpLowering =
141 VectorConvertToLLVMPattern<arith::MaxNumFOp, LLVM::MaxNumOp,
142 arith::AttrConvertFastMathToLLVM,
143 /*FailOnUnsupportedFP=*/true>;
144using MaximumNumFOpLowering =
145 VectorConvertToLLVMPattern<arith::MaximumNumFOp, LLVM::MaximumNumOp,
146 arith::AttrConvertFastMathToLLVM,
147 /*FailOnUnsupportedFP=*/true>;
148using MaxSIOpLowering =
149 VectorConvertToLLVMPattern<arith::MaxSIOp, LLVM::SMaxOp>;
150using MaxUIOpLowering =
151 VectorConvertToLLVMPattern<arith::MaxUIOp, LLVM::UMaxOp>;
152using MinimumFOpLowering =
153 VectorConvertToLLVMPattern<arith::MinimumFOp, LLVM::MinimumOp,
154 arith::AttrConvertFastMathToLLVM,
155 /*FailOnUnsupportedFP=*/true>;
156using MinNumFOpLowering =
157 VectorConvertToLLVMPattern<arith::MinNumFOp, LLVM::MinNumOp,
158 arith::AttrConvertFastMathToLLVM,
159 /*FailOnUnsupportedFP=*/true>;
160using MinimumNumFOpLowering =
161 VectorConvertToLLVMPattern<arith::MinimumNumFOp, LLVM::MinimumNumOp,
162 arith::AttrConvertFastMathToLLVM,
163 /*FailOnUnsupportedFP=*/true>;
164using MinSIOpLowering =
165 VectorConvertToLLVMPattern<arith::MinSIOp, LLVM::SMinOp>;
166using MinUIOpLowering =
167 VectorConvertToLLVMPattern<arith::MinUIOp, LLVM::UMinOp>;
168using MulFOpLowering =
169 ConstrainedVectorConvertToLLVMPattern<arith::MulFOp, LLVM::FMulOp,
170 /*HasRoundingMode=*/false,
171 arith::AttrConvertFastMathToLLVM,
172 /*FailOnUnsupportedFP=*/true>;
173using ConstrainedMulFOpLowering = ConstrainedVectorConvertToLLVMPattern<
174 arith::MulFOp, LLVM::ConstrainedFMulIntr, /*HasRoundingMode=*/true,
175 arith::AttrConverterConstrainedFPToLLVM, /*FailOnUnsupportedFP=*/true>;
176using MulIOpLowering =
177 VectorConvertToLLVMPattern<arith::MulIOp, LLVM::MulOp,
178 arith::AttrConvertOverflowToLLVM>;
179using NegFOpLowering =
180 VectorConvertToLLVMPattern<arith::NegFOp, LLVM::FNegOp,
181 arith::AttrConvertFastMathToLLVM,
182 /*FailOnUnsupportedFP=*/true>;
183using OrIOpLowering = VectorConvertToLLVMPattern<arith::OrIOp, LLVM::OrOp>;
184using RemFOpLowering =
185 VectorConvertToLLVMPattern<arith::RemFOp, LLVM::FRemOp,
186 arith::AttrConvertFastMathToLLVM,
187 /*FailOnUnsupportedFP=*/true>;
188using RemSIOpLowering =
189 VectorConvertToLLVMPattern<arith::RemSIOp, LLVM::SRemOp>;
190using RemUIOpLowering =
191 VectorConvertToLLVMPattern<arith::RemUIOp, LLVM::URemOp>;
192using SelectOpLowering =
193 VectorConvertToLLVMPattern<arith::SelectOp, LLVM::SelectOp>;
194using ShLIOpLowering =
195 VectorConvertToLLVMPattern<arith::ShLIOp, LLVM::ShlOp,
196 arith::AttrConvertOverflowToLLVM>;
197using ShRSIOpLowering =
198 VectorConvertToLLVMPattern<arith::ShRSIOp, LLVM::AShrOp>;
199using ShRUIOpLowering =
200 VectorConvertToLLVMPattern<arith::ShRUIOp, LLVM::LShrOp>;
201using SIToFPOpLowering =
202 VectorConvertToLLVMPattern<arith::SIToFPOp, LLVM::SIToFPOp,
203 AttrConvertPassThrough,
204 /*FailOnUnsupportedFP=*/true>;
205using SubFOpLowering =
206 ConstrainedVectorConvertToLLVMPattern<arith::SubFOp, LLVM::FSubOp,
207 /*HasRoundingMode=*/false,
208 arith::AttrConvertFastMathToLLVM,
209 /*FailOnUnsupportedFP=*/true>;
210using ConstrainedSubFOpLowering = ConstrainedVectorConvertToLLVMPattern<
211 arith::SubFOp, LLVM::ConstrainedFSubIntr, /*HasRoundingMode=*/true,
212 arith::AttrConverterConstrainedFPToLLVM, /*FailOnUnsupportedFP=*/true>;
213using SubIOpLowering =
214 VectorConvertToLLVMPattern<arith::SubIOp, LLVM::SubOp,
215 arith::AttrConvertOverflowToLLVM>;
216using TruncFOpLowering =
217 ConstrainedVectorConvertToLLVMPattern<arith::TruncFOp, LLVM::FPTruncOp,
218 /*HasRoundingMode=*/false,
219 arith::AttrConvertFastMathToLLVM,
220 /*FailOnUnsupportedFP=*/true>;
221using ConstrainedTruncFOpLowering = ConstrainedVectorConvertToLLVMPattern<
222 arith::TruncFOp, LLVM::ConstrainedFPTruncIntr, /*HasRoundingMode=*/true,
223 arith::AttrConverterConstrainedFPToLLVM, /*FailOnUnsupportedFP=*/true>;
224using TruncIOpLowering =
225 VectorConvertToLLVMPattern<arith::TruncIOp, LLVM::TruncOp,
226 arith::AttrConvertOverflowToLLVM>;
227using UIToFPOpLowering =
228 VectorConvertToLLVMPattern<arith::UIToFPOp, LLVM::UIToFPOp,
229 arith::AttrConvertNonNegToLLVM,
230 /*FailOnUnsupportedFP=*/true>;
231using XOrIOpLowering = VectorConvertToLLVMPattern<arith::XOrIOp, LLVM::XOrOp>;
232
233//===----------------------------------------------------------------------===//
234// Op Lowering Patterns
235//===----------------------------------------------------------------------===//
236
237/// Directly lower to LLVM op.
238struct ConstantOpLowering : public ConvertOpToLLVMPattern<arith::ConstantOp> {
239 using ConvertOpToLLVMPattern::ConvertOpToLLVMPattern;
240
241 LogicalResult
242 matchAndRewrite(arith::ConstantOp op, OpAdaptor adaptor,
243 ConversionPatternRewriter &rewriter) const override;
244};
245
246/// The lowering of index_cast becomes an integer conversion since index
247/// becomes an integer. If the bit width of the source and target integer
248/// types is the same, just erase the cast. If the target type is wider,
249/// sign-extend the value, otherwise truncate it.
250template <typename OpTy, typename ExtCastTy>
251struct IndexCastOpLowering : public ConvertOpToLLVMPattern<OpTy> {
252 using ConvertOpToLLVMPattern<OpTy>::ConvertOpToLLVMPattern;
253
254 LogicalResult
255 matchAndRewrite(OpTy op, typename OpTy::Adaptor adaptor,
256 ConversionPatternRewriter &rewriter) const override;
257};
258
259using IndexCastOpSILowering =
260 IndexCastOpLowering<arith::IndexCastOp, LLVM::SExtOp>;
261using IndexCastOpUILowering =
262 IndexCastOpLowering<arith::IndexCastUIOp, LLVM::ZExtOp>;
263
264struct AddUIExtendedOpLowering
265 : public ConvertOpToLLVMPattern<arith::AddUIExtendedOp> {
266 using ConvertOpToLLVMPattern::ConvertOpToLLVMPattern;
267
268 LogicalResult
269 matchAndRewrite(arith::AddUIExtendedOp op, OpAdaptor adaptor,
270 ConversionPatternRewriter &rewriter) const override;
271};
272
273struct SubUIExtendedOpLowering
274 : public ConvertOpToLLVMPattern<arith::SubUIExtendedOp> {
275 using ConvertOpToLLVMPattern::ConvertOpToLLVMPattern;
276
277 LogicalResult
278 matchAndRewrite(arith::SubUIExtendedOp op, OpAdaptor adaptor,
279 ConversionPatternRewriter &rewriter) const override;
280};
281
282template <typename ArithMulOp, bool IsSigned>
283struct MulIExtendedOpLowering : public ConvertOpToLLVMPattern<ArithMulOp> {
284 using ConvertOpToLLVMPattern<ArithMulOp>::ConvertOpToLLVMPattern;
285
286 LogicalResult
287 matchAndRewrite(ArithMulOp op, typename ArithMulOp::Adaptor adaptor,
288 ConversionPatternRewriter &rewriter) const override;
289};
290
291using MulSIExtendedOpLowering =
292 MulIExtendedOpLowering<arith::MulSIExtendedOp, true>;
293using MulUIExtendedOpLowering =
294 MulIExtendedOpLowering<arith::MulUIExtendedOp, false>;
295
296struct CmpIOpLowering : public ConvertOpToLLVMPattern<arith::CmpIOp> {
297 using ConvertOpToLLVMPattern::ConvertOpToLLVMPattern;
298
299 LogicalResult
300 matchAndRewrite(arith::CmpIOp op, OpAdaptor adaptor,
301 ConversionPatternRewriter &rewriter) const override;
302};
303
304struct CmpFOpLowering : public ConvertOpToLLVMPattern<arith::CmpFOp> {
305 using ConvertOpToLLVMPattern::ConvertOpToLLVMPattern;
306
307 LogicalResult
308 matchAndRewrite(arith::CmpFOp op, OpAdaptor adaptor,
309 ConversionPatternRewriter &rewriter) const override;
310};
311
312/// Lower arith.convertf (same-bitwidth FP cast) to LLVM.
313///
314/// Extends to f32 via llvm.fpext, then truncates to the target type via
315/// llvm.fptrunc. This handles bf16 <-> f16, which is the only same-bitwidth
316/// pair of LLVM-supported FP types.
317struct ConvertFOpLowering : public ConvertOpToLLVMPattern<arith::ConvertFOp> {
318 using ConvertOpToLLVMPattern::ConvertOpToLLVMPattern;
319
320 LogicalResult
321 matchAndRewrite(arith::ConvertFOp op, OpAdaptor adaptor,
322 ConversionPatternRewriter &rewriter) const override {
323 if (LLVM::detail::opHasUnsupportedFloatingPointTypes(op,
324 *getTypeConverter()))
325 return rewriter.notifyMatchFailure(op, "unsupported floating point type");
326
327 // Only bf16 <-> f16 conversions are supported. There is currently no other
328 // pair of FP types that are valid LLVM types.
329 [[maybe_unused]] auto srcType = getElementTypeOrSelf(op.getIn().getType());
330 [[maybe_unused]] auto dstType = getElementTypeOrSelf(op.getType());
331 assert(((srcType.isBF16() && dstType.isF16()) ||
332 (srcType.isF16() && dstType.isBF16())) &&
333 "only bf16 <-> f16 conversions are supported");
334
335 Type convertedType = getTypeConverter()->convertType(op.getType());
336 if (!convertedType)
337 return rewriter.notifyMatchFailure(op, "failed to convert result type");
338
339 Value input = adaptor.getIn();
340 Location loc = op.getLoc();
341
342 if (!isa<LLVM::LLVMArrayType>(input.getType())) {
343 rewriter.replaceOp(op,
344 emitConversion(rewriter, loc, input, convertedType));
345 return success();
346 }
347
348 if (!isa<VectorType>(op.getType()))
349 return rewriter.notifyMatchFailure(op, "expected vector result type");
350
351 return LLVM::detail::handleMultidimensionalVectors(
352 op.getOperation(), adaptor.getOperands(), *getTypeConverter(),
353 [&](Type llvm1DVectorTy, ValueRange operands) -> Value {
354 return emitConversion(rewriter, loc, operands.front(),
355 llvm1DVectorTy);
356 },
357 rewriter);
358 }
359
360private:
361 static Value emitConversion(ConversionPatternRewriter &rewriter, Location loc,
362 Value input, Type targetType) {
363 Type f32Scalar = Float32Type::get(rewriter.getContext());
364 Type f32Ty = f32Scalar;
365 if (auto vecTy = dyn_cast<VectorType>(targetType))
366 f32Ty = VectorType::get(vecTy.getShape(), f32Scalar);
367
368 Value ext = LLVM::FPExtOp::create(rewriter, loc, f32Ty, input);
369 return LLVM::FPTruncOp::create(rewriter, loc, targetType, ext);
370 }
371};
372
373struct SelectOpOneToNLowering : public ConvertOpToLLVMPattern<arith::SelectOp> {
374 using ConvertOpToLLVMPattern::ConvertOpToLLVMPattern;
375 using Adaptor = ConvertOpToLLVMPattern<arith::SelectOp>::OneToNOpAdaptor;
376
377 LogicalResult
378 matchAndRewrite(arith::SelectOp op, Adaptor adaptor,
379 ConversionPatternRewriter &rewriter) const override;
380};
381
382} // namespace
383
384//===----------------------------------------------------------------------===//
385// ConstantOpLowering
386//===----------------------------------------------------------------------===//
387
388/// Retypes `attr` for a `llvm.mlir.constant` of `resultType`. `arith.constant`
389/// requires the value attribute and the result to have the same type, but the
390/// type converter may map the element type to a different one, e.g. `index` to
391/// `i32` or `i64` depending on the configured index bitwidth. Build the
392/// attribute from the converted type so that the two agree wherever that is
393/// representable. Returns a null attribute if `attr` cannot be used for
394/// `resultType`.
395static TypedAttr convertConstantValue(TypedAttr attr, Type resultType) {
396 // Compare the element types, but explicitly ignore the non-scalar portions of
397 // types. This relaxation is required to support multi-dimensional vector
398 // constant that result in nested LLVM array values.
399 Type sourceElementType = LLVM::getConstantElementType(attr.getType());
400 Type targetElementType = LLVM::getConstantElementType(resultType);
401 if (sourceElementType == targetElementType)
402 return attr;
403
404 auto targetIntType = dyn_cast<IntegerType>(targetElementType);
405 if (!targetIntType)
406 return {};
407
408 // The converter maps the low-precision float types that have no LLVM
409 // equivalent to an integer of the same width. The attribute stays a float
410 // attribute in that case.
411 if (auto sourceFloatType = dyn_cast<FloatType>(sourceElementType)) {
412 if (sourceFloatType.getWidth() != targetIntType.getWidth())
413 return {};
414 return attr;
415 }
416
417 // Apart from those floats, `index` is the only element type the converter
418 // rewrites, so anything else is a malformed `arith.constant`. Bail out rather
419 // than reinterpret its value, which would silently change the constant (e.g.
420 // sign-extending an `i1` `true` to -1).
421 if (!isa<IndexType>(sourceElementType))
422 return {};
423 // `index` is signless but holds signed values, so narrowing to a smaller
424 // index bitwidth truncates and widening sign-extends.
425 unsigned width = targetIntType.getWidth();
426
427 if (auto intAttr = dyn_cast<IntegerAttr>(attr))
428 return IntegerAttr::get(targetIntType,
429 intAttr.getValue().sextOrTrunc(width));
430
431 auto retypeValues = [&](DenseIntElementsAttr values) {
432 return values.mapValues(targetIntType, [&](const APInt &value) {
433 return value.sextOrTrunc(width);
434 });
435 };
436
437 if (auto denseAttr = dyn_cast<DenseIntElementsAttr>(attr))
438 return retypeValues(denseAttr);
439
440 if (auto sparseAttr = dyn_cast<SparseElementsAttr>(attr))
441 return SparseElementsAttr::get(
442 cast<ShapedType>(attr.getType()).clone(targetIntType),
443 sparseAttr.getIndices(),
444 retypeValues(cast<DenseIntElementsAttr>(sparseAttr.getValues())));
445
446 // A resource-backed elements attribute refers to a blob laid out for its own
447 // element type. The blob cannot be rewritten here, only reinterpreted, which
448 // is correct exactly when the target type has the same width as the storage
449 // `index` uses in a blob.
450 if (auto resourceAttr = dyn_cast<DenseResourceElementsAttr>(attr)) {
451 if (width != IndexType::kInternalStorageBitWidth)
452 return {};
453 return DenseResourceElementsAttr::get(
454 cast<ShapedType>(attr.getType()).clone(targetIntType),
455 resourceAttr.getRawHandle());
456 }
457
458 return {};
459}
460
461LogicalResult
462ConstantOpLowering::matchAndRewrite(arith::ConstantOp op, OpAdaptor adaptor,
463 ConversionPatternRewriter &rewriter) const {
464 Type resultType = getTypeConverter()->convertType(op.getType());
465 if (!resultType)
466 return rewriter.notifyMatchFailure(op, "failed to convert result type");
467
468 TypedAttr value = convertConstantValue(op.getValue(), resultType);
469 if (!value)
470 return rewriter.notifyMatchFailure(
471 op, "failed to convert value attribute to the converted result type");
472
473 // `arith.constant` has no operands and a single result, so there is nothing
474 // for `oneToOneRewrite` to do here beyond converting `resultType` a second
475 // time.
476 DictionaryAttr discardableAttrs = op->getDiscardableAttrDictionary();
477 auto constantOp =
478 LLVM::ConstantOp::create(rewriter, op.getLoc(), resultType, value);
479 constantOp->setDiscardableAttrs(discardableAttrs);
480 rewriter.replaceOp(op, constantOp);
481 return success();
482}
483
484//===----------------------------------------------------------------------===//
485// IndexCastOpLowering
486//===----------------------------------------------------------------------===//
487
488template <typename OpTy, typename ExtCastTy>
489LogicalResult IndexCastOpLowering<OpTy, ExtCastTy>::matchAndRewrite(
490 OpTy op, typename OpTy::Adaptor adaptor,
491 ConversionPatternRewriter &rewriter) const {
492 Type resultType = op.getResult().getType();
493 Type targetElementType =
494 this->typeConverter->convertType(getElementTypeOrSelf(resultType));
495 Type sourceElementType =
496 this->typeConverter->convertType(getElementTypeOrSelf(op.getIn()));
497 unsigned targetBits = targetElementType.getIntOrFloatBitWidth();
498 unsigned sourceBits = sourceElementType.getIntOrFloatBitWidth();
499
500 if (targetBits == sourceBits) {
501 rewriter.replaceOp(op, adaptor.getIn());
502 return success();
503 }
504
505 // Memref index_cast is a no-op at the LLVM level since LLVM uses opaque
506 // pointers and memrefs of different integer/index element types all convert
507 // to the same LLVM struct type.
508 if (isa<MemRefType>(op.getIn().getType())) {
509 rewriter.replaceOp(op, adaptor.getIn());
510 return success();
511 }
512
513 bool isNonNeg = false;
514 if constexpr (std::is_same_v<ExtCastTy, LLVM::ZExtOp>)
515 isNonNeg = op.getNonNeg();
516
517 // Handle the scalar and 1D vector cases.
518 Type operandType = adaptor.getIn().getType();
519 if (!isa<LLVM::LLVMArrayType>(operandType)) {
520 Type targetType = this->typeConverter->convertType(resultType);
521 if (targetBits < sourceBits) {
522 rewriter.replaceOpWithNewOp<LLVM::TruncOp>(op, targetType,
523 adaptor.getIn());
524 } else {
525 auto extOp = rewriter.replaceOpWithNewOp<ExtCastTy>(op, targetType,
526 adaptor.getIn());
527 if constexpr (std::is_same_v<ExtCastTy, LLVM::ZExtOp>)
528 extOp.setNonNeg(isNonNeg);
529 }
530 return success();
531 }
532
533 if (!isa<VectorType>(resultType))
534 return rewriter.notifyMatchFailure(op, "expected vector result type");
535
536 return LLVM::detail::handleMultidimensionalVectors(
537 op.getOperation(), adaptor.getOperands(), *(this->getTypeConverter()),
538 [&](Type llvm1DVectorTy, ValueRange operands) -> Value {
539 typename OpTy::Adaptor adaptor(operands);
540 if (targetBits < sourceBits) {
541 return LLVM::TruncOp::create(rewriter, op.getLoc(), llvm1DVectorTy,
542 adaptor.getIn());
543 }
544 auto extOp = ExtCastTy::create(rewriter, op.getLoc(), llvm1DVectorTy,
545 adaptor.getIn());
546 if constexpr (std::is_same_v<ExtCastTy, LLVM::ZExtOp>) {
547 if (isNonNeg)
548 extOp.setNonNeg(true);
549 }
550 return extOp;
551 },
552 rewriter);
553}
554
555//===----------------------------------------------------------------------===//
556// AddUIExtendedOpLowering
557//===----------------------------------------------------------------------===//
558
559LogicalResult AddUIExtendedOpLowering::matchAndRewrite(
560 arith::AddUIExtendedOp op, OpAdaptor adaptor,
561 ConversionPatternRewriter &rewriter) const {
562 Type operandType = adaptor.getLhs().getType();
563 Type sumResultType = op.getSum().getType();
564 Type overflowResultType = op.getOverflow().getType();
565
566 if (!LLVM::isCompatibleType(operandType))
567 return failure();
568
569 MLIRContext *ctx = rewriter.getContext();
570 Location loc = op.getLoc();
571
572 // Handle the scalar and 1D vector cases.
573 if (!isa<LLVM::LLVMArrayType>(operandType)) {
574 Type newSumType = typeConverter->convertType(sumResultType);
575 Type newOverflowType = typeConverter->convertType(overflowResultType);
576 Type structType =
577 LLVM::LLVMStructType::getLiteral(ctx, {newSumType, newOverflowType});
578 Value addOverflow = LLVM::UAddWithOverflowOp::create(
579 rewriter, loc, structType, adaptor.getLhs(), adaptor.getRhs());
580 Value sumExtracted =
581 LLVM::ExtractValueOp::create(rewriter, loc, addOverflow, 0);
582 Value overflowExtracted =
583 LLVM::ExtractValueOp::create(rewriter, loc, addOverflow, 1);
584 rewriter.replaceOp(op, {sumExtracted, overflowExtracted});
585 return success();
586 }
587
588 if (!isa<VectorType>(sumResultType))
589 return rewriter.notifyMatchFailure(loc, "expected vector result types");
590
591 return rewriter.notifyMatchFailure(loc,
592 "ND vector types are not supported yet");
593}
594
595//===----------------------------------------------------------------------===//
596// SubUIExtendedOpLowering
597//===----------------------------------------------------------------------===//
598
599LogicalResult SubUIExtendedOpLowering::matchAndRewrite(
600 arith::SubUIExtendedOp op, OpAdaptor adaptor,
601 ConversionPatternRewriter &rewriter) const {
602 Type operandType = adaptor.getLhs().getType();
603 Type diffResultType = op.getDiff().getType();
604 Type borrowResultType = op.getBorrow().getType();
605
606 if (!LLVM::isCompatibleType(operandType))
607 return failure();
608
609 MLIRContext *ctx = rewriter.getContext();
610 Location loc = op.getLoc();
611
612 // Handle the scalar and 1D vector cases.
613 if (!isa<LLVM::LLVMArrayType>(operandType)) {
614 Type newBorrowType = typeConverter->convertType(borrowResultType);
615 Type structType =
616 LLVM::LLVMStructType::getLiteral(ctx, {diffResultType, newBorrowType});
617 Value subOverflow = LLVM::USubWithOverflowOp::create(
618 rewriter, loc, structType, adaptor.getLhs(), adaptor.getRhs());
619 Value diffExtracted =
620 LLVM::ExtractValueOp::create(rewriter, loc, subOverflow, 0);
621 Value borrowExtracted =
622 LLVM::ExtractValueOp::create(rewriter, loc, subOverflow, 1);
623 rewriter.replaceOp(op, {diffExtracted, borrowExtracted});
624 return success();
625 }
626
627 if (!isa<VectorType>(diffResultType))
628 return rewriter.notifyMatchFailure(loc, "expected vector result types");
629
630 return rewriter.notifyMatchFailure(loc,
631 "ND vector types are not supported yet");
632}
633
634//===----------------------------------------------------------------------===//
635// MulIExtendedOpLowering
636//===----------------------------------------------------------------------===//
637
638template <typename ArithMulOp, bool IsSigned>
639LogicalResult MulIExtendedOpLowering<ArithMulOp, IsSigned>::matchAndRewrite(
640 ArithMulOp op, typename ArithMulOp::Adaptor adaptor,
641 ConversionPatternRewriter &rewriter) const {
642 Type resultType = adaptor.getLhs().getType();
643
644 if (!LLVM::isCompatibleType(resultType))
645 return failure();
646
647 Location loc = op.getLoc();
648
649 // Handle the scalar and 1D vector cases. Because LLVM does not have a
650 // matching extended multiplication intrinsic, perform regular multiplication
651 // on operands zero-extended to i(2*N) bits, and truncate the results back to
652 // iN types.
653 if (!isa<LLVM::LLVMArrayType>(resultType)) {
654 // Shift amount necessary to extract the high bits from widened result.
655 TypedAttr shiftValAttr;
656
657 if (auto intTy = dyn_cast<IntegerType>(resultType)) {
658 unsigned resultBitwidth = intTy.getWidth();
659 auto attrTy = rewriter.getIntegerType(resultBitwidth * 2);
660 shiftValAttr = rewriter.getIntegerAttr(attrTy, resultBitwidth);
661 } else {
662 auto vecTy = cast<VectorType>(resultType);
663 unsigned resultBitwidth = vecTy.getElementTypeBitWidth();
664 auto attrTy = VectorType::get(
665 vecTy.getShape(), rewriter.getIntegerType(resultBitwidth * 2));
666 shiftValAttr = SplatElementsAttr::get(
667 attrTy, APInt(resultBitwidth * 2, resultBitwidth));
668 }
669 Type wideType = shiftValAttr.getType();
670 assert(LLVM::isCompatibleType(wideType) &&
671 "LLVM dialect should support all signless integer types");
672
673 using LLVMExtOp = std::conditional_t<IsSigned, LLVM::SExtOp, LLVM::ZExtOp>;
674 Value lhsExt = LLVMExtOp::create(rewriter, loc, wideType, adaptor.getLhs());
675 Value rhsExt = LLVMExtOp::create(rewriter, loc, wideType, adaptor.getRhs());
676 Value mulExt = LLVM::MulOp::create(rewriter, loc, wideType, lhsExt, rhsExt);
677
678 // Split the 2*N-bit wide result into two N-bit values.
679 Value low = LLVM::TruncOp::create(rewriter, loc, resultType, mulExt);
680 Value shiftVal = LLVM::ConstantOp::create(rewriter, loc, shiftValAttr);
681 Value highExt = LLVM::LShrOp::create(rewriter, loc, mulExt, shiftVal);
682 Value high = LLVM::TruncOp::create(rewriter, loc, resultType, highExt);
683
684 rewriter.replaceOp(op, {low, high});
685 return success();
686 }
687
688 if (!isa<VectorType>(resultType))
689 return rewriter.notifyMatchFailure(op, "expected vector result type");
690
691 return rewriter.notifyMatchFailure(op,
692 "ND vector types are not supported yet");
693}
694
695//===----------------------------------------------------------------------===//
696// CmpIOpLowering
697//===----------------------------------------------------------------------===//
698
699// Convert arith.cmp predicate into the LLVM dialect CmpPredicate. The two enums
700// share numerical values so just cast.
701template <typename LLVMPredType, typename PredType>
702static LLVMPredType convertCmpPredicate(PredType pred) {
703 return static_cast<LLVMPredType>(pred);
704}
705
706LogicalResult
707CmpIOpLowering::matchAndRewrite(arith::CmpIOp op, OpAdaptor adaptor,
708 ConversionPatternRewriter &rewriter) const {
709 Type operandType = adaptor.getLhs().getType();
710 Type resultType = op.getResult().getType();
711
712 // Handle the scalar and 1D vector cases.
713 if (!isa<LLVM::LLVMArrayType>(operandType)) {
714 rewriter.replaceOpWithNewOp<LLVM::ICmpOp>(
715 op, typeConverter->convertType(resultType),
716 convertCmpPredicate<LLVM::ICmpPredicate>(op.getPredicate()),
717 adaptor.getLhs(), adaptor.getRhs());
718 return success();
719 }
720
721 if (!isa<VectorType>(resultType))
722 return rewriter.notifyMatchFailure(op, "expected vector result type");
723
724 return LLVM::detail::handleMultidimensionalVectors(
725 op.getOperation(), adaptor.getOperands(), *getTypeConverter(),
726 [&](Type llvm1DVectorTy, ValueRange operands) {
727 OpAdaptor adaptor(operands);
728 return LLVM::ICmpOp::create(
729 rewriter, op.getLoc(), llvm1DVectorTy,
730 convertCmpPredicate<LLVM::ICmpPredicate>(op.getPredicate()),
731 adaptor.getLhs(), adaptor.getRhs());
732 },
733 rewriter);
734}
735
736//===----------------------------------------------------------------------===//
737// CmpFOpLowering
738//===----------------------------------------------------------------------===//
739
740LogicalResult
741CmpFOpLowering::matchAndRewrite(arith::CmpFOp op, OpAdaptor adaptor,
742 ConversionPatternRewriter &rewriter) const {
743 if (LLVM::detail::isUnsupportedFloatingPointType(*this->getTypeConverter(),
744 op.getLhs().getType()))
745 return rewriter.notifyMatchFailure(op, "unsupported floating point type");
746
747 Type operandType = adaptor.getLhs().getType();
748 Type resultType = op.getResult().getType();
749 LLVM::FastmathFlags fmf =
750 arith::convertArithFastMathFlagsToLLVM(op.getFastmath());
751
752 // Handle the scalar and 1D vector cases.
753 if (!isa<LLVM::LLVMArrayType>(operandType)) {
754 rewriter.replaceOpWithNewOp<LLVM::FCmpOp>(
755 op, typeConverter->convertType(resultType),
756 convertCmpPredicate<LLVM::FCmpPredicate>(op.getPredicate()),
757 adaptor.getLhs(), adaptor.getRhs(), fmf);
758 return success();
759 }
760
761 if (!isa<VectorType>(resultType))
762 return rewriter.notifyMatchFailure(op, "expected vector result type");
763
764 return LLVM::detail::handleMultidimensionalVectors(
765 op.getOperation(), adaptor.getOperands(), *getTypeConverter(),
766 [&](Type llvm1DVectorTy, ValueRange operands) {
767 OpAdaptor adaptor(operands);
768 return LLVM::FCmpOp::create(
769 rewriter, op.getLoc(), llvm1DVectorTy,
770 convertCmpPredicate<LLVM::FCmpPredicate>(op.getPredicate()),
771 adaptor.getLhs(), adaptor.getRhs(), fmf);
772 },
773 rewriter);
774}
775
776//===----------------------------------------------------------------------===//
777// SelectOpOneToNLowering
778//===----------------------------------------------------------------------===//
779
780/// Pattern for arith.select where the true/false values lower to multiple
781/// SSA values (1:N conversion). This pattern generates multiple arith.select
782/// than can be lowered by the 1:1 arith.select pattern.
783LogicalResult SelectOpOneToNLowering::matchAndRewrite(
784 arith::SelectOp op, Adaptor adaptor,
785 ConversionPatternRewriter &rewriter) const {
786 // In case of a 1:1 conversion, the 1:1 pattern will match.
787 if (llvm::hasSingleElement(adaptor.getTrueValue()))
788 return rewriter.notifyMatchFailure(
789 op, "not a 1:N conversion, 1:1 pattern will match");
790 if (!op.getCondition().getType().isInteger(1))
791 return rewriter.notifyMatchFailure(op,
792 "non-i1 conditions are not supported");
793 SmallVector<Value> results;
794 for (auto [trueValue, falseValue] :
795 llvm::zip_equal(adaptor.getTrueValue(), adaptor.getFalseValue()))
796 results.push_back(arith::SelectOp::create(
797 rewriter, op.getLoc(), op.getCondition(), trueValue, falseValue));
798 rewriter.replaceOpWithMultiple(op, {results});
799 return success();
800}
801
802//===----------------------------------------------------------------------===//
803// Pass Definition
804//===----------------------------------------------------------------------===//
805
806namespace {
807struct ArithToLLVMConversionPass
808 : public impl::ArithToLLVMConversionPassBase<ArithToLLVMConversionPass> {
809 using Base::Base;
810
811 void runOnOperation() override {
812 LLVMConversionTarget target(getContext());
813 RewritePatternSet patterns(&getContext());
814
815 const auto &dataLayoutAnalysis = getAnalysis<DataLayoutAnalysis>();
816 LowerToLLVMOptions options(&getContext(),
817 dataLayoutAnalysis.getAtOrAbove(getOperation()));
818 if (indexBitwidth != kDeriveIndexBitwidthFromDataLayout)
819 options.overrideIndexBitwidth(indexBitwidth);
820
821 LLVMTypeConverter converter(&getContext(), options, &dataLayoutAnalysis);
822 arith::populateCeilFloorDivExpandOpsPatterns(patterns);
823 arith::populateArithToLLVMConversionPatterns(converter, patterns);
824
825 if (failed(applyPartialConversion(getOperation(), target,
826 std::move(patterns))))
827 signalPassFailure();
828 }
829};
830} // namespace
831
832//===----------------------------------------------------------------------===//
833// ConvertToLLVMPatternInterface implementation
834//===----------------------------------------------------------------------===//
835
836namespace {
837/// Implement the interface to convert MemRef to LLVM.
838struct ArithToLLVMDialectInterface : public ConvertToLLVMPatternInterface {
839 ArithToLLVMDialectInterface(Dialect *dialect)
840 : ConvertToLLVMPatternInterface(dialect) {}
841
842 void loadDependentDialects(MLIRContext *context) const final {
843 context->loadDialect<LLVM::LLVMDialect>();
844 }
845
846 /// Hook for derived dialect interface to provide conversion patterns
847 /// and mark dialect legal for the conversion target.
848 void populateConvertToLLVMConversionPatterns(
849 ConversionTarget &target, LLVMTypeConverter &typeConverter,
850 RewritePatternSet &patterns) const final {
851 arith::populateCeilFloorDivExpandOpsPatterns(patterns);
852 arith::populateArithToLLVMConversionPatterns(typeConverter, patterns);
853 }
854};
855} // namespace
856
857void mlir::arith::registerConvertArithToLLVMInterface(
858 DialectRegistry &registry) {
859 registry.addExtension(+[](MLIRContext *ctx, arith::ArithDialect *dialect) {
860 dialect->addInterfaces<ArithToLLVMDialectInterface>();
861 });
862}
863
864//===----------------------------------------------------------------------===//
865// Pattern Population
866//===----------------------------------------------------------------------===//
867
868void mlir::arith::populateArithToLLVMConversionPatterns(
869 const LLVMTypeConverter &converter, RewritePatternSet &patterns) {
870
871 // Set a higher pattern benefit for IdentityBitcastLowering so it will run
872 // before BitcastOpLowering.
873 patterns.add<IdentityBitcastLowering>(converter, patterns.getContext(),
874 /*patternBenefit*/ 10);
875
876 // clang-format off
877 patterns.add<
878 AddFOpLowering,
879 ConstrainedAddFOpLowering,
880 AddIOpLowering,
881 AndIOpLowering,
882 AddUIExtendedOpLowering,
883 SubUIExtendedOpLowering,
884 BitcastOpLowering,
885 ConstantOpLowering,
886 CmpFOpLowering,
887 CmpIOpLowering,
888 DivFOpLowering,
889 ConstrainedDivFOpLowering,
890 DivSIOpLowering,
891 DivUIOpLowering,
892 ExtFOpLowering,
893 ExtSIOpLowering,
894 ExtUIOpLowering,
895 ConvertFOpLowering,
896 FPToSIOpLowering,
897 FPToUIOpLowering,
898 IndexCastOpSILowering,
899 IndexCastOpUILowering,
900 MaximumFOpLowering,
901 MaxNumFOpLowering,
902 MaximumNumFOpLowering,
903 MaxSIOpLowering,
904 MaxUIOpLowering,
905 MinimumFOpLowering,
906 MinNumFOpLowering,
907 MinimumNumFOpLowering,
908 MinSIOpLowering,
909 MinUIOpLowering,
910 MulFOpLowering,
911 ConstrainedMulFOpLowering,
912 MulIOpLowering,
913 MulSIExtendedOpLowering,
914 MulUIExtendedOpLowering,
915 NegFOpLowering,
916 OrIOpLowering,
917 RemFOpLowering,
918 RemSIOpLowering,
919 RemUIOpLowering,
920 SelectOpLowering,
921 SelectOpOneToNLowering,
922 ShLIOpLowering,
923 ShRSIOpLowering,
924 ShRUIOpLowering,
925 SIToFPOpLowering,
926 SubFOpLowering,
927 ConstrainedSubFOpLowering,
928 SubIOpLowering,
929 TruncFOpLowering,
930 ConstrainedTruncFOpLowering,
931 TruncIOpLowering,
932 UIToFPOpLowering,
933 XOrIOpLowering
934 >(converter);
935 // clang-format on
936}
success
return success()
convertCmpPredicate
static LLVMPredType convertCmpPredicate(PredType pred)
Definition ArithToLLVM.cpp:702
convertConstantValue
static TypedAttr convertConstantValue(TypedAttr attr, Type resultType)
Retypes attr for a llvm.mlir.constant of resultType.
Definition ArithToLLVM.cpp:395
getContext
b getContext())
options
static llvm::ManagedStatic< PassManagerOptions > options
Definition PassManagerOptions.cpp:89
mlir::ConvertOpToLLVMPattern
Utility class for operation conversions targeting the LLVM dialect that match exactly one source oper...
Definition Pattern.h:233
mlir::ConvertOpToLLVMPattern::ConvertOpToLLVMPattern
ConvertOpToLLVMPattern(const LLVMTypeConverter &typeConverter, PatternBenefit benefit=1)
Definition Pattern.h:239
mlir::ConvertOpToLLVMPattern::OneToNOpAdaptor
typename SourceOp::template GenericAdaptor< ArrayRef< ValueRange > > OneToNOpAdaptor
Definition Pattern.h:236
mlir::DenseIntElementsAttr
An attribute that represents a reference to a dense integer vector or tensor object.
Definition BuiltinAttributes.h:947
mlir::DialectRegistry
The DialectRegistry maps a dialect namespace to a constructor for the matching dialect.
Definition DialectRegistry.h:150
mlir::DialectRegistry::addExtension
bool addExtension(TypeID extensionID, std::unique_ptr< DialectExtensionBase > extension)
Add the given extension to the registry.
Definition DialectRegistry.h:255
mlir::LLVMTypeConverter
Conversion from types to the LLVM IR dialect.
Definition TypeConverter.h:35
mlir::MLIRContext
MLIRContext is the top-level object for a collection of MLIR operations.
Definition MLIRContext.h:64
mlir::RewritePatternSet::getContext
MLIRContext * getContext() const
Definition PatternMatch.h:844
mlir::RewritePatternSet::add
RewritePatternSet & add(ConstructorArg &&arg, ConstructorArgs &&...args)
Add an instance of each of the pattern types 'Ts' to the pattern list with the given arguments.
Definition PatternMatch.h:868
mlir::Type
Instances of the Type class are uniqued, have an immutable identifier and an optional mutable compone...
Definition Types.h:74
mlir::Type::getIntOrFloatBitWidth
unsigned getIntOrFloatBitWidth() const
Return the bit width of an integer or a float type, assert failure on other types.
Definition Types.cpp:124
mlir::Value::getType
Type getType() const
Return the type of this value.
Definition Value.h:105
mlir::VectorConvertToLLVMPattern
Basic lowering implementation to rewrite Ops with just one result to the LLVM Dialect.
Definition VectorPattern.h:99
mlir::LLVM::detail::handleMultidimensionalVectors
LogicalResult handleMultidimensionalVectors(Operation *op, ValueRange operands, const LLVMTypeConverter &typeConverter, std::function< Value(Type, ValueRange)> createOperand, ConversionPatternRewriter &rewriter)
Definition VectorPattern.cpp:80
mlir::LLVM::detail::isUnsupportedFloatingPointType
bool isUnsupportedFloatingPointType(const TypeConverter &typeConverter, Type type)
Return "true" if the given type is an unsupported floating point type.
Definition Pattern.cpp:678
mlir::LLVM::detail::opHasUnsupportedFloatingPointTypes
bool opHasUnsupportedFloatingPointTypes(Operation *op, const TypeConverter &typeConverter)
Return "true" if the given op has any unsupported floating point types (either operands or results).
Definition Pattern.cpp:689
mlir::LLVM::isCompatibleType
bool isCompatibleType(Type type)
Returns true if the given type is compatible with the LLVM dialect.
Definition LLVMTypes.cpp:873
mlir::LLVM::getConstantElementType
Type getConstantElementType(Type type)
Determines the element type of type the way the llvm.mlir.constant verifier does, i....
Definition LLVMDialect.cpp:3567
mlir::arith::populateArithToLLVMConversionPatterns
void populateArithToLLVMConversionPatterns(const LLVMTypeConverter &converter, RewritePatternSet &patterns)
Definition ArithToLLVM.cpp:868
mlir::arith::registerConvertArithToLLVMInterface
void registerConvertArithToLLVMInterface(DialectRegistry &registry)
Definition ArithToLLVM.cpp:857
mlir::remark::failed
detail::InFlightRemark failed(Location loc, RemarkOpts opts)
Report an optimization remark that failed.
Definition Remarks.h:734
mlir
Include the generated interface declarations.
Definition ABIRewriteContext.h:32
mlir::kDeriveIndexBitwidthFromDataLayout
static constexpr unsigned kDeriveIndexBitwidthFromDataLayout
Value to pass as bitwidth for the index type when the converter is expected to derive the bitwidth fr...
Definition LoweringOptions.h:26
mlir::getElementTypeOrSelf
Type getElementTypeOrSelf(Type type)
Return the element type or return the type itself.
Definition TypeUtilities.cpp:23
Generated on for MLIR by doxygen 1.14.0