11 #include "../GPUCommon/GPUOpsLowering.h"
34 #include "llvm/ADT/TypeSwitch.h"
35 #include "llvm/Support/FormatVariadic.h"
40 #define GEN_PASS_DEF_CONVERTGPUOPSTOLLVMSPVOPS
41 #include "mlir/Conversion/Passes.h.inc"
51 Type resultType,
bool isMemNone,
53 auto func = dyn_cast_or_null<LLVM::LLVMFuncOp>(
57 func = b.
create<LLVM::LLVMFuncOp>(
58 symbolTable->
getLoc(), name,
60 func.setCConv(LLVM::cconv::CConv::SPIR_FUNC);
61 func.setNoUnwind(
true);
62 func.setWillReturn(
true);
66 constexpr
auto noModRef = mlir::LLVM::ModRefInfo::NoModRef;
67 auto memAttr = b.
getAttr<LLVM::MemoryEffectsAttr>(
70 func.setMemoryEffectsAttr(memAttr);
73 func.setConvergent(isConvergent);
80 LLVM::LLVMFuncOp func,
82 auto call = rewriter.
create<LLVM::CallOp>(loc, func, args);
83 call.setCConv(func.getCConv());
84 call.setConvergentAttr(func.getConvergentAttr());
85 call.setNoUnwindAttr(func.getNoUnwindAttr());
86 call.setWillReturnAttr(func.getWillReturnAttr());
87 call.setMemoryEffectsAttr(func.getMemoryEffectsAttr());
107 matchAndRewrite(gpu::BarrierOp op, OpAdaptor adaptor,
109 constexpr StringLiteral funcName =
"_Z7barrierj";
112 assert(moduleOp &&
"Expecting module");
113 Type flagTy = rewriter.getI32Type();
114 Type voidTy = rewriter.getType<LLVM::LLVMVoidType>();
115 LLVM::LLVMFuncOp func =
121 constexpr int64_t localMemFenceFlag = 1;
124 rewriter.create<LLVM::ConstantOp>(loc, flagTy, localMemFenceFlag);
143 LaunchConfigConversion(StringRef funcName, StringRef rootOpName,
148 funcName(funcName) {}
150 virtual gpu::Dimension getDimension(
Operation *op)
const = 0;
156 assert(moduleOp &&
"Expecting module");
157 Type dimTy = rewriter.getI32Type();
158 Type indexTy = getTypeConverter()->getIndexType();
164 gpu::Dimension dim = getDimension(op);
165 Value dimVal = rewriter.create<LLVM::ConstantOp>(loc, dimTy,
166 static_cast<int64_t
>(dim));
174 template <
typename SourceOp>
175 struct LaunchConfigOpConversion final : LaunchConfigConversion {
176 static StringRef getFuncName();
180 : LaunchConfigConversion(getFuncName(), SourceOp::getOperationName(),
184 gpu::Dimension getDimension(
Operation *op)
const final {
185 return cast<SourceOp>(op).getDimension();
190 StringRef LaunchConfigOpConversion<gpu::BlockIdOp>::getFuncName() {
191 return "_Z12get_group_idj";
195 StringRef LaunchConfigOpConversion<gpu::GridDimOp>::getFuncName() {
196 return "_Z14get_num_groupsj";
200 StringRef LaunchConfigOpConversion<gpu::BlockDimOp>::getFuncName() {
201 return "_Z14get_local_sizej";
205 StringRef LaunchConfigOpConversion<gpu::ThreadIdOp>::getFuncName() {
206 return "_Z12get_local_idj";
210 StringRef LaunchConfigOpConversion<gpu::GlobalIdOp>::getFuncName() {
211 return "_Z13get_global_idj";
230 static StringRef getBaseName(gpu::ShuffleMode mode) {
232 case gpu::ShuffleMode::IDX:
233 return "sub_group_shuffle";
234 case gpu::ShuffleMode::XOR:
235 return "sub_group_shuffle_xor";
236 case gpu::ShuffleMode::UP:
237 return "sub_group_shuffle_up";
238 case gpu::ShuffleMode::DOWN:
239 return "sub_group_shuffle_down";
241 llvm_unreachable(
"Unhandled shuffle mode");
244 static std::optional<StringRef> getTypeMangling(
Type type) {
246 .Case<Float16Type>([](
auto) {
return "Dhj"; })
247 .Case<Float32Type>([](
auto) {
return "fj"; })
248 .Case<Float64Type>([](
auto) {
return "dj"; })
249 .Case<IntegerType>([](
auto intTy) -> std::optional<StringRef> {
250 switch (intTy.getWidth()) {
262 .Default([](
auto) {
return std::nullopt; });
265 static std::optional<std::string> getFuncName(gpu::ShuffleOp op) {
266 StringRef baseName = getBaseName(op.getMode());
267 std::optional<StringRef> typeMangling = getTypeMangling(op.getType(0));
270 return llvm::formatv(
"_Z{0}{1}{2}", baseName.size(), baseName,
271 typeMangling.value());
275 static int getSubgroupSize(
Operation *op) {
281 static bool hasValidWidth(gpu::ShuffleOp op) {
283 Value width = op.getWidth();
285 val == getSubgroupSize(op);
289 matchAndRewrite(gpu::ShuffleOp op, OpAdaptor adaptor,
291 if (!hasValidWidth(op))
292 return rewriter.notifyMatchFailure(
293 op,
"shuffle width and subgroup size mismatch");
295 std::optional<std::string> funcName = getFuncName(op);
297 return rewriter.notifyMatchFailure(op,
"unsupported value type");
300 assert(moduleOp &&
"Expecting module");
301 Type valueType = adaptor.getValue().getType();
302 Type offsetType = adaptor.getOffset().getType();
303 Type resultType = valueType;
305 moduleOp, funcName.value(), {valueType, offsetType}, resultType,
309 std::array<Value, 2> args{adaptor.getValue(), adaptor.getOffset()};
313 rewriter.create<LLVM::ConstantOp>(loc, rewriter.getI1Type(),
true);
314 rewriter.replaceOp(op, {result, trueVal});
323 struct GPUToLLVMSPVConversionPass final
324 : impl::ConvertGpuOpsToLLVMSPVOpsBase<GPUToLLVMSPVConversionPass> {
327 void runOnOperation() final {
333 options.overrideIndexBitwidth(indexBitwidth);
338 target.addIllegalOp<gpu::BarrierOp, gpu::BlockDimOp, gpu::BlockIdOp,
339 gpu::GPUFuncOp, gpu::GlobalIdOp, gpu::GridDimOp,
340 gpu::ReturnOp, gpu::ShuffleOp, gpu::ThreadIdOp>();
346 std::move(patterns))))
359 gpuAddressSpaceToOCLAddressSpace(gpu::AddressSpace addressSpace) {
369 LaunchConfigOpConversion<gpu::BlockIdOp>,
370 LaunchConfigOpConversion<gpu::GridDimOp>,
371 LaunchConfigOpConversion<gpu::BlockDimOp>,
372 LaunchConfigOpConversion<gpu::ThreadIdOp>,
373 LaunchConfigOpConversion<gpu::GlobalIdOp>>(typeConverter);
375 unsigned privateAddressSpace =
376 gpuAddressSpaceToOCLAddressSpace(gpu::AddressSpace::Private);
377 unsigned localAddressSpace =
378 gpuAddressSpaceToOCLAddressSpace(gpu::AddressSpace::Workgroup);
379 OperationName llvmFuncOpName(LLVM::LLVMFuncOp::getOperationName(), context);
380 StringAttr kernelBlockSizeAttributeName =
381 LLVM::LLVMFuncOp::getReqdWorkGroupSizeAttrName(llvmFuncOpName);
385 privateAddressSpace, localAddressSpace,
386 {}, kernelBlockSizeAttributeName,
387 LLVM::CConv::SPIR_KERNEL, LLVM::CConv::SPIR_FUNC,
393 gpuAddressSpaceToOCLAddressSpace);
static LLVM::CallOp createSPIRVBuiltinCall(Location loc, ConversionPatternRewriter &rewriter, LLVM::LLVMFuncOp func, ValueRange args)
static LLVM::LLVMFuncOp lookupOrCreateSPIRVFn(Operation *symbolTable, StringRef name, ArrayRef< Type > paramTypes, Type resultType, bool isMemNone, bool isConvergent)
static MLIRContext * getContext(OpFoldResult val)
static llvm::ManagedStatic< PassManagerOptions > options
Attr getAttr(Args &&...args)
Get or construct an instance of the attribute Attr with provided arguments.
This class implements a pattern rewriter for use with ConversionPatterns.
Utility class for operation conversions targeting the LLVM dialect that match exactly one source oper...
ConvertOpToLLVMPattern(const LLVMTypeConverter &typeConverter, PatternBenefit benefit=1)
Base class for operation conversions targeting the LLVM IR dialect.
Derived class that automatically populates legalization information for different LLVM ops.
Conversion from types to the LLVM IR dialect.
MLIRContext & getContext() const
Returns the MLIR context.
This class defines the main interface for locations in MLIR and acts as a non-nullable wrapper around...
Options to control the LLVM lowering.
MLIRContext is the top-level object for a collection of MLIR operations.
This class helps build Operations.
Operation * create(const OperationState &state)
Creates an operation given the fields represented as an OperationState.
A trait used to provide symbol table functionalities to a region operation.
Operation is the basic unit of execution within MLIR.
Location getLoc()
The source location the operation was defined or derived from.
Region & getRegion(unsigned index)
Returns the region held by this operation at position 'index'.
Operation * getParentWithTrait()
Returns the closest surrounding parent operation with trait Trait.
This class represents the benefit of a pattern match in a unitless scheme that ranges from 0 (very li...
RewritePatternSet & add(ConstructorArg &&arg, ConstructorArgs &&...args)
Add an instance of each of the pattern types 'Ts' to the pattern list with the given arguments.
static Operation * lookupSymbolIn(Operation *op, StringAttr symbol)
Returns the operation registered with the given symbol name with the regions of 'symbolTableOp'.
Instances of the Type class are uniqued, have an immutable identifier and an optional mutable compone...
This class provides an abstraction over the different types of ranges over Values.
This class represents an instance of an SSA value in the MLIR system, representing a computable value...
ResourceLimitsAttr getResourceLimits() const
Returns the target resource limits.
TargetEnvAttr lookupTargetEnvOrDefault(Operation *op)
Queries the target environment recursively from enclosing symbol table ops containing the given op or...
Include the generated interface declarations.
bool matchPattern(Value value, const Pattern &pattern)
Entry point for matching a pattern over a Value.
detail::constant_int_value_binder m_ConstantInt(IntegerAttr::ValueType *bind_value)
Matches a constant holding a scalar/vector/tensor integer (splat) and writes the integer value to bin...
unsigned storageClassToAddressSpace(spirv::ClientAPI clientAPI, spirv::StorageClass storageClass)
static constexpr unsigned kDeriveIndexBitwidthFromDataLayout
Value to pass as bitwidth for the index type when the converter is expected to derive the bitwidth fr...
spirv::StorageClass addressSpaceToStorageClass(gpu::AddressSpace addressSpace)
void populateGpuToLLVMSPVConversionPatterns(LLVMTypeConverter &converter, RewritePatternSet &patterns)
void populateGpuMemorySpaceAttributeConversions(TypeConverter &typeConverter, const MemorySpaceMapping &mapping)
Populates memory space attribute conversion rules for lowering gpu.address_space to integer values.
auto get(MLIRContext *context, Ts &&...params)
Helper method that injects context only if needed, this helps unify some of the attribute constructio...
LogicalResult applyPartialConversion(ArrayRef< Operation * > ops, const ConversionTarget &target, const FrozenRewritePatternSet &patterns, ConversionConfig config=ConversionConfig())
Below we define several entry points for operation conversion.