45 #include "llvm/Support/FormatVariadic.h"
47 #include "../GPUCommon/GPUOpsLowering.h"
48 #include "../GPUCommon/IndexIntrinsicsOpLowering.h"
49 #include "../GPUCommon/OpToFuncCallLowering.h"
52 #define GEN_PASS_DEF_CONVERTGPUOPSTOROCDLOPS
53 #include "mlir/Conversion/Passes.h.inc"
61 bool canBeBare =
true;
62 for (
Type type : func.getArgumentTypes())
63 if (
auto memrefTy = dyn_cast<BaseMemRefType>(type))
69 const unsigned indexBitwidth) {
71 Value zero = rewriter.
create<arith::ConstantIntOp>(loc, 0, 32);
72 Value minus1 = rewriter.
create<arith::ConstantIntOp>(loc, -1, 32);
73 Value mbcntLo = rewriter.
create<ROCDL::MbcntLoOp>(loc, int32Type,
75 Value laneId = rewriter.
create<ROCDL::MbcntHiOp>(loc, int32Type,
80 "e-p:64:64-p1:64:64-p2:32:32-p3:32:32-p4:64:64-p5:32:32-p6:32:32"
81 "-p7:160:256:256:32-p8:128:128-p9:192:256:256:32-i64:64-v16:16-v24:32-v32:"
82 "32-v48:64-v96:128-v192:256-v256:256-v512:512-v1024:1024-v2048:2048-n32:"
83 "64-S32-A5-G1-ni:7:8:9";
90 matchAndRewrite(gpu::LaneIdOp op, gpu::LaneIdOp::Adaptor adaptor,
98 Value zero = rewriter.
create<arith::ConstantIntOp>(loc, 0, 32);
99 Value minus1 = rewriter.
create<arith::ConstantIntOp>(loc, -1, 32);
106 const unsigned indexBitwidth = getTypeConverter()->getIndexTypeBitwidth();
107 if (indexBitwidth > 32) {
108 laneId = rewriter.
create<LLVM::SExtOp>(
110 }
else if (indexBitwidth < 32) {
111 laneId = rewriter.
create<LLVM::TruncOp>(
139 matchAndRewrite(gpu::ShuffleOp op, OpAdaptor adaptor,
143 if (adaptor.getValue().getType().getIntOrFloatBitWidth() != 32)
145 const unsigned indexBitwidth = getTypeConverter()->getIndexTypeBitwidth();
149 Value width = adaptor.getWidth();
150 Value zero = rewriter.
create<LLVM::ConstantOp>(loc, int32Type, 0);
151 Value negwidth = rewriter.
create<LLVM::SubOp>(loc, int32Type, zero, width);
152 Value add = rewriter.
create<LLVM::AddOp>(loc, int32Type, srcLaneId, width);
153 Value widthOrZeroIfOutside =
154 rewriter.
create<LLVM::AndOp>(loc, int32Type, add, negwidth);
159 switch (op.getMode()) {
160 case gpu::ShuffleMode::XOR:
161 dstLane = rewriter.
create<LLVM::XOrOp>(loc, int32Type, srcLaneId,
162 adaptor.getOffset());
164 case gpu::ShuffleMode::IDX:
165 dstLane = adaptor.getOffset();
170 Value isActiveSrcLane = rewriter.
create<LLVM::ICmpOp>(
171 loc, LLVM::ICmpPredicate::slt, dstLane, widthOrZeroIfOutside);
172 Value selectDstLane = rewriter.
create<LLVM::SelectOp>(loc, isActiveSrcLane,
174 Value two = rewriter.
create<LLVM::ConstantOp>(loc, int32Type, 2);
175 Value dwordAlignedDstLane =
176 rewriter.
create<LLVM::ShlOp>(loc, int32Type, selectDstLane, two);
177 Value initShflValue = adaptor.getValue();
178 if (adaptor.getValue().getType().isF32()) {
180 rewriter.
create<LLVM::BitcastOp>(loc, int32Type, initShflValue);
182 Value shflValue = rewriter.
create<ROCDL::DsBpermuteOp>(
183 loc, int32Type, dwordAlignedDstLane, initShflValue);
184 if (adaptor.getValue().getType().isF32()) {
185 shflValue = rewriter.
create<LLVM::BitcastOp>(
186 loc, adaptor.getValue().getType(), shflValue);
188 rewriter.
replaceOp(op, {shflValue, isActiveSrcLane});
194 #include "GPUToROCDL.cpp.inc"
201 struct LowerGpuOpsToROCDLOpsPass
202 :
public impl::ConvertGpuOpsToROCDLOpsBase<LowerGpuOpsToROCDLOpsPass> {
203 LowerGpuOpsToROCDLOpsPass() =
default;
204 LowerGpuOpsToROCDLOpsPass(
const std::string &chipset,
unsigned indexBitwidth,
205 bool useBarePtrCallConv,
207 if (this->chipset.getNumOccurrences() == 0)
208 this->chipset = chipset;
209 if (this->indexBitwidth.getNumOccurrences() == 0)
210 this->indexBitwidth = indexBitwidth;
211 if (this->useBarePtrCallConv.getNumOccurrences() == 0)
212 this->useBarePtrCallConv = useBarePtrCallConv;
213 if (this->runtime.getNumOccurrences() == 0)
214 this->runtime = runtime;
217 void runOnOperation()
override {
218 gpu::GPUModuleOp m = getOperation();
221 auto llvmDataLayout = m->getAttrOfType<StringAttr>(
222 LLVM::LLVMDialect::getDataLayoutAttrName());
223 if (!llvmDataLayout) {
225 m->setAttr(LLVM::LLVMDialect::getDataLayoutAttrName(), llvmDataLayout);
228 for (
auto func : m.getOps<func::FuncOp>()) {
229 func->setAttr(LLVM::LLVMDialect::getEmitCWrapperAttrName(),
234 if (failed(maybeChipset)) {
236 return signalPassFailure();
241 ctx,
DataLayout(cast<DataLayoutOpInterface>(m.getOperation())));
242 options.dataLayout = llvm::DataLayout(llvmDataLayout.getValue());
244 options.overrideIndexBitwidth(indexBitwidth);
246 if (useBarePtrCallConv) {
247 options.useBarePtrCallConv =
true;
249 m.walk([](gpu::GPUFuncOp func) ->
WalkResult {
256 "bare pointer calling convention requires all memrefs to "
257 "have static shape and use the identity map");
258 return signalPassFailure();
274 converter, [](gpu::AddressSpace space) {
276 case gpu::AddressSpace::Global:
278 case gpu::AddressSpace::Workgroup:
280 case gpu::AddressSpace::Private:
283 llvm_unreachable(
"unknown address space enum value");
302 auto reqdWorkGroupSizeAttrHelper =
303 rocdlDialect->getReqdWorkGroupSizeAttrHelper();
304 auto flatWorkGroupSizeAttrHelper =
305 rocdlDialect->getFlatWorkGroupSizeAttrHelper();
308 m.walk([&](LLVM::LLVMFuncOp op) {
309 if (reqdWorkGroupSizeAttrHelper.isAttrPresent(op)) {
310 auto blockSizes = reqdWorkGroupSizeAttrHelper.getAttr(op);
313 uint32_t flatSize = 1;
314 for (uint32_t size : blockSizes.asArrayRef()) {
317 StringAttr flatSizeAttr =
319 flatWorkGroupSizeAttrHelper.setAttr(op, flatSizeAttr);
332 target.
addIllegalOp<LLVM::CosOp, LLVM::ExpOp, LLVM::Exp2Op, LLVM::FAbsOp,
333 LLVM::FCeilOp, LLVM::FFloorOp, LLVM::FRemOp, LLVM::LogOp,
334 LLVM::Log10Op, LLVM::Log2Op, LLVM::PowOp, LLVM::SinOp,
338 target.
addLegalOp<gpu::YieldOp, gpu::GPUModuleOp, gpu::ModuleEndOp>();
341 template <
typename OpTy>
357 populateWithGenerated(patterns);
360 ROCDL::ThreadIdYOp, ROCDL::ThreadIdZOp>>(
361 converter, IndexKind::Block, IntrType::Id);
363 gpu::BlockIdOp, ROCDL::BlockIdXOp, ROCDL::BlockIdYOp, ROCDL::BlockIdZOp>>(
364 converter, IndexKind::Grid, IntrType::Id);
367 ROCDL::BlockDimYOp, ROCDL::BlockDimZOp>>(
368 converter, IndexKind::Block, IntrType::Dim);
370 gpu::GridDimOp, ROCDL::GridDimXOp, ROCDL::GridDimYOp, ROCDL::GridDimZOp>>(
371 converter, IndexKind::Grid, IntrType::Dim);
375 ROCDL::ROCDLDialect::kPrivateMemoryAddressSpace,
376 ROCDL::ROCDLDialect::kSharedMemoryAddressSpace,
377 rocdlDialect->getKernelAttrHelper().getName(),
378 rocdlDialect->getReqdWorkGroupSizeAttrHelper().getName());
388 patterns.
add<GPUShuffleOpLowering, GPULaneIdOpToROCDL>(converter);
393 std::unique_ptr<OperationPass<gpu::GPUModuleOp>>
395 unsigned indexBitwidth,
396 bool useBarePtrCallConv,
398 return std::make_unique<LowerGpuOpsToROCDLOpsPass>(
399 chipset, indexBitwidth, useBarePtrCallConv, runtime);
static MLIRContext * getContext(OpFoldResult val)
static bool canBeCalledWithBarePointers(gpu::GPUFuncOp func)
Returns true if the given gpu.func can be safely called using the bare pointer calling convention.
static constexpr StringLiteral amdgcnDataLayout
static void populateOpPatterns(LLVMTypeConverter &converter, RewritePatternSet &patterns, StringRef f32Func, StringRef f64Func)
Value getLaneId(ConversionPatternRewriter &rewriter, Location loc, const unsigned indexBitwidth)
static llvm::ManagedStatic< PassManagerOptions > options
MLIRContext * getContext() const
This class implements a pattern rewriter for use with ConversionPatterns.
void replaceOp(Operation *op, ValueRange newValues) override
PatternRewriter hook for replacing an operation.
This class describes a specific conversion target.
void addLegalOp(OperationName op)
Register the given operations as legal.
void addLegalDialect(StringRef name, Names... names)
Register the operations of the given dialects as legal.
void addIllegalDialect(StringRef name, Names... names)
Register the operations of the given dialects as illegal, i.e.
void addIllegalOp(OperationName op)
Register the given operation as illegal, i.e.
Utility class for operation conversions targeting the LLVM dialect that match exactly one source oper...
The main mechanism for performing data layout queries.
Derived class that automatically populates legalization information for different LLVM ops.
Conversion from types to the LLVM IR dialect.
static bool canConvertToBarePtr(BaseMemRefType type)
Check if a memref type can be converted to a bare pointer.
MLIRContext & getContext() const
Returns the MLIR context.
This class defines the main interface for locations in MLIR and acts as a non-nullable wrapper around...
Options to control the LLVM lowering.
MLIRContext is the top-level object for a collection of MLIR operations.
Dialect * getLoadedDialect(StringRef name)
Get a registered IR dialect with the given namespace.
Operation * create(const OperationState &state)
Creates an operation given the fields represented as an OperationState.
Location getLoc()
The source location the operation was defined or derived from.
RewritePatternSet & add(ConstructorArg &&arg, ConstructorArgs &&...args)
Add an instance of each of the pattern types 'Ts' to the pattern list with the given arguments.
Instances of the Type class are uniqued, have an immutable identifier and an optional mutable compone...
This class provides an abstraction over the different types of ranges over Values.
This class represents an instance of an SSA value in the MLIR system, representing a computable value...
A utility result that is used to signal how to proceed with an ongoing walk:
static WalkResult advance()
bool wasInterrupted() const
Returns true if the walk was interrupted.
static WalkResult interrupt()
void populateExpandBFloat16Patterns(RewritePatternSet &patterns)
Add patterns to expand Arith bf16 patterns to lower level bitcasts/shifts.
void populateArithToLLVMConversionPatterns(LLVMTypeConverter &converter, RewritePatternSet &patterns)
void populateControlFlowToLLVMConversionPatterns(LLVMTypeConverter &converter, RewritePatternSet &patterns)
Collect the patterns to convert from the ControlFlow dialect to LLVM.
Runtime
Potential runtimes for AMD GPU kernels.
Include the generated interface declarations.
void populateFuncToLLVMConversionPatterns(LLVMTypeConverter &converter, RewritePatternSet &patterns, const SymbolTable *symbolTable=nullptr)
Collect the patterns to convert from the Func dialect to LLVM.
static constexpr unsigned kDeriveIndexBitwidthFromDataLayout
Value to pass as bitwidth for the index type when the converter is expected to derive the bitwidth fr...
void populateGpuRewritePatterns(RewritePatternSet &patterns)
Collect all patterns to rewrite ops within the GPU dialect.
InFlightDiagnostic emitError(Location loc)
Utility method to emit an error message using this location.
void populateFinalizeMemRefToLLVMConversionPatterns(LLVMTypeConverter &converter, RewritePatternSet &patterns)
Collect a set of patterns to convert memory-related operations from the MemRef dialect to the LLVM di...
void configureGpuToROCDLConversionLegality(ConversionTarget &target)
Configure target to convert from the GPU dialect to ROCDL.
std::unique_ptr< OperationPass< gpu::GPUModuleOp > > createLowerGpuOpsToROCDLOpsPass(const std::string &chipset="gfx900", unsigned indexBitwidth=kDeriveIndexBitwidthFromDataLayout, bool useBarePtrCallConv=false, gpu::amd::Runtime runtime=gpu::amd::Runtime::Unknown)
Creates a pass that lowers GPU dialect operations to ROCDL counterparts.
void populateVectorToLLVMConversionPatterns(LLVMTypeConverter &converter, RewritePatternSet &patterns, bool reassociateFPReductions=false, bool force32BitVectorIndices=false)
Collect a set of patterns to convert from the Vector dialect to LLVM.
LogicalResult applyPatternsAndFoldGreedily(Region ®ion, const FrozenRewritePatternSet &patterns, GreedyRewriteConfig config=GreedyRewriteConfig(), bool *changed=nullptr)
Rewrite ops in the given region, which must be isolated from above, by repeatedly applying the highes...
void populateGpuMemorySpaceAttributeConversions(TypeConverter &typeConverter, const MemorySpaceMapping &mapping)
Populates memory space attribute conversion rules for lowering gpu.address_space to integer values.
void populateAMDGPUToROCDLConversionPatterns(LLVMTypeConverter &converter, RewritePatternSet &patterns, amdgpu::Chipset chipset)
Note: The ROCDL target does not support the LLVM bfloat type at this time and so this function will a...
auto get(MLIRContext *context, Ts &&...params)
Helper method that injects context only if needed, this helps unify some of the attribute constructio...
void populateGpuToROCDLConversionPatterns(LLVMTypeConverter &converter, RewritePatternSet &patterns, gpu::amd::Runtime runtime)
Collect a set of patterns to convert from the GPU dialect to ROCDL.
void populateMathToROCDLConversionPatterns(LLVMTypeConverter &converter, RewritePatternSet &patterns)
Populate the given list with patterns that convert from Math to ROCDL calls.
LogicalResult applyPartialConversion(ArrayRef< Operation * > ops, const ConversionTarget &target, const FrozenRewritePatternSet &patterns, ConversionConfig config=ConversionConfig())
Below we define several entry points for operation conversion.
Lowering for gpu.dynamic.shared.memory to LLVM dialect.
The lowering of gpu.printf to a call to HIP hostcalls.
The lowering of gpu.printf to a call to an external printf() function.
Rewriting that replace SourceOp with a CallOp to f32Func or f64Func or f32ApproxFunc depending on the...
Rewriting that unrolls SourceOp to scalars if it's operating on vectors.
static FailureOr< Chipset > parse(StringRef name)