46 #include "llvm/Support/FormatVariadic.h"
48 #include "../GPUCommon/GPUOpsLowering.h"
49 #include "../GPUCommon/IndexIntrinsicsOpLowering.h"
52 #define GEN_PASS_DEF_CONVERTGPUOPSTOROCDLOPS
53 #include "mlir/Conversion/Passes.h.inc"
61 bool canBeBare =
true;
62 for (
Type type : func.getArgumentTypes())
63 if (
auto memrefTy = dyn_cast<BaseMemRefType>(type))
69 const unsigned indexBitwidth) {
71 Value zero = rewriter.
create<arith::ConstantIntOp>(loc, 0, 32);
72 Value minus1 = rewriter.
create<arith::ConstantIntOp>(loc, -1, 32);
73 Value mbcntLo = rewriter.
create<ROCDL::MbcntLoOp>(loc, int32Type,
75 Value laneId = rewriter.
create<ROCDL::MbcntHiOp>(loc, int32Type,
80 "e-p:64:64-p1:64:64-p2:32:32-p3:32:32-p4:64:64-p5:32:32-p6:32:32"
81 "-p7:160:256:256:32-p8:128:128-p9:192:256:256:32-i64:64-v16:16-v24:32-v32:"
82 "32-v48:64-v96:128-v192:256-v256:256-v512:512-v1024:1024-v2048:2048-n32:"
83 "64-S32-A5-G1-ni:7:8:9";
90 matchAndRewrite(gpu::LaneIdOp op, gpu::LaneIdOp::Adaptor adaptor,
92 auto loc = op->getLoc();
98 Value zero = rewriter.
create<arith::ConstantIntOp>(loc, 0, 32);
99 Value minus1 = rewriter.
create<arith::ConstantIntOp>(loc, -1, 32);
106 const unsigned indexBitwidth = getTypeConverter()->getIndexTypeBitwidth();
107 if (indexBitwidth > 32) {
108 laneId = rewriter.
create<LLVM::SExtOp>(
110 }
else if (indexBitwidth < 32) {
111 laneId = rewriter.
create<LLVM::TruncOp>(
139 matchAndRewrite(gpu::ShuffleOp op, OpAdaptor adaptor,
143 if (adaptor.getValue().getType().getIntOrFloatBitWidth() != 32)
145 const unsigned indexBitwidth = getTypeConverter()->getIndexTypeBitwidth();
149 Value width = adaptor.getWidth();
150 Value zero = rewriter.
create<LLVM::ConstantOp>(loc, int32Type, 0);
151 Value negwidth = rewriter.
create<LLVM::SubOp>(loc, int32Type, zero, width);
152 Value add = rewriter.
create<LLVM::AddOp>(loc, int32Type, srcLaneId, width);
153 Value widthOrZeroIfOutside =
154 rewriter.
create<LLVM::AndOp>(loc, int32Type, add, negwidth);
159 switch (op.getMode()) {
160 case gpu::ShuffleMode::DOWN:
161 dstLane = rewriter.
create<LLVM::AddOp>(loc, int32Type, srcLaneId,
162 adaptor.getOffset());
164 case gpu::ShuffleMode::XOR:
165 dstLane = rewriter.
create<LLVM::XOrOp>(loc, int32Type, srcLaneId,
166 adaptor.getOffset());
168 case gpu::ShuffleMode::IDX:
169 dstLane = adaptor.getOffset();
174 Value isActiveSrcLane = rewriter.
create<LLVM::ICmpOp>(
175 loc, LLVM::ICmpPredicate::slt, dstLane, widthOrZeroIfOutside);
176 Value selectDstLane = rewriter.
create<LLVM::SelectOp>(loc, isActiveSrcLane,
178 Value two = rewriter.
create<LLVM::ConstantOp>(loc, int32Type, 2);
179 Value dwordAlignedDstLane =
180 rewriter.
create<LLVM::ShlOp>(loc, int32Type, selectDstLane, two);
181 Value initShflValue = adaptor.getValue();
182 if (adaptor.getValue().getType().isF32()) {
184 rewriter.
create<LLVM::BitcastOp>(loc, int32Type, initShflValue);
186 Value shflValue = rewriter.
create<ROCDL::DsBpermuteOp>(
187 loc, int32Type, dwordAlignedDstLane, initShflValue);
188 if (adaptor.getValue().getType().isF32()) {
189 shflValue = rewriter.
create<LLVM::BitcastOp>(
190 loc, adaptor.getValue().getType(), shflValue);
192 rewriter.
replaceOp(op, {shflValue, isActiveSrcLane});
198 #include "GPUToROCDL.cpp.inc"
205 struct LowerGpuOpsToROCDLOpsPass
206 :
public impl::ConvertGpuOpsToROCDLOpsBase<LowerGpuOpsToROCDLOpsPass> {
207 LowerGpuOpsToROCDLOpsPass() =
default;
208 LowerGpuOpsToROCDLOpsPass(
const std::string &chipset,
unsigned indexBitwidth,
209 bool useBarePtrCallConv,
211 if (this->chipset.getNumOccurrences() == 0)
212 this->chipset = chipset;
213 if (this->indexBitwidth.getNumOccurrences() == 0)
214 this->indexBitwidth = indexBitwidth;
215 if (this->useBarePtrCallConv.getNumOccurrences() == 0)
216 this->useBarePtrCallConv = useBarePtrCallConv;
217 if (this->runtime.getNumOccurrences() == 0)
218 this->runtime = runtime;
221 void runOnOperation()
override {
222 gpu::GPUModuleOp m = getOperation();
225 auto llvmDataLayout = m->getAttrOfType<StringAttr>(
226 LLVM::LLVMDialect::getDataLayoutAttrName());
227 if (!llvmDataLayout) {
229 m->setAttr(LLVM::LLVMDialect::getDataLayoutAttrName(), llvmDataLayout);
232 for (
auto func : m.getOps<func::FuncOp>()) {
233 func->setAttr(LLVM::LLVMDialect::getEmitCWrapperAttrName(),
238 if (failed(maybeChipset)) {
240 return signalPassFailure();
245 ctx,
DataLayout(cast<DataLayoutOpInterface>(m.getOperation())));
246 options.dataLayout = llvm::DataLayout(llvmDataLayout.getValue());
248 options.overrideIndexBitwidth(indexBitwidth);
250 if (useBarePtrCallConv) {
251 options.useBarePtrCallConv =
true;
253 m.walk([](gpu::GPUFuncOp func) ->
WalkResult {
260 "bare pointer calling convention requires all memrefs to "
261 "have static shape and use the identity map");
262 return signalPassFailure();
278 converter, [](gpu::AddressSpace space) {
280 case gpu::AddressSpace::Global:
282 case gpu::AddressSpace::Workgroup:
284 case gpu::AddressSpace::Private:
287 llvm_unreachable(
"unknown address space enum value");
308 auto reqdWorkGroupSizeAttrHelper =
309 rocdlDialect->getReqdWorkGroupSizeAttrHelper();
310 auto flatWorkGroupSizeAttrHelper =
311 rocdlDialect->getFlatWorkGroupSizeAttrHelper();
314 m.walk([&](LLVM::LLVMFuncOp op) {
315 if (reqdWorkGroupSizeAttrHelper.isAttrPresent(op)) {
316 auto blockSizes = reqdWorkGroupSizeAttrHelper.getAttr(op);
319 uint32_t flatSize = 1;
320 for (uint32_t size : blockSizes.asArrayRef()) {
323 StringAttr flatSizeAttr =
325 flatWorkGroupSizeAttrHelper.setAttr(op, flatSizeAttr);
338 target.
addIllegalOp<LLVM::CosOp, LLVM::ExpOp, LLVM::Exp2Op, LLVM::FCeilOp,
339 LLVM::FFloorOp, LLVM::FRemOp, LLVM::LogOp, LLVM::Log10Op,
340 LLVM::Log2Op, LLVM::PowOp, LLVM::SinOp>();
343 return any_of(op->getOperandTypes(), llvm::IsaPred<Float32Type>);
346 target.
addLegalOp<gpu::YieldOp, gpu::GPUModuleOp>();
360 ROCDL::ThreadIdYOp, ROCDL::ThreadIdZOp>>(
361 converter, IndexKind::Block, IntrType::Id);
363 gpu::BlockIdOp, ROCDL::BlockIdXOp, ROCDL::BlockIdYOp, ROCDL::BlockIdZOp>>(
364 converter, IndexKind::Grid, IntrType::Id);
367 ROCDL::BlockDimYOp, ROCDL::BlockDimZOp>>(
368 converter, IndexKind::Block, IntrType::Dim);
370 gpu::GridDimOp, ROCDL::GridDimXOp, ROCDL::GridDimYOp, ROCDL::GridDimZOp>>(
371 converter, IndexKind::Grid, IntrType::Dim);
376 ROCDL::ROCDLDialect::kPrivateMemoryAddressSpace,
377 ROCDL::ROCDLDialect::kSharedMemoryAddressSpace,
378 rocdlDialect->getKernelAttrHelper().getName(),
379 rocdlDialect->getReqdWorkGroupSizeAttrHelper().getName()});
389 patterns.add<GPUShuffleOpLowering, GPULaneIdOpToROCDL>(converter);
394 std::unique_ptr<OperationPass<gpu::GPUModuleOp>>
396 unsigned indexBitwidth,
397 bool useBarePtrCallConv,
399 return std::make_unique<LowerGpuOpsToROCDLOpsPass>(
400 chipset, indexBitwidth, useBarePtrCallConv, runtime);
static MLIRContext * getContext(OpFoldResult val)
static bool canBeCalledWithBarePointers(gpu::GPUFuncOp func)
Returns true if the given gpu.func can be safely called using the bare pointer calling convention.
static constexpr StringLiteral amdgcnDataLayout
Value getLaneId(ConversionPatternRewriter &rewriter, Location loc, const unsigned indexBitwidth)
static llvm::ManagedStatic< PassManagerOptions > options
MLIRContext * getContext() const
This class implements a pattern rewriter for use with ConversionPatterns.
void replaceOp(Operation *op, ValueRange newValues) override
Replace the given operation with the new values.
This class describes a specific conversion target.
void addLegalOp(OperationName op)
Register the given operations as legal.
void addLegalDialect(StringRef name, Names... names)
Register the operations of the given dialects as legal.
void addDynamicallyLegalOp(OperationName op, const DynamicLegalityCallbackFn &callback)
Register the given operation as dynamically legal and set the dynamic legalization callback to the on...
void addIllegalDialect(StringRef name, Names... names)
Register the operations of the given dialects as illegal, i.e.
void addIllegalOp(OperationName op)
Register the given operation as illegal, i.e.
Utility class for operation conversions targeting the LLVM dialect that match exactly one source oper...
The main mechanism for performing data layout queries.
Derived class that automatically populates legalization information for different LLVM ops.
Conversion from types to the LLVM IR dialect.
static bool canConvertToBarePtr(BaseMemRefType type)
Check if a memref type can be converted to a bare pointer.
MLIRContext & getContext() const
Returns the MLIR context.
This class defines the main interface for locations in MLIR and acts as a non-nullable wrapper around...
Options to control the LLVM lowering.
MLIRContext is the top-level object for a collection of MLIR operations.
Dialect * getLoadedDialect(StringRef name)
Get a registered IR dialect with the given namespace.
Operation * create(const OperationState &state)
Creates an operation given the fields represented as an OperationState.
Operation is the basic unit of execution within MLIR.
Instances of the Type class are uniqued, have an immutable identifier and an optional mutable compone...
This class provides an abstraction over the different types of ranges over Values.
This class represents an instance of an SSA value in the MLIR system, representing a computable value...
A utility result that is used to signal how to proceed with an ongoing walk:
static WalkResult advance()
bool wasInterrupted() const
Returns true if the walk was interrupted.
static WalkResult interrupt()
void populateExpandBFloat16Patterns(RewritePatternSet &patterns)
Add patterns to expand Arith bf16 patterns to lower level bitcasts/shifts.
void populateArithToLLVMConversionPatterns(const LLVMTypeConverter &converter, RewritePatternSet &patterns)
void populateAssertToLLVMConversionPattern(const LLVMTypeConverter &converter, RewritePatternSet &patterns, bool abortOnFailure=true)
Populate the cf.assert to LLVM conversion pattern.
void populateControlFlowToLLVMConversionPatterns(const LLVMTypeConverter &converter, RewritePatternSet &patterns)
Collect the patterns to convert from the ControlFlow dialect to LLVM.
Runtime
Potential runtimes for AMD GPU kernels.
Include the generated interface declarations.
void populateMathToLLVMConversionPatterns(const LLVMTypeConverter &converter, RewritePatternSet &patterns, bool approximateLog1p=true)
static constexpr unsigned kDeriveIndexBitwidthFromDataLayout
Value to pass as bitwidth for the index type when the converter is expected to derive the bitwidth fr...
LogicalResult applyPatternsGreedily(Region ®ion, const FrozenRewritePatternSet &patterns, GreedyRewriteConfig config=GreedyRewriteConfig(), bool *changed=nullptr)
Rewrite ops in the given region, which must be isolated from above, by repeatedly applying the highes...
void populateGpuToROCDLConversionPatterns(const LLVMTypeConverter &converter, RewritePatternSet &patterns, gpu::amd::Runtime runtime)
Collect a set of patterns to convert from the GPU dialect to ROCDL.
void populateGpuRewritePatterns(RewritePatternSet &patterns)
Collect all patterns to rewrite ops within the GPU dialect.
InFlightDiagnostic emitError(Location loc)
Utility method to emit an error message using this location.
void populateFinalizeMemRefToLLVMConversionPatterns(const LLVMTypeConverter &converter, RewritePatternSet &patterns)
Collect a set of patterns to convert memory-related operations from the MemRef dialect to the LLVM di...
void populateAMDGPUToROCDLConversionPatterns(const LLVMTypeConverter &converter, RewritePatternSet &patterns, amdgpu::Chipset chipset)
Note: The ROCDL target does not support the LLVM bfloat type at this time and so this function will a...
void configureGpuToROCDLConversionLegality(ConversionTarget &target)
Configure target to convert from the GPU dialect to ROCDL.
std::unique_ptr< OperationPass< gpu::GPUModuleOp > > createLowerGpuOpsToROCDLOpsPass(const std::string &chipset="gfx900", unsigned indexBitwidth=kDeriveIndexBitwidthFromDataLayout, bool useBarePtrCallConv=false, gpu::amd::Runtime runtime=gpu::amd::Runtime::Unknown)
Creates a pass that lowers GPU dialect operations to ROCDL counterparts.
const FrozenRewritePatternSet & patterns
void populateGpuMemorySpaceAttributeConversions(TypeConverter &typeConverter, const MemorySpaceMapping &mapping)
Populates memory space attribute conversion rules for lowering gpu.address_space to integer values.
auto get(MLIRContext *context, Ts &&...params)
Helper method that injects context only if needed, this helps unify some of the attribute constructio...
void populateVectorToLLVMConversionPatterns(const LLVMTypeConverter &converter, RewritePatternSet &patterns, bool reassociateFPReductions=false, bool force32BitVectorIndices=false)
Collect a set of patterns to convert from the Vector dialect to LLVM.
void populateFuncToLLVMConversionPatterns(const LLVMTypeConverter &converter, RewritePatternSet &patterns, const SymbolTable *symbolTable=nullptr)
Collect the patterns to convert from the Func dialect to LLVM.
LogicalResult applyPartialConversion(ArrayRef< Operation * > ops, const ConversionTarget &target, const FrozenRewritePatternSet &patterns, ConversionConfig config=ConversionConfig())
Below we define several entry points for operation conversion.
void populateMathToROCDLConversionPatterns(const LLVMTypeConverter &converter, RewritePatternSet &patterns)
Populate the given list with patterns that convert from Math to ROCDL calls.
Lowering for gpu.dynamic.shared.memory to LLVM dialect.
The lowering of gpu.printf to a call to HIP hostcalls.
The lowering of gpu.printf to a call to an external printf() function.
static FailureOr< Chipset > parse(StringRef name)
Parses the chipset version string and returns the chipset on success, and failure otherwise.