MLIR
22.0.0git
|
#include "mlir/Dialect/LLVMIR/NVVMDialect.h"
#include "mlir/Conversion/ConvertToLLVM/ToLLVMInterface.h"
#include "mlir/Dialect/GPU/IR/CompilationInterfaces.h"
#include "mlir/Dialect/GPU/IR/GPUDialect.h"
#include "mlir/IR/Builders.h"
#include "mlir/IR/BuiltinAttributes.h"
#include "mlir/IR/BuiltinTypes.h"
#include "mlir/IR/Diagnostics.h"
#include "mlir/IR/DialectImplementation.h"
#include "mlir/IR/MLIRContext.h"
#include "mlir/IR/Operation.h"
#include "mlir/IR/OperationSupport.h"
#include "mlir/IR/Types.h"
#include "llvm/ADT/STLExtras.h"
#include "llvm/ADT/TypeSwitch.h"
#include "llvm/IR/IRBuilder.h"
#include "llvm/Support/Casting.h"
#include "llvm/Support/FormatVariadic.h"
#include "llvm/Support/NVPTXAddrSpace.h"
#include "llvm/Support/raw_ostream.h"
#include <cassert>
#include <optional>
#include <string>
#include "mlir/Dialect/LLVMIR/NVVMOpsDialect.cpp.inc"
#include "mlir/Dialect/LLVMIR/NVVMOpsEnums.cpp.inc"
#include "mlir/Dialect/LLVMIR/NVVMOps.cpp.inc"
#include "mlir/Dialect/LLVMIR/NVVMOpsAttributes.cpp.inc"
Go to the source code of this file.
Macros | |
#define | CP_ASYNC_ID_IMPL(mod, size, suffix) llvm::Intrinsic::nvvm_cp_async_##mod##_shared_global_##size##suffix |
#define | GET_CP_ASYNC_ID(mod, size, has_cpsize) has_cpsize ? CP_ASYNC_ID_IMPL(mod, size, _s) : CP_ASYNC_ID_IMPL(mod, size, ) |
#define | _none |
#define | CVT_F2TF32_ID_IMPL(rnd, relu, sf) |
#define | GET_CVT_F2TF32_ID(rnd, relu, sf) |
#define | GET_F32x2_TO_F6x2_ID(type, has_relu) |
#define | GET_F32x2_TO_F8X2_US_ID(rnd, has_satf) |
#define | GET_F32x2_TO_F8X2_S_ID(type, has_relu) |
#define | GET_F16x2_TO_F8X2_ID(type, has_relu) |
#define | GET_BF16X2_TO_F8X2_ID(rnd, has_satf) |
#define | TCGEN05_COMMIT_IMPL(cg, is_shared, mc) |
#define | GET_TCGEN05_COMMIT_ID(cta_group, is_shared, has_mc) |
#define | TCGEN05_CP_IMPL(shape_mc, src_fmt, cg) llvm::Intrinsic::nvvm_tcgen05_cp##shape_mc##src_fmt##cg |
#define | TCGEN05_CP_2CTA(shape_mc, src_fmt, is_2cta) |
#define | GET_TCGEN05_CP_ID(shape_mc, src_fmt, is_2cta) |
#define | GET_OP_LIST |
#define | GET_ATTRDEF_LIST |
#define | GET_OP_CLASSES |
#define | GET_ATTRDEF_CLASSES |
Functions | |
static LogicalResult | cpAsyncBulkTensorCommonVerifier (size_t tensorDims, bool isIm2Col, size_t numIm2ColOffsets, Location loc) |
static LogicalResult | verifyTMALoadParams (size_t tensorDims, size_t numIm2colOff, TMALoadMode mode, Location loc) |
static bool | isInt4PtxType (MMATypes type) |
static bool | isInt8PtxType (MMATypes type) |
static bool | isIntegerPtxType (MMATypes type) |
static std::pair< mlir::Type, unsigned > | inferMMATypeFromMNK (NVVM::MMATypes type, NVVM::MMAFrag frag, int m, int n, int k, MLIRContext *context) |
static FailureOr< int > | getAllowedSizeK (NVVM::WGMMATypes typeA) |
static LogicalResult | isAllowedWGMMADataType (NVVM::WGMMATypes typeD, NVVM::WGMMATypes typeA, NVVM::WGMMATypes typeB) |
static LogicalResult | isAllowedSizeN (int sizeN, NVVM::WGMMATypes typeA) |
static llvm::Value * | packValInto64Bits (llvm::IRBuilderBase &builder, llvm::Value *result, llvm::Value *field, unsigned sizeInBits, unsigned start) |
Packs the given field into the result . More... | |
static unsigned | isValidVectorLength (NVVM::Tcgen05LdStShape shape, unsigned vecLen) |
static void | nvvmInferResultRanges (Operation *op, Value result, ArrayRef<::mlir::ConstantIntRanges > argRanges, SetIntRangeFn setResultRanges) |
Infer the result ranges for the NVVM SpecialRangeableRegisterOp that might have ConstantRangeAttr. More... | |
static llvm::Value * | getAsPackedI32 (llvm::Value *arg, llvm::IRBuilderBase &builder) |
static llvm::Value * | getParamCastedAddr (llvm::Value *addr, llvm::IRBuilderBase &builder) |
Variables | |
static constexpr unsigned | notIntrinsic = llvm::Intrinsic::not_intrinsic |
#define _none |
Definition at line 1949 of file NVVMDialect.cpp.
#define CP_ASYNC_ID_IMPL | ( | mod, | |
size, | |||
suffix | |||
) | llvm::Intrinsic::nvvm_cp_async_##mod##_shared_global_##size##suffix |
Definition at line 1497 of file NVVMDialect.cpp.
#define CVT_F2TF32_ID_IMPL | ( | rnd, | |
relu, | |||
sf | |||
) |
Definition at line 1951 of file NVVMDialect.cpp.
#define GET_ATTRDEF_CLASSES |
Definition at line 2619 of file NVVMDialect.cpp.
#define GET_ATTRDEF_LIST |
#define GET_BF16X2_TO_F8X2_ID | ( | rnd, | |
has_satf | |||
) |
Definition at line 2037 of file NVVMDialect.cpp.
#define GET_CP_ASYNC_ID | ( | mod, | |
size, | |||
has_cpsize | |||
) | has_cpsize ? CP_ASYNC_ID_IMPL(mod, size, _s) : CP_ASYNC_ID_IMPL(mod, size, ) |
Definition at line 1500 of file NVVMDialect.cpp.
#define GET_CVT_F2TF32_ID | ( | rnd, | |
relu, | |||
sf | |||
) |
Definition at line 1955 of file NVVMDialect.cpp.
#define GET_F16x2_TO_F8X2_ID | ( | type, | |
has_relu | |||
) |
Definition at line 2021 of file NVVMDialect.cpp.
#define GET_F32x2_TO_F6x2_ID | ( | type, | |
has_relu | |||
) |
Definition at line 1976 of file NVVMDialect.cpp.
#define GET_F32x2_TO_F8X2_S_ID | ( | type, | |
has_relu | |||
) |
Definition at line 1995 of file NVVMDialect.cpp.
#define GET_F32x2_TO_F8X2_US_ID | ( | rnd, | |
has_satf | |||
) |
Definition at line 1991 of file NVVMDialect.cpp.
#define GET_OP_CLASSES |
Definition at line 2616 of file NVVMDialect.cpp.
#define GET_OP_LIST |
#define GET_TCGEN05_COMMIT_ID | ( | cta_group, | |
is_shared, | |||
has_mc | |||
) |
Definition at line 2100 of file NVVMDialect.cpp.
#define GET_TCGEN05_CP_ID | ( | shape_mc, | |
src_fmt, | |||
is_2cta | |||
) |
Definition at line 2134 of file NVVMDialect.cpp.
#define TCGEN05_COMMIT_IMPL | ( | cg, | |
is_shared, | |||
mc | |||
) |
Definition at line 2096 of file NVVMDialect.cpp.
#define TCGEN05_CP_2CTA | ( | shape_mc, | |
src_fmt, | |||
is_2cta | |||
) |
Definition at line 2130 of file NVVMDialect.cpp.
#define TCGEN05_CP_IMPL | ( | shape_mc, | |
src_fmt, | |||
cg | |||
) | llvm::Intrinsic::nvvm_tcgen05_cp##shape_mc##src_fmt##cg |
Definition at line 2127 of file NVVMDialect.cpp.
|
static |
Definition at line 57 of file NVVMDialect.cpp.
References mlir::emitError().
|
static |
Definition at line 981 of file NVVMDialect.cpp.
|
static |
Definition at line 2222 of file NVVMDialect.cpp.
|
static |
Definition at line 2271 of file NVVMDialect.cpp.
References mlir::get().
|
static |
Definition at line 810 of file NVVMDialect.cpp.
References mlir::NVVM::inferMMAType().
|
static |
Definition at line 1037 of file NVVMDialect.cpp.
|
static |
Definition at line 995 of file NVVMDialect.cpp.
|
static |
Definition at line 330 of file NVVMDialect.cpp.
Referenced by isIntegerPtxType().
|
static |
Definition at line 334 of file NVVMDialect.cpp.
Referenced by isIntegerPtxType().
|
static |
Definition at line 338 of file NVVMDialect.cpp.
References isInt4PtxType(), and isInt8PtxType().
|
static |
Definition at line 2168 of file NVVMDialect.cpp.
|
static |
Infer the result ranges for the NVVM SpecialRangeableRegisterOp that might have ConstantRangeAttr.
Definition at line 2213 of file NVVMDialect.cpp.
References mlir::Operation::getAttrOfType().
|
static |
Packs the given field
into the result
.
The result
is 64-bits and each field
can be 32-bits or narrower.
Definition at line 1452 of file NVVMDialect.cpp.
|
static |
Definition at line 116 of file NVVMDialect.cpp.
References mlir::emitError().
|
staticconstexpr |
Definition at line 48 of file NVVMDialect.cpp.