#include "mlir/Dialect/LLVMIR/NVVMDialect.h"
#include "IR/NVVMOps.h"
#include "mlir/Conversion/ConvertToLLVM/ToLLVMInterface.h"
#include "mlir/Dialect/GPU/IR/CompilationInterfaces.h"
#include "mlir/IR/Builders.h"
#include "mlir/IR/BuiltinAttributes.h"
#include "mlir/IR/BuiltinTypes.h"
#include "mlir/IR/Diagnostics.h"
#include "mlir/IR/DialectImplementation.h"
#include "mlir/IR/MLIRContext.h"
#include "mlir/IR/Operation.h"
#include "mlir/IR/OperationSupport.h"
#include "mlir/IR/Types.h"
#include "mlir/Target/LLVMIR/ModuleTranslation.h"
#include "mlir/Transforms/InliningUtils.h"
#include "llvm/ADT/STLExtras.h"
#include "llvm/ADT/StringExtras.h"
#include "llvm/ADT/TypeSwitch.h"
#include "llvm/IR/ConstantRange.h"
#include "llvm/IR/IRBuilder.h"
#include "llvm/IR/NVVMIntrinsicUtils.h"
#include "llvm/Support/Casting.h"
#include "llvm/Support/FormatVariadic.h"
#include "llvm/Support/MathExtras.h"
#include "llvm/Support/NVPTXAddrSpace.h"
#include "llvm/Support/raw_ostream.h"
#include <array>
#include <cassert>
#include <cmath>
#include <optional>
#include <string>
#include <utility>
#include "mlir/Dialect/LLVMIR/NVVMOpsDialect.cpp.inc"

Classes
struct	ConvertFsubToFnegFadd

Macros
#define	CP_ASYNC_ID_IMPL(mod, size, suffix)
#define	GET_CP_ASYNC_ID(mod, size, has_cpsize)
#define	_none
#define	CVT_F2TF32_ID_IMPL(rnd, relu, sf)
#define	GET_CVT_F2TF32_ID(rnd, relu, sf)
#define	TCGEN05_CP_IMPL(shape_mc, src_fmt, cg)
#define	TCGEN05_CP_2CTA(shape_mc, src_fmt, is_2cta)
#define	GET_TCGEN05_CP_ID(shape_mc, src_fmt, is_2cta)
#define	TCGEN05LDRED(SHAPE, NUM, TYPE)

Functions
static bool	isPtrInAddrSpace (mlir::Value ptr, NVVMMemorySpace targetAS)
static bool	isPtrInGenericSpace (mlir::Value ptr)
static bool	isPtrInSharedCTASpace (mlir::Value ptr)
static bool	isPtrInSharedClusterSpace (mlir::Value ptr)
static llvm::Value *	castPtrToAddrSpace (llvm::IRBuilderBase &builder, llvm::Value *ptr, NVVMMemorySpace targetAS)
static llvm::nvvm::CTAGroupKind	getNVVMCtaGroupKind (NVVM::CTAGroupKind ctaGroup)
template<typename AttrTy>
static ParseResult	parseEnumKeyword (OpAsmParser &parser, AttrTy &attr)
static LogicalResult	cpAsyncBulkTensorCommonVerifier (size_t tensorDims, bool isIm2Col, size_t numIm2ColOffsets, Location loc)
LogicalResult	CpAsyncBulkTensorOverrideAddrCommonVerifier (OperandRange coordinates, OperandRange tensorSize, OperandRange lowerStride, Value upperStride, bool isTile, Location loc)
static LogicalResult	verifyTMALoadParams (size_t tensorDims, size_t numIm2colOff, TMALoadMode mode, Location loc)
static LogicalResult	verifyMBarrierArriveLikeOp (Operation *op, Value addr, NVVM::MemScopeKind scope, Value retVal=nullptr, Value multicastMask=nullptr)
static LogicalResult	inferMBarrierArriveResultTypes (MLIRContext *context, Value addr, SmallVectorImpl< Type > &inferredReturnTypes)
	Only shared_cluster (ptr<7>) produces zero results; all other address spaces (including generic) return i64.
static bool	isCompatibleReturnTypesOptionalResult (TypeRange inferred, TypeRange actual)
	For ops with optional results, allow the user to omit the result even when inference would produce one.
static LogicalResult	verifyConvertF32x2ToFP16x2Op (Twine dstType, FPRoundingMode rnd, bool hasRandomBits, bool hasPzo, Operation *op)
static bool	isInt4PtxType (MMATypes type)
static bool	isInt8PtxType (MMATypes type)
static bool	isIntegerPtxType (MMATypes type)
template<typename AttrTy>
static void	printMmaProperty (OpAsmPrinter &printer, bool &isFirst, StringRef keyword, AttrTy value)
template<typename AttrTy>
static void	printMmaEnumProperty (OpAsmPrinter &printer, bool &isFirst, StringRef keyword, AttrTy value)
static void	printMmaUnitProperty (OpAsmPrinter &printer, bool &isFirst, StringRef keyword)
template<typename AttrTy>
static ParseResult	parseMmaPropertyValue (OpAsmParser &parser, NamedAttrList &attributes, StringRef name)
template<typename AttrTy>
static ParseResult	parseMmaEnumPropertyValue (OpAsmParser &parser, NamedAttrList &attributes, StringRef name)
static bool	isMmaPropertyName (StringRef name)
static ParseResult	parseMmaProperties (OpAsmParser &parser, NamedAttrList &attributes, ArrayRef< StringRef > allowedKeywords, ArrayRef< StringRef > requiredProperties)
static void	printOperandList (OpAsmPrinter &p, StringRef name, ArrayRef< Value > operands)
static LogicalResult	parseMmaOperand (OpAsmParser &parser, StringRef operandName, SmallVectorImpl< OpAsmParser::UnresolvedOperand > &regs)
template<typename Op>
static void	processOperandFragments (Op &op, std::array< MMAOperandFragment, 3 > &frags, SmallVectorImpl< Type > &regTypes, SmallVectorImpl< StringRef > &ignoreAttrNames)
static LogicalResult	parseMmaTypeSignature (OpAsmParser &parser, SmallVectorImpl< Type > &operandTypes)
static void	inferAndSetMultiplicandTypes (MLIRContext *ctx, NamedAttrList &attrs, const SmallVectorImpl< Type > &operandTypes)
template<typename OpType>
static void	addBlockScaleProperties (OpBuilder &builder, OperationState &result, ArrayRef< int64_t > shape, ScaleVecSize scaleVecSize, BlockScaleFormat blockScaleFormat, MMABlockScaleKind kind)
static void	addInferredMultiplicandTypes (MLIRContext *ctx, OperationState &result, ValueRange operandA, ValueRange operandB, std::optional< std::array< MMATypes, 2 > > multiplicandPtxTypes)
template<typename OpTy>
static MMATypes	inferPtxTypeFromResult (OpTy op)
static std::pair< mlir::Type, unsigned >	inferMMATypeFromMNK (NVVM::MMATypes type, NVVM::MMAFrag frag, int m, int n, int k, MLIRContext *context)
static FailureOr< int >	getAllowedSizeK (NVVM::WGMMATypes typeA)
static LogicalResult	isAllowedWGMMADataType (NVVM::WGMMATypes typeD, NVVM::WGMMATypes typeA, NVVM::WGMMATypes typeB)
static LogicalResult	isAllowedSizeN (int sizeN, NVVM::WGMMATypes typeA)
template<typename OpType>
static LogicalResult	verifyFloatBinaryOp (OpType op, StringRef opKind)
static llvm::Value *	packValInto64Bits (llvm::IRBuilderBase &builder, llvm::Value result, llvm::Value field, unsigned sizeInBits, unsigned start)
	Packs the given field into the result.
static llvm::Intrinsic::ID	getBarrierSyncIntrinsic (bool aligned, bool hasCount)
	Maps the (aligned, hasCount) pair to the @llvm.nvvm.barrier.cta.sync.
static llvm::Intrinsic::ID	getBarrierReductionIntrinsic (bool aligned, NVVM::BarrierReduction kind)
	Maps the (aligned, kind) pair to the @llvm.nvvm.barrier.cta.red.
static void	appendMBarrierMulticastArgs (Value multicastMask, LLVM::ModuleTranslation &mt, llvm::IRBuilderBase &builder, SmallVectorImpl< llvm::Value * > &args)
static void	appendN1ScaleArg (Value scaleFactor, LLVM::ModuleTranslation &mt, llvm::IRBuilderBase &builder, llvm::SmallVectorImpl< llvm::Value * > &args, std::optional< uint16_t > defaultScale=std::nullopt)
static llvm::Intrinsic::ID	getUE5M3IntrinsicID (NVVM::FPRoundingMode rnd, bool hasSatFinite, bool hasScale, ArrayRef< llvm::Intrinsic::ID > ids, ArrayRef< llvm::Intrinsic::ID > scaledIds)
static unsigned	isValidVectorLength (NVVM::Tcgen05LdStShape shape, unsigned vecLen)
static llvm::Value *	getAsPackedI32 (llvm::Value *arg, llvm::IRBuilderBase &builder)
static llvm::Value *	getParamCastedAddr (llvm::Value *addr, llvm::IRBuilderBase &builder)
static llvm::nvvm::Tcgen05MMAKind	getNVVMTcgen05MMAKind (NVVM::Tcgen05MMAKind kind)
static LogicalResult	verifyTcgen05MMAOp (bool isATensor, mlir::Value disableOutputLane, NVVM::CTAGroupKind ctaGroup, bool hasAShift, NVVM::Tcgen05MMACollectorOp collectorOp, Location loc)
static LogicalResult	verifyTcgen05MMABlockScaleOp (NVVM::Tcgen05MMACollectorOp collectorOp, NVVM::Tcgen05MMAKind kind, NVVM::Tcgen05MMABlockScale blockScale, Location loc)
static unsigned	getNumSPRegisters (Type type)
static bool	isValidSPRepFactor (unsigned repFactor)

Variables
static constexpr unsigned	notIntrinsic = llvm::Intrinsic::not_intrinsic

Macro Definition Documentation

◆ _none

#define _none

Definition at line 5190 of file NVVMDialect.cpp.

◆ CP_ASYNC_ID_IMPL

#define CP_ASYNC_ID_IMPL	(	mod,
		size,
		suffix )

Value:

llvm::Intrinsic::nvvm_cp_async_##mod##_shared_global_##size##suffix

Definition at line 4640 of file NVVMDialect.cpp.

◆ CVT_F2TF32_ID_IMPL

#define CVT_F2TF32_ID_IMPL	(	rnd,
		relu,
		sf )

Value:

hasRelu ? llvm::Intrinsic::nvvm_f2tf32_##rnd##relu##sf \

: llvm::Intrinsic::nvvm_f2tf32_##rnd##sf

Definition at line 5192 of file NVVMDialect.cpp.

◆ GET_CP_ASYNC_ID

#define GET_CP_ASYNC_ID	(	mod,
		size,
		has_cpsize )

Value:

has_cpsize ? CP_ASYNC_ID_IMPL(mod, size, _s) : CP_ASYNC_ID_IMPL(mod, size, )

CP_ASYNC_ID_IMPL

#define CP_ASYNC_ID_IMPL(mod, size, suffix)

Definition NVVMDialect.cpp:4640

Definition at line 4643 of file NVVMDialect.cpp.

◆ GET_CVT_F2TF32_ID

#define GET_CVT_F2TF32_ID	(	rnd,
		relu,
		sf )

Value:

hasSatFinite ? CVT_F2TF32_ID_IMPL(rnd, relu, sf) \

: CVT_F2TF32_ID_IMPL(rnd, relu, )

CVT_F2TF32_ID_IMPL

#define CVT_F2TF32_ID_IMPL(rnd, relu, sf)

Definition NVVMDialect.cpp:5192

Definition at line 5196 of file NVVMDialect.cpp.

◆ GET_TCGEN05_CP_ID

#define GET_TCGEN05_CP_ID	(	shape_mc,
		src_fmt,
		is_2cta )

Value:

  [&]() -> auto {                                                              \
    if ((src_fmt) == Tcgen05CpSrcFormat::B6x16_P32)                            \
      return TCGEN05_CP_2CTA(shape_mc, _b6x16_p32, is_2cta);                   \
    if ((src_fmt) == Tcgen05CpSrcFormat::B4x16_P64)                            \
      return TCGEN05_CP_2CTA(shape_mc, _b4x16_p64, is_2cta);                   \
    return TCGEN05_CP_2CTA(shape_mc, , is_2cta);                               \
  }()

Definition at line 6065 of file NVVMDialect.cpp.

◆ TCGEN05_CP_2CTA

#define TCGEN05_CP_2CTA	(	shape_mc,
		src_fmt,
		is_2cta )

Value:

is_2cta ? TCGEN05_CP_IMPL(shape_mc, src_fmt, _cg2) \

: TCGEN05_CP_IMPL(shape_mc, src_fmt, _cg1)

TCGEN05_CP_IMPL

#define TCGEN05_CP_IMPL(shape_mc, src_fmt, cg)

Definition NVVMDialect.cpp:6058

Definition at line 6061 of file NVVMDialect.cpp.

◆ TCGEN05_CP_IMPL

#define TCGEN05_CP_IMPL	(	shape_mc,
		src_fmt,
		cg )

Value:

llvm::Intrinsic::nvvm_tcgen05_cp##shape_mc##src_fmt##cg

Definition at line 6058 of file NVVMDialect.cpp.

◆ TCGEN05LDRED

#define TCGEN05LDRED	(	SHAPE,
		NUM,
		TYPE )

Value:

llvm::Intrinsic::nvvm_tcgen05_ld_red_##SHAPE##_##NUM##_##TYPE

Definition at line 7351 of file NVVMDialect.cpp.

Function Documentation

◆ addBlockScaleProperties()

template<typename OpType>

void addBlockScaleProperties	(	OpBuilder &	builder,
		OperationState &	result,
		ArrayRef< int64_t >	shape,
		ScaleVecSize	scaleVecSize,
		BlockScaleFormat	blockScaleFormat,
		MMABlockScaleKind	kind )

static

Definition at line 2184 of file NVVMDialect.cpp.

References mlir::Builder::getAttr(), mlir::Builder::getContext(), and result.

◆ addInferredMultiplicandTypes()

void addInferredMultiplicandTypes	(	MLIRContext *	ctx,
		OperationState &	result,
		ValueRange	operandA,
		ValueRange	operandB,
		std::optional< std::array< MMATypes, 2 > >	multiplicandPtxTypes )

static

Definition at line 2200 of file NVVMDialect.cpp.

References mlir::getType(), and result.

◆ appendMBarrierMulticastArgs()

void appendMBarrierMulticastArgs	(	Value	multicastMask,
		LLVM::ModuleTranslation &	mt,
		llvm::IRBuilderBase &	builder,
		SmallVectorImpl< llvm::Value * > &	args )

static

Definition at line 4251 of file NVVMDialect.cpp.

References mlir::LLVM::ModuleTranslation::lookupValue().

◆ appendN1ScaleArg()

void appendN1ScaleArg	(	Value	scaleFactor,
		LLVM::ModuleTranslation &	mt,
		llvm::IRBuilderBase &	builder,
		llvm::SmallVectorImpl< llvm::Value * > &	args,
		std::optional< uint16_t >	defaultScale = std::nullopt )

static

Definition at line 5218 of file NVVMDialect.cpp.

References mlir::LLVM::ModuleTranslation::lookupValue().

◆ castPtrToAddrSpace()

llvm::Value * castPtrToAddrSpace	(	llvm::IRBuilderBase &	builder,
		llvm::Value *	ptr,
		NVVMMemorySpace	targetAS )

static

Definition at line 80 of file NVVMDialect.cpp.

◆ cpAsyncBulkTensorCommonVerifier()

LogicalResult cpAsyncBulkTensorCommonVerifier	(	size_t	tensorDims,
		bool	isIm2Col,
		size_t	numIm2ColOffsets,
		Location	loc )

static

Definition at line 141 of file NVVMDialect.cpp.

References mlir::emitError(), and success().

◆ CpAsyncBulkTensorOverrideAddrCommonVerifier()

LogicalResult CpAsyncBulkTensorOverrideAddrCommonVerifier	(	OperandRange	coordinates,
		OperandRange	tensorSize,
		OperandRange	lowerStride,
		Value	upperStride,
		bool	isTile,
		Location	loc )

Definition at line 163 of file NVVMDialect.cpp.

References mlir::emitError(), and success().

◆ getAllowedSizeK()

FailureOr< int > getAllowedSizeK ( NVVM::WGMMATypes typeA )

static

Definition at line 3085 of file NVVMDialect.cpp.

◆ getAsPackedI32()

llvm::Value * getAsPackedI32	(	llvm::Value *	arg,
		llvm::IRBuilderBase &	builder )

static

Definition at line 6348 of file NVVMDialect.cpp.

◆ getBarrierReductionIntrinsic()

llvm::Intrinsic::ID getBarrierReductionIntrinsic	(	bool	aligned,
		NVVM::BarrierReduction	kind )

static

Maps the (aligned, kind) pair to the @llvm.nvvm.barrier.cta.red.

* intrinsic ID.

Definition at line 3939 of file NVVMDialect.cpp.

◆ getBarrierSyncIntrinsic()

llvm::Intrinsic::ID getBarrierSyncIntrinsic	(	bool	aligned,
		bool	hasCount )

static

Maps the (aligned, hasCount) pair to the @llvm.nvvm.barrier.cta.sync.

* intrinsic ID.

Definition at line 3926 of file NVVMDialect.cpp.

◆ getNumSPRegisters()

unsigned getNumSPRegisters ( Type type )

static

Definition at line 7440 of file NVVMDialect.cpp.

◆ getNVVMCtaGroupKind()

llvm::nvvm::CTAGroupKind getNVVMCtaGroupKind ( NVVM::CTAGroupKind ctaGroup )

static

Definition at line 90 of file NVVMDialect.cpp.

◆ getNVVMTcgen05MMAKind()

llvm::nvvm::Tcgen05MMAKind getNVVMTcgen05MMAKind ( NVVM::Tcgen05MMAKind kind )

static

Definition at line 6597 of file NVVMDialect.cpp.

◆ getParamCastedAddr()

llvm::Value * getParamCastedAddr	(	llvm::Value *	addr,
		llvm::IRBuilderBase &	builder )

static

Definition at line 6397 of file NVVMDialect.cpp.

◆ getUE5M3IntrinsicID()

llvm::Intrinsic::ID getUE5M3IntrinsicID	(	NVVM::FPRoundingMode	rnd,
		bool	hasSatFinite,
		bool	hasScale,
		ArrayRef< llvm::Intrinsic::ID >	ids,
		ArrayRef< llvm::Intrinsic::ID >	scaledIds )

static

Definition at line 5424 of file NVVMDialect.cpp.

◆ inferAndSetMultiplicandTypes()

void inferAndSetMultiplicandTypes	(	MLIRContext *	ctx,
		NamedAttrList &	attrs,
		const SmallVectorImpl< Type > &	operandTypes )

static

Definition at line 2166 of file NVVMDialect.cpp.

References mlir::NamedAttrList::get(), and mlir::NamedAttrList::set().

◆ inferMBarrierArriveResultTypes()

LogicalResult inferMBarrierArriveResultTypes	(	MLIRContext *	context,
		Value	addr,
		SmallVectorImpl< Type > &	inferredReturnTypes )

static

Only shared_cluster (ptr<7>) produces zero results; all other address spaces (including generic) return i64.

Definition at line 433 of file NVVMDialect.cpp.

References isPtrInSharedClusterSpace(), and success().

◆ inferMMATypeFromMNK()

std::pair< mlir::Type, unsigned > inferMMATypeFromMNK	(	NVVM::MMATypes	type,
		NVVM::MMAFrag	frag,
		int	m,
		int	n,
		int	k,
		MLIRContext *	context )

static

Definition at line 2877 of file NVVMDialect.cpp.

References mlir::NVVM::inferMMAType().

◆ inferPtxTypeFromResult()

template<typename OpTy>

MMATypes inferPtxTypeFromResult ( OpTy op )

static

Definition at line 2219 of file NVVMDialect.cpp.

◆ isAllowedSizeN()

LogicalResult isAllowedSizeN	(	int	sizeN,
		NVVM::WGMMATypes	typeA )

static

Definition at line 3140 of file NVVMDialect.cpp.

References success().

◆ isAllowedWGMMADataType()

LogicalResult isAllowedWGMMADataType	(	NVVM::WGMMATypes	typeD,
		NVVM::WGMMATypes	typeA,
		NVVM::WGMMATypes	typeB )

static

Definition at line 3099 of file NVVMDialect.cpp.

References success().

◆ isCompatibleReturnTypesOptionalResult()

bool isCompatibleReturnTypesOptionalResult	(	TypeRange	inferred,
		TypeRange	actual )

static

For ops with optional results, allow the user to omit the result even when inference would produce one.

This preserves backward compatibility: the result can be silently discarded (e.g., for fire-and-forget arrive ops).

Definition at line 480 of file NVVMDialect.cpp.

◆ isInt4PtxType()

bool isInt4PtxType ( MMATypes type )

static

Definition at line 945 of file NVVMDialect.cpp.

Referenced by isIntegerPtxType().

◆ isInt8PtxType()

bool isInt8PtxType ( MMATypes type )

static

Definition at line 949 of file NVVMDialect.cpp.

Referenced by isIntegerPtxType().

◆ isIntegerPtxType()

bool isIntegerPtxType ( MMATypes type )

static

Definition at line 953 of file NVVMDialect.cpp.

References isInt4PtxType(), and isInt8PtxType().

◆ isMmaPropertyName()

bool isMmaPropertyName ( StringRef name )

static

Definition at line 1022 of file NVVMDialect.cpp.

Referenced by parseMmaProperties().

◆ isPtrInAddrSpace()

bool isPtrInAddrSpace	(	mlir::Value	ptr,
		NVVMMemorySpace	targetAS )

static

Definition at line 63 of file NVVMDialect.cpp.

Referenced by isPtrInGenericSpace(), isPtrInSharedClusterSpace(), and isPtrInSharedCTASpace().

◆ isPtrInGenericSpace()

bool isPtrInGenericSpace ( mlir::Value ptr )

static

Definition at line 68 of file NVVMDialect.cpp.

References isPtrInAddrSpace().

◆ isPtrInSharedClusterSpace()

bool isPtrInSharedClusterSpace ( mlir::Value ptr )

static

Definition at line 76 of file NVVMDialect.cpp.

References isPtrInAddrSpace().

Referenced by inferMBarrierArriveResultTypes(), and verifyMBarrierArriveLikeOp().

◆ isPtrInSharedCTASpace()

bool isPtrInSharedCTASpace ( mlir::Value ptr )

static

Definition at line 72 of file NVVMDialect.cpp.

References isPtrInAddrSpace().

◆ isValidSPRepFactor()

bool isValidSPRepFactor ( unsigned repFactor )

static

Definition at line 7447 of file NVVMDialect.cpp.

◆ isValidVectorLength()

unsigned isValidVectorLength	(	NVVM::Tcgen05LdStShape	shape,
		unsigned	vecLen )

static

Definition at line 6254 of file NVVMDialect.cpp.

◆ packValInto64Bits()

llvm::Value * packValInto64Bits	(	llvm::IRBuilderBase &	builder,
		llvm::Value *	result,
		llvm::Value *	field,
		unsigned	sizeInBits,
		unsigned	start )

static

Packs the given field into the result.

The result is 64-bits and each field can be 32-bits or narrower.

Definition at line 3819 of file NVVMDialect.cpp.

References result.

◆ parseEnumKeyword()

template<typename AttrTy>

ParseResult parseEnumKeyword	(	OpAsmParser &	parser,
		AttrTy &	attr )

static

Definition at line 119 of file NVVMDialect.cpp.

References mlir::AsmParser::emitError(), mlir::AsmParser::getContext(), mlir::AsmParser::getCurrentLocation(), mlir::AsmParser::parseKeywordOrString(), and success().

Referenced by parseMmaEnumPropertyValue().

◆ parseMmaEnumPropertyValue()

template<typename AttrTy>

ParseResult parseMmaEnumPropertyValue	(	OpAsmParser &	parser,
		NamedAttrList &	attributes,
		StringRef	name )

static

Definition at line 1009 of file NVVMDialect.cpp.

References mlir::NamedAttrList::append(), mlir::AsmParser::emitError(), mlir::NamedAttrList::get(), mlir::AsmParser::getCurrentLocation(), parseEnumKeyword(), mlir::AsmParser::parseEqual(), and success().

Referenced by parseMmaProperties().

◆ parseMmaOperand()

LogicalResult parseMmaOperand	(	OpAsmParser &	parser,
		StringRef	operandName,
		SmallVectorImpl< OpAsmParser::UnresolvedOperand > &	regs )

static

Definition at line 2101 of file NVVMDialect.cpp.

References mlir::AsmParser::OptionalSquare, mlir::AsmParser::parseKeyword(), mlir::OpAsmParser::parseOperandList(), and success().

◆ parseMmaProperties()

ParseResult parseMmaProperties	(	OpAsmParser &	parser,
		NamedAttrList &	attributes,
		ArrayRef< StringRef >	allowedKeywords,
		ArrayRef< StringRef >	requiredProperties )

static

Definition at line 1031 of file NVVMDialect.cpp.

References mlir::NamedAttrList::append(), mlir::AsmParser::emitError(), mlir::NamedAttrList::get(), mlir::AsmParser::getBuilder(), mlir::AsmParser::getCurrentLocation(), mlir::Builder::getUnitAttr(), isMmaPropertyName(), mlir::AsmParser::parseKeyword(), parseMmaEnumPropertyValue(), parseMmaPropertyValue(), mlir::AsmParser::parseOptionalAttrDict(), mlir::AsmParser::parseOptionalComma(), and success().

◆ parseMmaPropertyValue()

template<typename AttrTy>

ParseResult parseMmaPropertyValue	(	OpAsmParser &	parser,
		NamedAttrList &	attributes,
		StringRef	name )

static

Definition at line 995 of file NVVMDialect.cpp.

References mlir::NamedAttrList::append(), mlir::AsmParser::emitError(), mlir::NamedAttrList::get(), mlir::AsmParser::getCurrentLocation(), mlir::AsmParser::parseCustomAttributeWithFallback(), mlir::AsmParser::parseEqual(), and success().

Referenced by parseMmaProperties().

◆ parseMmaTypeSignature()

LogicalResult parseMmaTypeSignature	(	OpAsmParser &	parser,
		SmallVectorImpl< Type > &	operandTypes )

static

Definition at line 2142 of file NVVMDialect.cpp.

References mlir::AsmParser::emitError(), mlir::AsmParser::getCurrentLocation(), mlir::AsmParser::parseColon(), mlir::AsmParser::parseCommaSeparatedList(), mlir::AsmParser::parseLParen(), mlir::AsmParser::parseRParen(), mlir::AsmParser::parseType(), and success().

◆ printMmaEnumProperty()

template<typename AttrTy>

void printMmaEnumProperty	(	OpAsmPrinter &	printer,
		bool &	isFirst,
		StringRef	keyword,
		AttrTy	value )

static

Definition at line 981 of file NVVMDialect.cpp.

◆ printMmaProperty()

template<typename AttrTy>

void printMmaProperty	(	OpAsmPrinter &	printer,
		bool &	isFirst,
		StringRef	keyword,
		AttrTy	value )

static

Definition at line 973 of file NVVMDialect.cpp.

References mlir::AsmPrinter::printStrippedAttrOrType().

◆ printMmaUnitProperty()

void printMmaUnitProperty	(	OpAsmPrinter &	printer,
		bool &	isFirst,
		StringRef	keyword )

static

Definition at line 988 of file NVVMDialect.cpp.

◆ printOperandList()

void printOperandList	(	OpAsmPrinter &	p,
		StringRef	name,
		ArrayRef< Value >	operands )

static

Definition at line 2092 of file NVVMDialect.cpp.

References mlir::OpAsmPrinter::printOperands().

◆ processOperandFragments()

template<typename Op>

void processOperandFragments	(	Op &	op,
		std::array< MMAOperandFragment, 3 > &	frags,
		SmallVectorImpl< Type > &	regTypes,
		SmallVectorImpl< StringRef > &	ignoreAttrNames )

static

Definition at line 2115 of file NVVMDialect.cpp.

◆ verifyConvertF32x2ToFP16x2Op()

LogicalResult verifyConvertF32x2ToFP16x2Op	(	Twine	dstType,
		FPRoundingMode	rnd,
		bool	hasRandomBits,
		bool	hasPzo,
		Operation *	op )

static

Definition at line 798 of file NVVMDialect.cpp.

References mlir::Operation::emitOpError(), and success().

◆ verifyFloatBinaryOp()

template<typename OpType>

LogicalResult verifyFloatBinaryOp	(	OpType	op,
		StringRef	opKind )

static

Definition at line 3684 of file NVVMDialect.cpp.

References mlir::Type::isBF16(), mlir::Type::isF16(), mlir::Type::isF64(), and success().

◆ verifyMBarrierArriveLikeOp()

LogicalResult verifyMBarrierArriveLikeOp	(	Operation *	op,
		Value	addr,
		NVVM::MemScopeKind	scope,
		Value	retVal = nullptr,
		Value	multicastMask = nullptr )

static

Definition at line 369 of file NVVMDialect.cpp.

References mlir::Operation::emitError(), isPtrInSharedClusterSpace(), and success().

◆ verifyTcgen05MMABlockScaleOp()

LogicalResult verifyTcgen05MMABlockScaleOp	(	NVVM::Tcgen05MMACollectorOp	collectorOp,
		NVVM::Tcgen05MMAKind	kind,
		NVVM::Tcgen05MMABlockScale	blockScale,
		Location	loc )

static

Definition at line 7036 of file NVVMDialect.cpp.

References mlir::emitError(), and success().

◆ verifyTcgen05MMAOp()

LogicalResult verifyTcgen05MMAOp	(	bool	isATensor,
		mlir::Value	disableOutputLane,
		NVVM::CTAGroupKind	ctaGroup,
		bool	hasAShift,
		NVVM::Tcgen05MMACollectorOp	collectorOp,
		Location	loc )

static

Definition at line 6765 of file NVVMDialect.cpp.

References mlir::emitError(), mlir::Value::getType(), and success().

◆ verifyTMALoadParams()

LogicalResult verifyTMALoadParams	(	size_t	tensorDims,
		size_t	numIm2colOff,
		TMALoadMode	mode,
		Location	loc )

static

Definition at line 255 of file NVVMDialect.cpp.

References mlir::emitError(), and success().

Variable Documentation

◆ notIntrinsic

unsigned notIntrinsic = llvm::Intrinsic::not_intrinsic

staticconstexpr

Definition at line 57 of file NVVMDialect.cpp.

Classes

Macros

Functions

Variables

Macro Definition Documentation

◆ _none

◆ CP_ASYNC_ID_IMPL

◆ CVT_F2TF32_ID_IMPL

◆ GET_CP_ASYNC_ID

◆ GET_CVT_F2TF32_ID

◆ GET_TCGEN05_CP_ID

◆ TCGEN05_CP_2CTA

◆ TCGEN05_CP_IMPL

◆ TCGEN05LDRED

Function Documentation

◆ addBlockScaleProperties()

◆ addInferredMultiplicandTypes()

◆ appendMBarrierMulticastArgs()

◆ appendN1ScaleArg()

◆ castPtrToAddrSpace()

◆ cpAsyncBulkTensorCommonVerifier()

◆ CpAsyncBulkTensorOverrideAddrCommonVerifier()

◆ getAllowedSizeK()

◆ getAsPackedI32()

◆ getBarrierReductionIntrinsic()

◆ getBarrierSyncIntrinsic()

◆ getNumSPRegisters()

◆ getNVVMCtaGroupKind()

◆ getNVVMTcgen05MMAKind()

◆ getParamCastedAddr()

◆ getUE5M3IntrinsicID()

◆ inferAndSetMultiplicandTypes()

◆ inferMBarrierArriveResultTypes()

◆ inferMMATypeFromMNK()

◆ inferPtxTypeFromResult()

◆ isAllowedSizeN()

◆ isAllowedWGMMADataType()

◆ isCompatibleReturnTypesOptionalResult()

◆ isInt4PtxType()

◆ isInt8PtxType()

◆ isIntegerPtxType()

◆ isMmaPropertyName()

◆ isPtrInAddrSpace()

◆ isPtrInGenericSpace()

◆ isPtrInSharedClusterSpace()

◆ isPtrInSharedCTASpace()

◆ isValidSPRepFactor()

◆ isValidVectorLength()

◆ packValInto64Bits()

◆ parseEnumKeyword()

◆ parseMmaEnumPropertyValue()

◆ parseMmaOperand()

◆ parseMmaProperties()

◆ parseMmaPropertyValue()

◆ parseMmaTypeSignature()

◆ printMmaEnumProperty()

◆ printMmaProperty()

◆ printMmaUnitProperty()

◆ printOperandList()

◆ processOperandFragments()

◆ verifyConvertF32x2ToFP16x2Op()

◆ verifyFloatBinaryOp()

◆ verifyMBarrierArriveLikeOp()

◆ verifyTcgen05MMABlockScaleOp()

◆ verifyTcgen05MMAOp()

◆ verifyTMALoadParams()

Variable Documentation

◆ notIntrinsic