#include "mlir/ExecutionEngine/CRunnerUtils.h"
#include <cstdio>
#include "cuda.h"
#include "cuda_bf16.h"
#include "cuda_fp16.h"

Classes
class	ScopedContext

Macros
#define	MLIR_CUDA_WRAPPERS_EXPORT __attribute__((visibility("default")))
#define	CUDA_REPORT_IF_ERROR(expr)
#define	CUDA_REPORT_IF_ERROR_IGNORE_SHUTDOWN(expr)
	Like CUDA_REPORT_IF_ERROR, but silences errors caused by CUDA context shutdown.
#define	CUSPARSE_REPORT_IF_ERROR(expr)
#define	debug_print(fmt, ...)

Functions
static bool	isCudaContextShutdownError (CUresult result)
	Helper to check if a CUDA error is due to the context being destroyed during program shutdown.
static bool	isDebugEnabled ()
	Helper method that checks environment value for debugging.
static CUdevice	getDefaultCuDevice ()
MLIR_CUDA_WRAPPERS_EXPORT CUmodule	mgpuModuleLoad (void *data, size_t)
MLIR_CUDA_WRAPPERS_EXPORT CUmodule	mgpuModuleLoadJIT (void *data, int optLevel, size_t)
MLIR_CUDA_WRAPPERS_EXPORT void	mgpuModuleUnload (CUmodule module)
MLIR_CUDA_WRAPPERS_EXPORT CUfunction	mgpuModuleGetFunction (CUmodule module, const char *name)
MLIR_CUDA_WRAPPERS_EXPORT void	mgpuLaunchKernel (CUfunction function, intptr_t gridX, intptr_t gridY, intptr_t gridZ, intptr_t blockX, intptr_t blockY, intptr_t blockZ, int32_t smem, CUstream stream, void params, void extra, size_t)
MLIR_CUDA_WRAPPERS_EXPORT CUstream	mgpuStreamCreate ()
MLIR_CUDA_WRAPPERS_EXPORT void	mgpuStreamDestroy (CUstream stream)
MLIR_CUDA_WRAPPERS_EXPORT void	mgpuStreamSynchronize (CUstream stream)
MLIR_CUDA_WRAPPERS_EXPORT void	mgpuStreamWaitEvent (CUstream stream, CUevent event)
MLIR_CUDA_WRAPPERS_EXPORT CUevent	mgpuEventCreate ()
MLIR_CUDA_WRAPPERS_EXPORT void	mgpuEventDestroy (CUevent event)
MLIR_CUDA_WRAPPERS_EXPORT void	mgpuEventSynchronize (CUevent event)
MLIR_CUDA_WRAPPERS_EXPORT void	mgpuEventRecord (CUevent event, CUstream stream)
MLIR_CUDA_WRAPPERS_EXPORT void *	mgpuMemAlloc (uint64_t sizeBytes, CUstream stream, bool isHostShared)
MLIR_CUDA_WRAPPERS_EXPORT void	mgpuMemFree (void *ptr, CUstream)
MLIR_CUDA_WRAPPERS_EXPORT void	mgpuMemcpy (void dst, void src, size_t sizeBytes, CUstream stream)
MLIR_CUDA_WRAPPERS_EXPORT void	mgpuMemset32 (void *dst, unsigned int value, size_t count, CUstream stream)
MLIR_CUDA_WRAPPERS_EXPORT void	mgpuMemset16 (void *dst, unsigned short value, size_t count, CUstream stream)
MLIR_CUDA_WRAPPERS_EXPORT void	mgpuMemHostRegister (void *ptr, uint64_t sizeBytes)
	Helper functions for writing mlir example code.
MLIR_CUDA_WRAPPERS_EXPORT void	mgpuMemHostRegisterMemRef (int64_t rank, StridedMemRefType< char, 1 > *descriptor, int64_t elementSizeBytes)
	Registers a memref with the CUDA runtime.
MLIR_CUDA_WRAPPERS_EXPORT void	mgpuMemHostUnregister (void *ptr)
MLIR_CUDA_WRAPPERS_EXPORT void	mgpuMemHostUnregisterMemRef (int64_t rank, StridedMemRefType< char, 1 > *descriptor, int64_t elementSizeBytes)
	Unregisters a memref with the CUDA runtime.
MLIR_CUDA_WRAPPERS_EXPORT void	mgpuSetDefaultDevice (int32_t device)

Variables
static thread_local int32_t	defaultDevice = 0

Macro Definition Documentation

◆ CUDA_REPORT_IF_ERROR

#define CUDA_REPORT_IF_ERROR ( expr )

Value:

  [](CUresult result) {                                                        \
    if (!result)                                                               \
      return;                                                                  \
    const char *name = nullptr;                                                \
    cuGetErrorName(result, &name);                                             \
    if (!name)                                                                 \
      name = "<unknown>";                                                      \
    fprintf(stderr, "'%s' failed with '%s'\n", #expr, name);                   \
  }(expr)

Definition at line 37 of file CudaRuntimeWrappers.cpp.

Referenced by getDefaultCuDevice(), mgpuEventCreate(), mgpuEventRecord(), mgpuLaunchKernel(), mgpuMemAlloc(), mgpuMemcpy(), mgpuMemFree(), mgpuMemHostRegister(), mgpuMemHostUnregister(), mgpuMemset16(), mgpuMemset32(), mgpuModuleGetFunction(), mgpuModuleLoad(), mgpuModuleLoadJIT(), mgpuStreamCreate(), mgpuStreamSynchronize(), ScopedContext::ScopedContext(), and ScopedContext::~ScopedContext().

◆ CUDA_REPORT_IF_ERROR_IGNORE_SHUTDOWN

#define CUDA_REPORT_IF_ERROR_IGNORE_SHUTDOWN ( expr )

Value:

  [](CUresult result) {                                                        \
    if (!result || isCudaContextShutdownError(result))                         \
      return;                                                                  \
    const char *name = nullptr;                                                \
    cuGetErrorName(result, &name);                                             \
    if (!name)                                                                 \
      name = "<unknown>";                                                      \
    fprintf(stderr, "'%s' failed with '%s'\n", #expr, name);                   \
  }(expr)

Like CUDA_REPORT_IF_ERROR, but silences errors caused by CUDA context shutdown.

These errors are benign when they occur during program exit, as all resources are freed with the context.

Definition at line 60 of file CudaRuntimeWrappers.cpp.

Referenced by mgpuEventDestroy(), mgpuEventSynchronize(), mgpuModuleUnload(), mgpuStreamDestroy(), and mgpuStreamWaitEvent().

◆ CUSPARSE_REPORT_IF_ERROR

#define CUSPARSE_REPORT_IF_ERROR ( expr )

Value:

  {                                                                            \
    cusparseStatus_t status = (expr);                                          \
    if (status != CUSPARSE_STATUS_SUCCESS) {                                   \
      fprintf(stderr, "cuSPARSE '%s' failed with '%s'\n", #expr,               \
              cusparseGetErrorString(status));                                 \
    }                                                                          \
  }

Definition at line 71 of file CudaRuntimeWrappers.cpp.

◆ debug_print

#define debug_print	(		fmt,
			... )

Value:

  do {                                                                         \
    if (isDebugEnabled())                                                      \
      fprintf(stderr, "%s:%d:%s(): " fmt, "CudaRuntimeWrappers.cpp", __LINE__, \
              __func__, __VA_ARGS__);                                          \
  } while (0)

Definition at line 89 of file CudaRuntimeWrappers.cpp.

Referenced by mgpuLaunchKernel().

◆ MLIR_CUDA_WRAPPERS_EXPORT

#define MLIR_CUDA_WRAPPERS_EXPORT __attribute__((visibility("default")))

Definition at line 34 of file CudaRuntimeWrappers.cpp.

Referenced by mgpuEventCreate(), mgpuEventDestroy(), mgpuEventRecord(), mgpuEventSynchronize(), mgpuMemFree(), mgpuMemHostUnregister(), mgpuModuleUnload(), mgpuSetDefaultDevice(), mgpuStreamCreate(), mgpuStreamDestroy(), and mgpuStreamWaitEvent().

Function Documentation

◆ getDefaultCuDevice()

CUdevice getDefaultCuDevice ( )

static

Definition at line 97 of file CudaRuntimeWrappers.cpp.

References CUDA_REPORT_IF_ERROR, and defaultDevice.

Referenced by mgpuLaunchKernel(), and ScopedContext::ScopedContext().

◆ isCudaContextShutdownError()

bool isCudaContextShutdownError ( CUresult result )

static

Helper to check if a CUDA error is due to the context being destroyed during program shutdown.

Both CUDA_ERROR_DEINITIALIZED and CUDA_ERROR_CONTEXT_IS_DESTROYED indicate that the CUDA context has been torn down and any associated resources are already freed.

Definition at line 52 of file CudaRuntimeWrappers.cpp.

References result.

◆ isDebugEnabled()

bool isDebugEnabled ( )

static

Helper method that checks environment value for debugging.

Definition at line 83 of file CudaRuntimeWrappers.cpp.

◆ mgpuEventCreate()

MLIR_CUDA_WRAPPERS_EXPORT CUevent mgpuEventCreate ( )

Definition at line 239 of file CudaRuntimeWrappers.cpp.

References CUDA_REPORT_IF_ERROR, and MLIR_CUDA_WRAPPERS_EXPORT.

◆ mgpuEventDestroy()

MLIR_CUDA_WRAPPERS_EXPORT void mgpuEventDestroy ( CUevent event )

Definition at line 246 of file CudaRuntimeWrappers.cpp.

References CUDA_REPORT_IF_ERROR_IGNORE_SHUTDOWN, and MLIR_CUDA_WRAPPERS_EXPORT.

◆ mgpuEventRecord()

MLIR_CUDA_WRAPPERS_EXPORT void mgpuEventRecord	(	CUevent	event,
		CUstream	stream )

Definition at line 254 of file CudaRuntimeWrappers.cpp.

References CUDA_REPORT_IF_ERROR, and MLIR_CUDA_WRAPPERS_EXPORT.

◆ mgpuEventSynchronize()

MLIR_CUDA_WRAPPERS_EXPORT void mgpuEventSynchronize ( CUevent event )

Definition at line 250 of file CudaRuntimeWrappers.cpp.

References CUDA_REPORT_IF_ERROR_IGNORE_SHUTDOWN, and MLIR_CUDA_WRAPPERS_EXPORT.

◆ mgpuLaunchKernel()

MLIR_CUDA_WRAPPERS_EXPORT void mgpuLaunchKernel	(	CUfunction	function,
		intptr_t	gridX,
		intptr_t	gridY,
		intptr_t	gridZ,
		intptr_t	blockX,
		intptr_t	blockY,
		intptr_t	blockZ,
		int32_t	smem,
		CUstream	stream,
		void **	params,
		void **	extra,
		size_t	)