17 #include "llvm/ADT/SmallVectorExtras.h"
18 #include "llvm/ADT/StringSet.h"
19 #include "llvm/Support/FormatVariadic.h"
25 LLVM::LLVMFunctionType type) {
26 auto existing = dyn_cast_or_null<LLVM::LLVMFuncOp>(
33 return LLVM::LLVMFuncOp::create(b, loc, name, type, LLVM::Linkage::External);
39 unsigned stringNumber = 0;
42 stringConstName.clear();
43 (prefix + Twine(stringNumber++)).toStringRef(stringConstName);
45 return stringConstName;
55 nullTermStr.push_back(
'\0');
62 if (globalOp.getGlobalType() == globalType && globalOp.getConstant() &&
63 globalOp.getValueAttr() == attr &&
64 globalOp.getAlignment().value_or(0) == alignment &&
65 globalOp.getAddrSpace() == addrSpace)
72 return LLVM::GlobalOp::create(b, loc, globalType,
73 true, LLVM::Linkage::Internal,
74 name, attr, alignment, addrSpace);
83 if (encodeWorkgroupAttributionsAsArguments) {
88 gpuFuncOp.getWorkgroupAttributions();
89 size_t numAttributions = workgroupAttributions.size();
92 unsigned index = gpuFuncOp.getNumArguments();
96 Type workgroupPtrType =
97 rewriter.
getType<LLVM::LLVMPointerType>(workgroupAddrSpace);
102 rewriter.
getNamedAttr(LLVM::LLVMDialect::getNoAliasAttrName(),
105 getDialect().getWorkgroupAttributionAttrHelper().getName(),
110 auto attributionType = cast<MemRefType>(attribution.getType());
111 IntegerAttr numElements =
113 Type llvmElementType =
115 if (!llvmElementType)
118 attrs.back().setValue(
119 rewriter.
getAttr<LLVM::WorkgroupAttributionAttr>(numElements, type));
128 gpuFuncOp, [gpuFuncOp, &argIndices, &argTypes, &argAttrs, &argLocs]() {
129 LogicalResult inserted =
130 static_cast<FunctionOpInterface
>(gpuFuncOp).insertArguments(
131 argIndices, argTypes, argAttrs, argLocs);
133 assert(succeeded(inserted) &&
134 "expected GPU funcs to support inserting any argument");
137 workgroupBuffers.reserve(gpuFuncOp.getNumWorkgroupAttributions());
138 for (
auto [idx, attribution] :
140 auto type = dyn_cast<MemRefType>(attribution.getType());
141 assert(type && type.hasStaticShape() &&
"unexpected type in attribution");
143 uint64_t numElements = type.getNumElements();
149 std::string(llvm::formatv(
"__wg_{0}_{1}", gpuFuncOp.getName(), idx));
150 uint64_t alignment = 0;
151 if (
auto alignAttr = dyn_cast_or_null<IntegerAttr>(
152 gpuFuncOp.getWorkgroupAttributionAttr(
153 idx, LLVM::LLVMDialect::getAlignAttrName())))
154 alignment = alignAttr.getInt();
155 auto globalOp = LLVM::GlobalOp::create(
156 rewriter, gpuFuncOp.getLoc(), arrayType,
false,
157 LLVM::Linkage::Internal, name,
Attribute(), alignment,
159 workgroupBuffers.push_back(globalOp);
165 gpuFuncOp.front().getNumArguments());
168 gpuFuncOp.getFunctionType(),
false,
172 diag <<
"failed to convert function signature type for: "
173 << gpuFuncOp.getFunctionType();
181 for (
const auto &attr : gpuFuncOp->getAttrs()) {
183 attr.getName() == gpuFuncOp.getFunctionTypeAttrName() ||
185 gpu::GPUFuncOp::getNumWorkgroupAttributionsAttrName() ||
186 attr.getName() == gpuFuncOp.getWorkgroupAttribAttrsAttrName() ||
187 attr.getName() == gpuFuncOp.getPrivateAttribAttrsAttrName() ||
188 attr.getName() == gpuFuncOp.getKnownBlockSizeAttrName() ||
189 attr.getName() == gpuFuncOp.getKnownGridSizeAttrName())
191 if (attr.getName() == gpuFuncOp.getArgAttrsAttrName()) {
192 argAttrs = gpuFuncOp.getArgAttrsAttr();
195 attributes.push_back(attr);
202 auto *gpuDialect = cast<gpu::GPUDialect>(gpuFuncOp->getDialect());
204 attributes.emplace_back(gpuDialect->getKnownBlockSizeAttrHelper().getName(),
207 attributes.emplace_back(gpuDialect->getKnownGridSizeAttrHelper().getName(),
213 if (gpuFuncOp.isKernel()) {
214 if (kernelAttributeName)
215 attributes.emplace_back(kernelAttributeName, rewriter.
getUnitAttr());
217 if (kernelBlockSizeAttributeName && knownBlockSize) {
218 attributes.emplace_back(kernelBlockSizeAttributeName, knownBlockSize);
221 LLVM::CConv callingConvention = gpuFuncOp.isKernel()
222 ? kernelCallingConvention
223 : nonKernelCallingConvention;
224 auto llvmFuncOp = LLVM::LLVMFuncOp::create(
225 rewriter, gpuFuncOp.getLoc(), gpuFuncOp.getName(), funcType,
226 LLVM::Linkage::External,
false, callingConvention,
227 nullptr, attributes);
239 unsigned numProperArguments = gpuFuncOp.getNumArguments();
241 if (encodeWorkgroupAttributionsAsArguments) {
244 unsigned numAttributions = gpuFuncOp.getNumWorkgroupAttributions();
245 assert(numProperArguments >= numAttributions &&
246 "Expecting attributions to be encoded as arguments already");
251 gpuFuncOp.getArguments().slice(numProperArguments - numAttributions,
254 gpuFuncOp.getWorkgroupAttributions(), attributionArguments))) {
255 auto [attribution, arg] = vals;
256 auto type = cast<MemRefType>(attribution.getType());
264 signatureConversion.
remapInput(numProperArguments + idx, descr);
269 global.getAddrSpace());
270 Value address = LLVM::AddressOfOp::create(rewriter, loc, ptrType,
271 global.getSymNameAttr());
273 LLVM::GEPOp::create(rewriter, loc, ptrType, global.getType(),
280 Value attribution = gpuFuncOp.getWorkgroupAttributions()[idx];
281 auto type = cast<MemRefType>(attribution.
getType());
284 signatureConversion.
remapInput(numProperArguments + idx, descr);
289 unsigned numWorkgroupAttributions = gpuFuncOp.getNumWorkgroupAttributions();
291 for (
const auto [idx, attribution] :
293 auto type = cast<MemRefType>(attribution.getType());
294 assert(type && type.hasStaticShape() &&
"unexpected type in attribution");
302 Value numElements = LLVM::ConstantOp::create(
303 rewriter, gpuFuncOp.getLoc(), int64Ty, type.getNumElements());
304 uint64_t alignment = 0;
306 dyn_cast_or_null<IntegerAttr>(gpuFuncOp.getPrivateAttributionAttr(
307 idx, LLVM::LLVMDialect::getAlignAttrName())))
308 alignment = alignAttr.getInt();
310 LLVM::AllocaOp::create(rewriter, gpuFuncOp.getLoc(), ptrType,
311 elementType, numElements, alignment);
315 numProperArguments + numWorkgroupAttributions + idx, descr);
323 &signatureConversion)))
328 for (
const auto [idx, argTy] :
332 argAttrs ? cast<DictionaryAttr>(argAttrs[idx]) :
NamedAttrList();
333 auto copyAttribute = [&](StringRef attrName) {
337 for (
size_t i = 0, e = remapping->size; i < e; ++i)
338 llvmFuncOp.setArgAttr(remapping->inputNo + i, attrName, attr);
340 auto copyPointerAttribute = [&](StringRef attrName) {
345 if (remapping->size > 1 &&
346 attrName == LLVM::LLVMDialect::getNoAliasAttrName()) {
348 "Cannot copy noalias with non-bare pointers.\n");
351 for (
size_t i = 0, e = remapping->size; i < e; ++i) {
352 if (isa<LLVM::LLVMPointerType>(
353 llvmFuncOp.getArgument(remapping->inputNo + i).getType())) {
354 llvmFuncOp.setArgAttr(remapping->inputNo + i, attrName, attr);
362 copyAttribute(LLVM::LLVMDialect::getReturnedAttrName());
363 copyAttribute(LLVM::LLVMDialect::getNoUndefAttrName());
364 copyAttribute(LLVM::LLVMDialect::getInRegAttrName());
365 bool lowersToPointer =
false;
366 for (
size_t i = 0, e = remapping->size; i < e; ++i) {
367 lowersToPointer |= isa<LLVM::LLVMPointerType>(
368 llvmFuncOp.getArgument(remapping->inputNo + i).getType());
371 if (lowersToPointer) {
372 copyPointerAttribute(LLVM::LLVMDialect::getNoAliasAttrName());
373 copyPointerAttribute(LLVM::LLVMDialect::getNoCaptureAttrName());
374 copyPointerAttribute(LLVM::LLVMDialect::getNoFreeAttrName());
375 copyPointerAttribute(LLVM::LLVMDialect::getAlignAttrName());
376 copyPointerAttribute(LLVM::LLVMDialect::getReadonlyAttrName());
377 copyPointerAttribute(LLVM::LLVMDialect::getWriteOnlyAttrName());
378 copyPointerAttribute(LLVM::LLVMDialect::getReadnoneAttrName());
379 copyPointerAttribute(LLVM::LLVMDialect::getNonNullAttrName());
380 copyPointerAttribute(LLVM::LLVMDialect::getDereferenceableAttrName());
381 copyPointerAttribute(
382 LLVM::LLVMDialect::getDereferenceableOrNullAttrName());
383 copyPointerAttribute(
384 LLVM::LLVMDialect::WorkgroupAttributionAttrHelper::getNameStr());
392 gpu::PrintfOp gpuPrintfOp, gpu::PrintfOpAdaptor adaptor,
394 Location loc = gpuPrintfOp->getLoc();
404 "Couldn't find a parent module");
409 LLVM::LLVMFuncOp ocklAppendArgs;
410 if (!adaptor.getArgs().empty()) {
412 moduleOp, loc, rewriter,
"__ockl_printf_append_args",
414 llvmI64, {llvmI64, llvmI32, llvmI64, llvmI64, llvmI64,
415 llvmI64, llvmI64, llvmI64, llvmI64, llvmI32}));
418 moduleOp, loc, rewriter,
"__ockl_printf_append_string_n",
421 {llvmI64, ptrType, llvmI64, llvmI32}));
424 Value zeroI64 = LLVM::ConstantOp::create(rewriter, loc, llvmI64, 0);
425 auto printfBeginCall =
426 LLVM::CallOp::create(rewriter, loc, ocklBegin, zeroI64);
427 Value printfDesc = printfBeginCall.getResult();
431 rewriter, loc, moduleOp, llvmI8,
"printfFormat_", adaptor.getFormat());
434 Value globalPtr = LLVM::AddressOfOp::create(
437 global.getSymNameAttr());
439 LLVM::GEPOp::create(rewriter, loc, ptrType, global.getGlobalType(),
441 Value stringLen = LLVM::ConstantOp::create(
442 rewriter, loc, llvmI64, cast<StringAttr>(global.getValueAttr()).size());
444 Value oneI32 = LLVM::ConstantOp::create(rewriter, loc, llvmI32, 1);
445 Value zeroI32 = LLVM::ConstantOp::create(rewriter, loc, llvmI32, 0);
447 auto appendFormatCall = LLVM::CallOp::create(
448 rewriter, loc, ocklAppendStringN,
449 ValueRange{printfDesc, stringStart, stringLen,
450 adaptor.getArgs().empty() ? oneI32 : zeroI32});
451 printfDesc = appendFormatCall.getResult();
454 constexpr
size_t argsPerAppend = 7;
455 size_t nArgs = adaptor.getArgs().size();
456 for (
size_t group = 0; group < nArgs; group += argsPerAppend) {
457 size_t bound =
std::min(group + argsPerAppend, nArgs);
458 size_t numArgsThisCall = bound - group;
461 arguments.push_back(printfDesc);
463 LLVM::ConstantOp::create(rewriter, loc, llvmI32, numArgsThisCall));
464 for (
size_t i = group; i < bound; ++i) {
465 Value arg = adaptor.getArgs()[i];
466 if (
auto floatType = dyn_cast<FloatType>(arg.
getType())) {
467 if (!floatType.isF64())
468 arg = LLVM::FPExtOp::create(
471 arg = LLVM::BitcastOp::create(rewriter, loc, llvmI64, arg);
474 arg = LLVM::ZExtOp::create(rewriter, loc, llvmI64, arg);
476 arguments.push_back(arg);
479 for (
size_t extra = numArgsThisCall; extra < argsPerAppend; ++extra) {
480 arguments.push_back(zeroI64);
483 auto isLast = (bound == nArgs) ? oneI32 : zeroI32;
484 arguments.push_back(isLast);
485 auto call = LLVM::CallOp::create(rewriter, loc, ocklAppendArgs, arguments);
486 printfDesc = call.getResult();
493 gpu::PrintfOp gpuPrintfOp, gpu::PrintfOpAdaptor adaptor,
495 Location loc = gpuPrintfOp->getLoc();
504 "Couldn't find a parent module");
509 LLVM::LLVMFuncOp printfDecl =
514 rewriter, loc, moduleOp, llvmI8,
"printfFormat_", adaptor.getFormat(),
518 Value globalPtr = LLVM::AddressOfOp::create(
521 global.getSymNameAttr());
523 LLVM::GEPOp::create(rewriter, loc, ptrType, global.getGlobalType(),
527 auto argsRange = adaptor.getArgs();
529 printfArgs.reserve(argsRange.size() + 1);
530 printfArgs.push_back(stringStart);
531 printfArgs.append(argsRange.begin(), argsRange.end());
533 LLVM::CallOp::create(rewriter, loc, printfDecl, printfArgs);
539 gpu::PrintfOp gpuPrintfOp, gpu::PrintfOpAdaptor adaptor,
541 Location loc = gpuPrintfOp->getLoc();
549 "Couldn't find a parent module");
559 moduleOp, globalLoc, rewriter,
"vprintf", vprintfType);
562 LLVM::GlobalOp global =
564 "printfFormat_", adaptor.getFormat());
567 Value globalPtr = LLVM::AddressOfOp::create(rewriter, loc, global);
569 LLVM::GEPOp::create(rewriter, loc, ptrType, global.getGlobalType(),
574 for (
Value arg : adaptor.getArgs()) {
575 Type type = arg.getType();
576 Value promotedArg = arg;
578 if (isa<FloatType>(type)) {
580 promotedArg = LLVM::FPExtOp::create(rewriter, loc, type, arg);
582 types.push_back(type);
583 args.push_back(promotedArg);
586 LLVM::LLVMStructType::getLiteral(gpuPrintfOp.getContext(), types);
587 Value one = LLVM::ConstantOp::create(rewriter, loc, rewriter.
getI64Type(),
590 LLVM::AllocaOp::create(rewriter, loc, ptrType, structType, one,
593 Value ptr = LLVM::GEPOp::create(
594 rewriter, loc, ptrType, structType, tempAlloc,
596 LLVM::StoreOp::create(rewriter, loc, arg, ptr);
598 std::array<Value, 2> printfArgs = {stringStart, tempAlloc};
600 LLVM::CallOp::create(rewriter, loc, vprintfDecl, printfArgs);
613 VectorType vectorType = cast<VectorType>(llvm1DVectorTy);
615 Value result = LLVM::PoisonOp::create(rewriter, loc, vectorType);
618 Type elementType = vectorType.getElementType();
620 for (int64_t i = 0; i < vectorType.getNumElements(); ++i) {
621 Value index = LLVM::ConstantOp::create(rewriter, loc, indexType, i);
622 auto extractElement = [&](
Value operand) ->
Value {
623 if (!isa<VectorType>(operand.getType()))
625 return LLVM::ExtractElementOp::create(rewriter, loc, operand, index);
627 auto scalarOperands = llvm::map_to_vector(operands, extractElement);
629 rewriter.
create(loc, name, scalarOperands, elementType, op->
getAttrs());
630 result = LLVM::InsertElementOp::create(rewriter, loc, result,
641 if (llvm::any_of(operandTypes, llvm::IsaPred<VectorType>)) {
642 VectorType vectorType =
645 rewriter, converter));
649 if (llvm::any_of(operandTypes, llvm::IsaPred<LLVM::LLVMArrayType>)) {
651 op, operands, converter,
671 MemRefType memrefType,
unsigned alignmentBit) {
672 uint64_t alignmentByte = alignmentBit / memrefType.getElementTypeBitWidth();
674 FailureOr<unsigned> addressSpace =
676 if (
failed(addressSpace)) {
677 op->emitError() <<
"conversion of memref memory space "
678 << memrefType.getMemorySpace()
679 <<
" to integer address space "
680 "failed. Consider adding memory space conversions.";
686 for (
auto globalOp : moduleOp.getBody()->getOps<LLVM::GlobalOp>()) {
687 existingGlobalNames.insert(globalOp.getSymName());
688 if (
auto arrayType = dyn_cast<LLVM::LLVMArrayType>(globalOp.getType())) {
689 if (globalOp.getAddrSpace() == addressSpace.value() &&
690 arrayType.getNumElements() == 0 &&
691 globalOp.getAlignment().value_or(0) == alignmentByte) {
698 unsigned uniquingCounter = 0;
701 [&](StringRef candidate) {
702 return existingGlobalNames.contains(candidate);
711 typeConverter->
convertType(memrefType.getElementType()), 0);
713 return LLVM::GlobalOp::create(rewriter, op->getLoc(), zeroSizedArrayType,
714 false, LLVM::Linkage::Internal,
716 addressSpace.value());
720 gpu::DynamicSharedMemoryOp op,
OpAdaptor adaptor,
723 MemRefType memrefType = op.getResultMemref().getType();
727 MemRefLayoutAttrInterface layout = {};
729 MemRefType::get({0}, elementType, layout, memrefType.getMemorySpace());
733 auto moduleOp = op->getParentOfType<gpu::GPUModuleOp>();
740 auto basePtr = LLVM::AddressOfOp::create(rewriter, loc, shmemOp);
741 Type baseType = basePtr->getResultTypes().front();
745 Value shmemPtr = LLVM::GEPOp::create(rewriter, loc, baseType, elementType,
753 loc, memrefType0sz, shmemPtr, shmemPtr, shape, strides, rewriter);
756 rewriter.
replaceOp(op, {memRefDescriptor});
764 unsigned numArguments = op.getNumOperands();
768 if (useBarePtrCallConv) {
771 for (
auto it : llvm::zip(op->getOperands(), adaptor.getOperands())) {
772 Type oldTy = std::get<0>(it).getType();
773 Value newOperand = std::get<1>(it);
775 cast<BaseMemRefType>(oldTy))) {
778 }
else if (isa<UnrankedMemRefType>(oldTy)) {
783 updatedOperands.push_back(newOperand);
786 updatedOperands = llvm::to_vector<4>(adaptor.getOperands());
793 if (numArguments <= 1) {
795 op,
TypeRange(), updatedOperands, op->getAttrs());
802 op.getOperandTypes(), useBarePtrCallConv);
807 Value packed = LLVM::PoisonOp::create(rewriter, loc, packedType);
809 packed = LLVM::InsertValueOp::create(rewriter, loc, packed, operand, idx);
819 [mapping](
BaseMemRefType type, gpu::AddressSpaceAttr memorySpaceAttr) {
820 gpu::AddressSpace memorySpace = memorySpaceAttr.getValue();
821 unsigned addressSpace = mapping(memorySpace);
static LLVM::GlobalOp getDynamicSharedMemorySymbol(ConversionPatternRewriter &rewriter, gpu::GPUModuleOp moduleOp, gpu::DynamicSharedMemoryOp op, const LLVMTypeConverter *typeConverter, MemRefType memrefType, unsigned alignmentBit)
Generates a symbol with 0-sized array type for dynamic shared memory usage, or uses existing symbol.
static IntegerAttr wrapNumericMemorySpace(MLIRContext *ctx, unsigned space)
static SmallString< 16 > getUniqueSymbolName(Operation *moduleOp, StringRef prefix)
static Value scalarizeVectorOpHelper(Operation *op, ValueRange operands, Type llvm1DVectorTy, ConversionPatternRewriter &rewriter, const LLVMTypeConverter &converter)
Helper for impl::scalarizeVectorOp.
static std::string diag(const llvm::Value &value)
static Value min(ImplicitLocOpBuilder &builder, Value value, Value bound)
Attributes are known-constant values of operations.
This class provides a shared interface for ranked and unranked memref types.
This class represents an argument of a Block.
IntegerAttr getIndexAttr(int64_t value)
IntegerAttr getI64IntegerAttr(int64_t value)
IntegerType getIntegerType(unsigned width)
Ty getType(Args &&...args)
Get or construct an instance of the type Ty with provided arguments.
StringAttr getStringAttr(const Twine &bytes)
MLIRContext * getContext() const
DictionaryAttr getDictionaryAttr(ArrayRef< NamedAttribute > value)
NamedAttribute getNamedAttr(StringRef name, Attribute val)
Attr getAttr(Args &&...args)
Get or construct an instance of the attribute Attr with provided arguments.
This class implements a pattern rewriter for use with ConversionPatterns.
void replaceOp(Operation *op, ValueRange newValues) override
Replace the given operation with the new values.
FailureOr< Block * > convertRegionTypes(Region *region, const TypeConverter &converter, TypeConverter::SignatureConversion *entryConversion=nullptr)
Apply a signature conversion to each block in the given region.
void eraseOp(Operation *op) override
PatternRewriter hook for erasing a dead operation.
const TypeConverter * typeConverter
An optional type converter for use by this pattern.
MemRefDescriptor createMemRefDescriptor(Location loc, MemRefType memRefType, Value allocatedPtr, Value alignedPtr, ArrayRef< Value > sizes, ArrayRef< Value > strides, ConversionPatternRewriter &rewriter) const
Creates and populates a canonical memref descriptor struct.
void getMemRefDescriptorSizes(Location loc, MemRefType memRefType, ValueRange dynamicSizes, ConversionPatternRewriter &rewriter, SmallVectorImpl< Value > &sizes, SmallVectorImpl< Value > &strides, Value &size, bool sizeInBytes=true) const
Computes sizes, strides and buffer size of memRefType with identity layout.
const LLVMTypeConverter * getTypeConverter() const
LLVM::LLVMDialect & getDialect() const
Returns the LLVM dialect.
LogicalResult copyUnrankedDescriptors(OpBuilder &builder, Location loc, TypeRange origTypes, SmallVectorImpl< Value > &operands, bool toDynamic) const
Copies the memory descriptor for any operands that were unranked descriptors originally to heap-alloc...
This class contains all of the information necessary to report a diagnostic to the DiagnosticEngine.
An instance of this location represents a tuple of file, line number, and column number.
Conversion from types to the LLVM IR dialect.
Type packFunctionResults(TypeRange types, bool useBarePointerCallConv=false, SmallVector< SmallVector< Type >> *groupedTypes=nullptr, int64_t *numConvertedTypes=nullptr) const
Convert a non-empty list of types to be returned from a function into an LLVM-compatible type.
const LowerToLLVMOptions & getOptions() const
Type convertFunctionSignature(FunctionType funcTy, bool isVariadic, bool useBarePtrCallConv, SignatureConversion &result) const
Convert a function type.
LogicalResult convertType(Type t, SmallVectorImpl< Type > &results) const
Convert the given type.
FailureOr< unsigned > getMemRefAddressSpace(BaseMemRefType type) const
Return the LLVM address space corresponding to the memory space of the memref type type or failure if...
LocationAttr findInstanceOfOrUnknown()
Return an instance of the given location type if one is nested under the current location else return...
This class defines the main interface for locations in MLIR and acts as a non-nullable wrapper around...
MLIRContext is the top-level object for a collection of MLIR operations.
Helper class to produce LLVM dialect operations extracting or inserting elements of a MemRef descript...
static MemRefDescriptor fromStaticShape(OpBuilder &builder, Location loc, const LLVMTypeConverter &typeConverter, MemRefType type, Value memory)
Builds IR creating a MemRef descriptor that represents type and populates it with static shape and st...
Value allocatedPtr(OpBuilder &builder, Location loc)
Builds IR extracting the allocated pointer from the descriptor.
NamedAttrList is array of NamedAttributes that tracks whether it is sorted and does some basic work t...
Attribute erase(StringAttr name)
Erase the attribute with the given name from the list.
RAII guard to reset the insertion point of the builder when destroyed.
This class helps build Operations.
void setInsertionPointToStart(Block *block)
Sets the insertion point to the start of the specified block.
void setInsertionPoint(Block *block, Block::iterator insertPoint)
Set the insertion point to the specified location.
Operation * create(const OperationState &state)
Creates an operation given the fields represented as an OperationState.
A trait used to provide symbol table functionalities to a region operation.
StringAttr getIdentifier() const
Return the name of this operation as a StringAttr.
Operation is the basic unit of execution within MLIR.
OpResult getResult(unsigned idx)
Get the 'idx'th result of this operation.
Location getLoc()
The source location the operation was defined or derived from.
ArrayRef< NamedAttribute > getAttrs()
Return all of the attributes on this operation.
Region & getRegion(unsigned index)
Returns the region held by this operation at position 'index'.
Operation * getParentWithTrait()
Returns the closest surrounding parent operation with trait Trait.
OperationName getName()
The name of an operation is the key identifier for it.
result_type_range getResultTypes()
iterator_range< OpIterator > getOps()
std::enable_if_t<!std::is_convertible< CallbackT, Twine >::value, LogicalResult > notifyMatchFailure(Location loc, CallbackT &&reasonCallback)
Used to notify the listener that the IR failed to be rewritten because of a match failure,...
void modifyOpInPlace(Operation *root, CallableT &&callable)
This method is a utility wrapper around an in-place modification of an operation.
void inlineRegionBefore(Region ®ion, Region &parent, Region::iterator before)
Move the blocks that belong to "region" before the given position in another region "parent".
OpTy replaceOpWithNewOp(Operation *op, Args &&...args)
Replace the results of the given (original) op with a new op that is created without verification (re...
static StringRef getSymbolAttrName()
Return the name of the attribute used for symbol names.
static Operation * lookupSymbolIn(Operation *op, StringAttr symbol)
Returns the operation registered with the given symbol name with the regions of 'symbolTableOp'.
This class provides all of the information necessary to convert a type signature.
std::optional< InputMapping > getInputMapping(unsigned input) const
Get the input mapping for the given argument.
void remapInput(unsigned origInputNo, ArrayRef< Value > replacements)
Remap an input of the original signature to replacements values.
LogicalResult convertType(Type t, SmallVectorImpl< Type > &results) const
Convert the given type.
void addTypeAttributeConversion(FnT &&callback)
Register a conversion function for attributes within types.
This class provides an abstraction over the various different ranges of value types.
Instances of the Type class are uniqued, have an immutable identifier and an optional mutable compone...
bool isIntOrFloat() const
Return true if this is an integer (of any signedness) or a float type.
unsigned getIntOrFloatBitWidth() const
Return the bit width of an integer or a float type, assert failure on other types.
This class provides an abstraction over the different types of ranges over Values.
This class represents an instance of an SSA value in the MLIR system, representing a computable value...
Type getType() const
Return the type of this value.
LogicalResult handleMultidimensionalVectors(Operation *op, ValueRange operands, const LLVMTypeConverter &typeConverter, std::function< Value(Type, ValueRange)> createOperand, ConversionPatternRewriter &rewriter)
constexpr void enumerate(std::tuple< Tys... > &tuple, CallbackT &&callback)
LogicalResult scalarizeVectorOp(Operation *op, ValueRange operands, ConversionPatternRewriter &rewriter, const LLVMTypeConverter &converter)
Unrolls op to array/vector elements.
Include the generated interface declarations.
InFlightDiagnostic emitWarning(Location loc)
Utility method to emit a warning message using this location.
std::function< unsigned(gpu::AddressSpace)> MemorySpaceMapping
A function that maps a MemorySpace enum to a target-specific integer value.
LLVM::LLVMFuncOp getOrDefineFunction(Operation *moduleOp, Location loc, OpBuilder &b, StringRef name, LLVM::LLVMFunctionType type)
Note that these functions don't take a SymbolTable because GPU module lowerings can have name collisi...
void populateGpuMemorySpaceAttributeConversions(TypeConverter &typeConverter, const MemorySpaceMapping &mapping)
Populates memory space attribute conversion rules for lowering gpu.address_space to integer values.
auto get(MLIRContext *context, Ts &&...params)
Helper method that injects context only if needed, this helps unify some of the attribute constructio...
LLVM::GlobalOp getOrCreateStringConstant(OpBuilder &b, Location loc, Operation *moduleOp, Type llvmI8, StringRef namePrefix, StringRef str, uint64_t alignment=0, unsigned addrSpace=0)
Create a global that contains the given string.
LogicalResult matchAndRewrite(gpu::DynamicSharedMemoryOp op, OpAdaptor adaptor, ConversionPatternRewriter &rewriter) const override
Methods that operate on the SourceOp type.
LogicalResult matchAndRewrite(gpu::GPUFuncOp gpuFuncOp, OpAdaptor adaptor, ConversionPatternRewriter &rewriter) const override
Methods that operate on the SourceOp type.
LogicalResult matchAndRewrite(gpu::PrintfOp gpuPrintfOp, gpu::PrintfOpAdaptor adaptor, ConversionPatternRewriter &rewriter) const override
LogicalResult matchAndRewrite(gpu::PrintfOp gpuPrintfOp, gpu::PrintfOpAdaptor adaptor, ConversionPatternRewriter &rewriter) const override
LogicalResult matchAndRewrite(gpu::PrintfOp gpuPrintfOp, gpu::PrintfOpAdaptor adaptor, ConversionPatternRewriter &rewriter) const override
LogicalResult matchAndRewrite(gpu::ReturnOp op, OpAdaptor adaptor, ConversionPatternRewriter &rewriter) const override
Methods that operate on the SourceOp type.