37 #include "llvm/ADT/PostOrderIterator.h"
38 #include "llvm/ADT/SetVector.h"
39 #include "llvm/ADT/StringExtras.h"
40 #include "llvm/ADT/TypeSwitch.h"
41 #include "llvm/Analysis/TargetFolder.h"
42 #include "llvm/Frontend/OpenMP/OMPIRBuilder.h"
43 #include "llvm/IR/BasicBlock.h"
44 #include "llvm/IR/CFG.h"
45 #include "llvm/IR/Constants.h"
46 #include "llvm/IR/DerivedTypes.h"
47 #include "llvm/IR/IRBuilder.h"
48 #include "llvm/IR/InlineAsm.h"
49 #include "llvm/IR/IntrinsicsNVPTX.h"
50 #include "llvm/IR/LLVMContext.h"
51 #include "llvm/IR/MDBuilder.h"
52 #include "llvm/IR/Module.h"
53 #include "llvm/IR/Verifier.h"
54 #include "llvm/Support/Debug.h"
55 #include "llvm/Support/raw_ostream.h"
56 #include "llvm/Transforms/Utils/BasicBlockUtils.h"
57 #include "llvm/Transforms/Utils/Cloning.h"
58 #include "llvm/Transforms/Utils/ModuleUtils.h"
62 #define DEBUG_TYPE "llvm-dialect-to-llvm-ir"
70 #include "mlir/Dialect/LLVMIR/LLVMConversionEnumsToLLVM.inc"
93 class InstructionCapturingInserter :
public llvm::IRBuilderCallbackInserter {
96 InstructionCapturingInserter()
97 : llvm::IRBuilderCallbackInserter([
this](llvm::Instruction *instruction) {
98 if (LLVM_LIKELY(enabled))
99 capturedInstructions.push_back(instruction);
104 return capturedInstructions;
108 void clearCapturedInstructions() { capturedInstructions.clear(); }
111 class CollectionScope {
114 CollectionScope(llvm::IRBuilderBase &irBuilder,
bool isBuilderCapturing);
122 return inserter->getCapturedInstructions();
127 InstructionCapturingInserter *inserter =
nullptr;
137 void setEnabled(
bool enabled =
true) { this->enabled = enabled; }
144 bool enabled =
false;
147 using CapturingIRBuilder =
148 llvm::IRBuilder<llvm::TargetFolder, InstructionCapturingInserter>;
151 InstructionCapturingInserter::CollectionScope::CollectionScope(
152 llvm::IRBuilderBase &irBuilder,
bool isBuilderCapturing) {
154 if (!isBuilderCapturing)
157 auto &capturingIRBuilder =
static_cast<CapturingIRBuilder &
>(irBuilder);
158 inserter = &capturingIRBuilder.getInserter();
159 wasEnabled = inserter->enabled;
161 previouslyCollectedInstructions.swap(inserter->capturedInstructions);
162 inserter->setEnabled(
true);
165 InstructionCapturingInserter::CollectionScope::~CollectionScope() {
169 previouslyCollectedInstructions.swap(inserter->capturedInstructions);
173 llvm::append_range(inserter->capturedInstructions,
174 previouslyCollectedInstructions);
176 inserter->setEnabled(wasEnabled);
181 static FailureOr<llvm::DataLayout>
184 std::optional<Location> loc = std::nullopt) {
189 std::string llvmDataLayout;
190 llvm::raw_string_ostream layoutStream(llvmDataLayout);
191 for (DataLayoutEntryInterface entry : attribute.getEntries()) {
192 auto key = llvm::dyn_cast_if_present<StringAttr>(entry.getKey());
195 if (key.getValue() == DLTIDialect::kDataLayoutEndiannessKey) {
196 auto value = cast<StringAttr>(entry.getValue());
197 bool isLittleEndian =
198 value.getValue() == DLTIDialect::kDataLayoutEndiannessLittle;
199 layoutStream <<
"-" << (isLittleEndian ?
"e" :
"E");
202 if (key.getValue() == DLTIDialect::kDataLayoutProgramMemorySpaceKey) {
203 auto value = cast<IntegerAttr>(entry.getValue());
204 uint64_t space = value.getValue().getZExtValue();
208 layoutStream <<
"-P" << space;
211 if (key.getValue() == DLTIDialect::kDataLayoutGlobalMemorySpaceKey) {
212 auto value = cast<IntegerAttr>(entry.getValue());
213 uint64_t space = value.getValue().getZExtValue();
217 layoutStream <<
"-G" << space;
220 if (key.getValue() == DLTIDialect::kDataLayoutAllocaMemorySpaceKey) {
221 auto value = cast<IntegerAttr>(entry.getValue());
222 uint64_t space = value.getValue().getZExtValue();
226 layoutStream <<
"-A" << space;
229 if (key.getValue() == DLTIDialect::kDataLayoutStackAlignmentKey) {
230 auto value = cast<IntegerAttr>(entry.getValue());
231 uint64_t alignment = value.getValue().getZExtValue();
235 layoutStream <<
"-S" << alignment;
238 emitError(*loc) <<
"unsupported data layout key " << key;
245 for (DataLayoutEntryInterface entry : attribute.getEntries()) {
246 auto type = llvm::dyn_cast_if_present<Type>(entry.getKey());
250 if (isa<IndexType>(type))
253 LogicalResult result =
255 .Case<IntegerType, Float16Type, Float32Type, Float64Type,
256 Float80Type, Float128Type>([&](
Type type) -> LogicalResult {
257 if (
auto intType = dyn_cast<IntegerType>(type)) {
258 if (intType.getSignedness() != IntegerType::Signless)
260 <<
"unsupported data layout for non-signless integer "
270 layoutStream << size <<
":" << abi;
271 if (abi != preferred)
272 layoutStream <<
":" << preferred;
275 .Case([&](LLVMPointerType type) {
276 layoutStream <<
"p" << type.getAddressSpace() <<
":";
282 layoutStream << size <<
":" << abi <<
":" << preferred <<
":"
286 .Default([loc](
Type type) {
288 <<
"unsupported type in data layout: " << type;
293 StringRef layoutSpec(llvmDataLayout);
294 if (layoutSpec.starts_with(
"-"))
295 layoutSpec = layoutSpec.drop_front();
297 return llvm::DataLayout(layoutSpec);
304 static llvm::Constant *
309 llvm::Constant *result = constants.front();
310 constants = constants.drop_front();
314 llvm::Type *elementType;
315 if (
auto *arrayTy = dyn_cast<llvm::ArrayType>(type)) {
316 elementType = arrayTy->getElementType();
317 }
else if (
auto *vectorTy = dyn_cast<llvm::VectorType>(type)) {
318 elementType = vectorTy->getElementType();
320 emitError(loc) <<
"expected sequential LLVM types wrapping a scalar";
325 nested.reserve(shape.front());
326 for (int64_t i = 0; i < shape.front(); ++i) {
333 if (shape.size() == 1 && type->isVectorTy())
342 if (
auto *arrayTy = dyn_cast<llvm::ArrayType>(type)) {
343 type = arrayTy->getElementType();
344 }
else if (
auto *vectorTy = dyn_cast<llvm::VectorType>(type)) {
345 type = vectorTy->getElementType();
359 static llvm::Constant *
361 llvm::Type *llvmType,
363 if (!denseElementsAttr)
367 if (!llvm::ConstantDataSequential::isElementTypeCompatible(innermostLLVMType))
370 ShapedType type = denseElementsAttr.
getType();
371 if (type.getNumElements() == 0)
379 int64_t elementByteSize = denseElementsAttr.
getRawData().size() /
381 if (8 * elementByteSize != innermostLLVMType->getScalarSizeInBits())
386 bool hasVectorElementType = isa<VectorType>(type.getElementType());
387 int64_t numAggregates =
389 (hasVectorElementType ? 1
390 : denseElementsAttr.
getType().getShape().back());
392 if (!hasVectorElementType)
393 outerShape = outerShape.drop_back();
396 if (denseElementsAttr.
isSplat() &&
397 (isa<VectorType>(type) || hasVectorElementType)) {
401 llvm::Constant *splatVector =
402 llvm::ConstantDataVector::getSplat(0, splatValue);
407 if (denseElementsAttr.
isSplat())
412 std::function<llvm::Constant *(StringRef)> buildCstData;
413 if (isa<TensorType>(type)) {
414 auto vectorElementType = dyn_cast<VectorType>(type.getElementType());
415 if (vectorElementType && vectorElementType.getRank() == 1) {
416 buildCstData = [&](StringRef data) {
417 return llvm::ConstantDataVector::getRaw(
418 data, vectorElementType.getShape().back(), innermostLLVMType);
420 }
else if (!vectorElementType) {
421 buildCstData = [&](StringRef data) {
422 return llvm::ConstantDataArray::getRaw(data, type.getShape().back(),
426 }
else if (isa<VectorType>(type)) {
427 buildCstData = [&](StringRef data) {
428 return llvm::ConstantDataVector::getRaw(data, type.getShape().back(),
438 int64_t aggregateSize = denseElementsAttr.
getType().getShape().back() *
439 (innermostLLVMType->getScalarSizeInBits() / 8);
440 constants.reserve(numAggregates);
441 for (
unsigned i = 0; i < numAggregates; ++i) {
442 StringRef data(denseElementsAttr.
getRawData().data() + i * aggregateSize,
444 constants.push_back(buildCstData(data));
460 assert(denseResourceAttr &&
"expected non-null attribute");
463 if (!llvm::ConstantDataSequential::isElementTypeCompatible(
464 innermostLLVMType)) {
465 emitError(loc,
"no known conversion for innermost element type");
469 ShapedType type = denseResourceAttr.getType();
470 assert(type.getNumElements() > 0 &&
"Expected non-empty elements attribute");
474 emitError(loc,
"resource does not exist");
485 int64_t numElements = denseResourceAttr.getType().getNumElements();
486 int64_t elementByteSize = rawData.size() / numElements;
487 if (8 * elementByteSize != innermostLLVMType->getScalarSizeInBits()) {
488 emitError(loc,
"raw data size does not match element type size");
494 bool hasVectorElementType = isa<VectorType>(type.getElementType());
495 int64_t numAggregates =
496 numElements / (hasVectorElementType
498 : denseResourceAttr.getType().getShape().back());
500 if (!hasVectorElementType)
501 outerShape = outerShape.drop_back();
504 std::function<llvm::Constant *(StringRef)> buildCstData;
505 if (isa<TensorType>(type)) {
506 auto vectorElementType = dyn_cast<VectorType>(type.getElementType());
507 if (vectorElementType && vectorElementType.getRank() == 1) {
508 buildCstData = [&](StringRef data) {
509 return llvm::ConstantDataVector::getRaw(
510 data, vectorElementType.getShape().back(), innermostLLVMType);
512 }
else if (!vectorElementType) {
513 buildCstData = [&](StringRef data) {
514 return llvm::ConstantDataArray::getRaw(data, type.getShape().back(),
518 }
else if (isa<VectorType>(type)) {
519 buildCstData = [&](StringRef data) {
520 return llvm::ConstantDataVector::getRaw(data, type.getShape().back(),
525 emitError(loc,
"unsupported dense_resource type");
532 int64_t aggregateSize = denseResourceAttr.getType().getShape().back() *
533 (innermostLLVMType->getScalarSizeInBits() / 8);
534 constants.reserve(numAggregates);
535 for (
unsigned i = 0; i < numAggregates; ++i) {
536 StringRef data(rawData.data() + i * aggregateSize, aggregateSize);
537 constants.push_back(buildCstData(data));
554 if (
auto *structType = dyn_cast<::llvm::StructType>(llvmType)) {
555 auto arrayAttr = dyn_cast<ArrayAttr>(attr);
557 emitError(loc,
"expected an array attribute for a struct constant");
561 structElements.reserve(structType->getNumElements());
562 for (
auto [elemType, elemAttr] :
563 zip_equal(structType->elements(), arrayAttr)) {
564 llvm::Constant *element =
568 structElements.push_back(element);
574 if (
auto intAttr = dyn_cast<IntegerAttr>(attr))
577 intAttr.getValue().sextOrTrunc(llvmType->getIntegerBitWidth()));
578 if (
auto floatAttr = dyn_cast<FloatAttr>(attr)) {
579 const llvm::fltSemantics &sem = floatAttr.getValue().getSemantics();
584 unsigned floatWidth = APFloat::getSizeInBits(sem);
585 if (llvmType->isIntegerTy(floatWidth))
587 floatAttr.getValue().bitcastToAPInt());
589 llvm::Type::getFloatingPointTy(llvmType->getContext(),
590 floatAttr.getValue().getSemantics())) {
591 emitError(loc,
"FloatAttr does not match expected type of the constant");
596 if (
auto funcAttr = dyn_cast<FlatSymbolRefAttr>(attr))
597 return llvm::ConstantExpr::getBitCast(
599 if (
auto splatAttr = dyn_cast<SplatElementsAttr>(attr)) {
600 llvm::Type *elementType;
601 uint64_t numElements;
602 bool isScalable =
false;
603 if (
auto *arrayTy = dyn_cast<llvm::ArrayType>(llvmType)) {
604 elementType = arrayTy->getElementType();
605 numElements = arrayTy->getNumElements();
606 }
else if (
auto *fVectorTy = dyn_cast<llvm::FixedVectorType>(llvmType)) {
607 elementType = fVectorTy->getElementType();
608 numElements = fVectorTy->getNumElements();
609 }
else if (
auto *sVectorTy = dyn_cast<llvm::ScalableVectorType>(llvmType)) {
610 elementType = sVectorTy->getElementType();
611 numElements = sVectorTy->getMinNumElements();
614 llvm_unreachable(
"unrecognized constant vector type");
619 bool elementTypeSequential =
620 isa<llvm::ArrayType, llvm::VectorType>(elementType);
623 elementTypeSequential ? splatAttr
625 loc, moduleTranslation);
628 if (llvmType->isVectorTy())
629 return llvm::ConstantVector::getSplat(
631 if (llvmType->isArrayTy()) {
633 if (child->isZeroValue()) {
636 if (llvm::ConstantDataSequential::isElementTypeCompatible(
639 if (isa<llvm::IntegerType>(elementType)) {
640 if (llvm::ConstantInt *ci = dyn_cast<llvm::ConstantInt>(child)) {
641 if (ci->getBitWidth() == 8) {
646 if (ci->getBitWidth() == 16) {
651 if (ci->getBitWidth() == 32) {
656 if (ci->getBitWidth() == 64) {
666 std::vector<llvm::Constant *> constants(numElements, child);
673 if (llvm::Constant *result =
675 llvmType, moduleTranslation)) {
679 if (
auto denseResourceAttr = dyn_cast<DenseResourceElementsAttr>(attr)) {
685 if (
auto elementsAttr = dyn_cast<ElementsAttr>(attr)) {
686 assert(elementsAttr.getShapedType().hasStaticShape());
687 assert(!elementsAttr.getShapedType().getShape().empty() &&
688 "unexpected empty elements attribute shape");
691 constants.reserve(elementsAttr.getNumElements());
693 for (
auto n : elementsAttr.getValues<
Attribute>()) {
696 if (!constants.back())
701 constantsRef, elementsAttr.getShapedType().getShape(), llvmType, loc);
702 assert(constantsRef.empty() &&
"did not consume all elemental constants");
706 if (
auto stringAttr = dyn_cast<StringAttr>(attr)) {
710 stringAttr.getValue().size()});
712 emitError(loc,
"unsupported constant value");
716 ModuleTranslation::ModuleTranslation(
Operation *module,
717 std::unique_ptr<llvm::Module> llvmModule)
718 : mlirModule(module), llvmModule(std::move(llvmModule)),
722 *this, *this->llvmModule)),
723 typeTranslator(this->llvmModule->
getContext()),
726 "mlirModule should honor LLVM's module semantics.");
729 ModuleTranslation::~ModuleTranslation() {
731 ompBuilder->finalize();
736 toProcess.push_back(®ion);
737 while (!toProcess.empty()) {
738 Region *current = toProcess.pop_back_val();
739 for (
Block &block : *current) {
740 blockMapping.erase(&block);
741 for (
Value arg : block.getArguments())
742 valueMapping.erase(arg);
744 for (
Value value : op.getResults())
745 valueMapping.erase(value);
746 if (op.hasSuccessors())
747 branchMapping.erase(&op);
748 if (isa<LLVM::GlobalOp>(op))
749 globalsMapping.erase(&op);
750 if (isa<LLVM::AliasOp>(op))
751 aliasesMapping.erase(&op);
752 if (isa<LLVM::CallOp>(op))
753 callMapping.erase(&op);
756 llvm::map_range(op.getRegions(), [](
Region &r) { return &r; }));
765 unsigned numArguments,
unsigned index) {
767 if (isa<LLVM::BrOp>(terminator))
774 auto branch = cast<BranchOpInterface>(terminator);
777 (!seenSuccessors.contains(successor) || successorOperands.
empty()) &&
778 "successors with arguments in LLVM branches must be different blocks");
779 seenSuccessors.insert(successor);
785 if (
auto condBranchOp = dyn_cast<LLVM::CondBrOp>(terminator)) {
788 return condBranchOp.getSuccessor(0) == current
789 ? condBranchOp.getTrueDestOperands()[index]
790 : condBranchOp.getFalseDestOperands()[index];
793 if (
auto switchOp = dyn_cast<LLVM::SwitchOp>(terminator)) {
796 if (switchOp.getDefaultDestination() == current)
797 return switchOp.getDefaultOperands()[index];
799 if (i.value() == current)
800 return switchOp.getCaseOperands(i.index())[index];
803 if (
auto invokeOp = dyn_cast<LLVM::InvokeOp>(terminator)) {
804 return invokeOp.getNormalDest() == current
805 ? invokeOp.getNormalDestOperands()[index]
806 : invokeOp.getUnwindDestOperands()[index];
810 "only branch, switch or invoke operations can be terminators "
811 "of a block that has successors");
819 for (
Block &bb : llvm::drop_begin(region)) {
820 llvm::BasicBlock *llvmBB = state.lookupBlock(&bb);
821 auto phis = llvmBB->phis();
822 auto numArguments = bb.getNumArguments();
823 assert(numArguments == std::distance(phis.begin(), phis.end()));
825 for (
auto *pred : bb.getPredecessors()) {
831 llvm::Instruction *terminator =
832 state.lookupBranch(pred->getTerminator());
833 assert(terminator &&
"missing the mapping for a terminator");
835 &bb, pred, numArguments, index)),
836 terminator->getParent());
845 llvm::Module *module = builder.GetInsertBlock()->getModule();
847 llvm::Intrinsic::getOrInsertDeclaration(module, intrinsic, tys);
848 return builder.CreateCall(fn, args);
857 assert(immArgPositions.size() == immArgAttrNames.size() &&
858 "LLVM `immArgPositions` and MLIR `immArgAttrNames` should have equal "
862 size_t numOpBundleOperands = 0;
863 auto opBundleSizesAttr = cast_if_present<DenseI32ArrayAttr>(
864 intrOp->
getAttr(LLVMDialect::getOpBundleSizesAttrName()));
865 auto opBundleTagsAttr = cast_if_present<ArrayAttr>(
866 intrOp->
getAttr(LLVMDialect::getOpBundleTagsAttrName()));
868 if (opBundleSizesAttr && opBundleTagsAttr) {
869 ArrayRef<int> opBundleSizes = opBundleSizesAttr.asArrayRef();
870 assert(opBundleSizes.size() == opBundleTagsAttr.size() &&
871 "operand bundles and tags do not match");
873 numOpBundleOperands =
874 std::accumulate(opBundleSizes.begin(), opBundleSizes.end(),
size_t(0));
875 assert(numOpBundleOperands <= intrOp->getNumOperands() &&
876 "operand bundle operands is more than the number of operands");
879 size_t nextOperandIdx = 0;
880 opBundles.reserve(opBundleSizesAttr.size());
882 for (
auto [opBundleTagAttr, bundleSize] :
883 llvm::zip(opBundleTagsAttr, opBundleSizes)) {
884 auto bundleTag = cast<StringAttr>(opBundleTagAttr).str();
886 operands.slice(nextOperandIdx, bundleSize));
887 opBundles.emplace_back(std::move(bundleTag), std::move(bundleOperands));
888 nextOperandIdx += bundleSize;
893 auto opOperands = intrOp->
getOperands().drop_back(numOpBundleOperands);
894 auto operands = moduleTranslation.
lookupValues(opOperands);
896 for (
auto [immArgPos, immArgName] :
897 llvm::zip(immArgPositions, immArgAttrNames)) {
898 auto attr = llvm::cast<TypedAttr>(intrOp->
getAttr(immArgName));
899 assert(attr.getType().isIntOrFloat() &&
"expected int or float immarg");
900 auto *type = moduleTranslation.
convertType(attr.getType());
902 type, attr, intrOp->
getLoc(), moduleTranslation);
905 for (
auto &arg : args) {
907 arg = operands[opArg++];
912 for (
unsigned overloadedResultIdx : overloadedResults) {
913 if (numResults > 1) {
915 overloadedTypes.push_back(moduleTranslation.
convertType(
917 .getBody()[overloadedResultIdx]));
919 overloadedTypes.push_back(
923 for (
unsigned overloadedOperandIdx : overloadedOperands)
924 overloadedTypes.push_back(args[overloadedOperandIdx]->
getType());
925 llvm::Module *module = builder.GetInsertBlock()->getModule();
926 llvm::Function *llvmIntr = llvm::Intrinsic::getOrInsertDeclaration(
927 module, intrinsic, overloadedTypes);
929 return builder.CreateCall(llvmIntr, args, opBundles);
934 LogicalResult ModuleTranslation::convertOperation(
Operation &op,
935 llvm::IRBuilderBase &builder,
936 bool recordInsertions) {
939 return op.
emitError(
"cannot be converted to LLVM IR: missing "
940 "`LLVMTranslationDialectInterface` registration for "
944 InstructionCapturingInserter::CollectionScope scope(builder,
947 return op.
emitError(
"LLVM Translation failed for operation: ")
950 return convertDialectAttributes(&op, scope.getCapturedInstructions());
960 LogicalResult ModuleTranslation::convertBlockImpl(
Block &bb,
961 bool ignoreArguments,
962 llvm::IRBuilderBase &builder,
963 bool recordInsertions) {
965 auto *subprogram = builder.GetInsertBlock()->getParent()->getSubprogram();
973 if (!ignoreArguments) {
975 unsigned numPredecessors =
976 std::distance(predecessors.begin(), predecessors.end());
978 auto wrappedType = arg.getType();
981 "block argument does not have an LLVM type");
982 builder.SetCurrentDebugLocation(
983 debugTranslation->translateLoc(arg.getLoc(), subprogram));
985 llvm::PHINode *phi = builder.CreatePHI(type, numPredecessors);
991 for (
auto &op : bb) {
993 builder.SetCurrentDebugLocation(
994 debugTranslation->translateLoc(op.
getLoc(), subprogram));
996 if (failed(convertOperation(op, builder, recordInsertions)))
1000 if (
auto iface = dyn_cast<BranchWeightOpInterface>(op))
1019 llvm::Constant *cst) {
1020 return (linkage == llvm::GlobalVariable::ExternalLinkage && !cst) ||
1021 linkage == llvm::GlobalVariable::ExternalWeakLinkage;
1027 llvm::GlobalValue *gv) {
1028 if (dsoLocalRequested)
1029 gv->setDSOLocal(
true);
1032 LogicalResult ModuleTranslation::convertGlobalsAndAliases() {
1041 for (
auto op :
getModuleBody(mlirModule).getOps<LLVM::GlobalOp>()) {
1043 llvm::Constant *cst =
nullptr;
1044 if (op.getValueOrNull()) {
1047 if (
auto strAttr = dyn_cast_or_null<StringAttr>(op.getValueOrNull())) {
1048 cst = llvm::ConstantDataArray::getString(
1049 llvmModule->getContext(), strAttr.getValue(),
false);
1050 type = cst->getType();
1057 auto linkage = convertLinkageToLLVM(op.getLinkage());
1063 if (!dropInitializer && !cst)
1065 else if (dropInitializer && cst)
1068 auto *var =
new llvm::GlobalVariable(
1069 *llvmModule, type, op.getConstant(), linkage, cst, op.getSymName(),
1071 op.getThreadLocal_() ? llvm::GlobalValue::GeneralDynamicTLSModel
1072 : llvm::GlobalValue::NotThreadLocal,
1073 op.getAddrSpace(), op.getExternallyInitialized());
1075 if (std::optional<mlir::SymbolRefAttr> comdat = op.getComdat()) {
1076 auto selectorOp = cast<ComdatSelectorOp>(
1078 var->setComdat(comdatMapping.lookup(selectorOp));
1081 if (op.getUnnamedAddr().has_value())
1082 var->setUnnamedAddr(convertUnnamedAddrToLLVM(*op.getUnnamedAddr()));
1084 if (op.getSection().has_value())
1085 var->setSection(*op.getSection());
1089 std::optional<uint64_t> alignment = op.getAlignment();
1090 if (alignment.has_value())
1091 var->setAlignment(llvm::MaybeAlign(alignment.value()));
1093 var->setVisibility(convertVisibilityToLLVM(op.getVisibility_()));
1095 globalsMapping.try_emplace(op, var);
1098 if (op.getDbgExprs()) {
1099 for (
auto exprAttr :
1100 op.getDbgExprs()->getAsRange<DIGlobalVariableExpressionAttr>()) {
1101 llvm::DIGlobalVariableExpression *diGlobalExpr =
1102 debugTranslation->translateGlobalVariableExpression(exprAttr);
1103 llvm::DIGlobalVariable *diGlobalVar = diGlobalExpr->getVariable();
1104 var->addDebugInfo(diGlobalExpr);
1123 llvm::DIScope *scope = diGlobalVar->getScope();
1124 if (
auto *mod = dyn_cast_if_present<llvm::DIModule>(scope))
1125 scope = mod->getScope();
1126 else if (
auto *cb = dyn_cast_if_present<llvm::DICommonBlock>(scope)) {
1128 dyn_cast_if_present<llvm::DISubprogram>(cb->getScope()))
1129 scope = sp->getUnit();
1130 }
else if (
auto *sp = dyn_cast_if_present<llvm::DISubprogram>(scope))
1131 scope = sp->getUnit();
1134 if (llvm::DICompileUnit *compileUnit =
1135 dyn_cast_if_present<llvm::DICompileUnit>(scope)) {
1138 allGVars[compileUnit].push_back(diGlobalExpr);
1145 for (
auto op :
getModuleBody(mlirModule).getOps<LLVM::AliasOp>()) {
1147 llvm::Constant *cst =
nullptr;
1148 llvm::GlobalValue::LinkageTypes linkage =
1149 convertLinkageToLLVM(op.getLinkage());
1150 llvm::Module &llvmMod = *llvmModule;
1153 llvm::GlobalAlias *var = llvm::GlobalAlias::create(
1154 type, op.getAddrSpace(), linkage, op.getSymName(), cst,
1157 var->setThreadLocalMode(op.getThreadLocal_()
1158 ? llvm::GlobalAlias::GeneralDynamicTLSModel
1159 : llvm::GlobalAlias::NotThreadLocal);
1164 if (op.getUnnamedAddr().has_value())
1165 var->setUnnamedAddr(convertUnnamedAddrToLLVM(*op.getUnnamedAddr()));
1167 var->setVisibility(convertVisibilityToLLVM(op.getVisibility_()));
1169 aliasesMapping.try_emplace(op, var);
1173 for (
auto op :
getModuleBody(mlirModule).getOps<LLVM::GlobalOp>()) {
1174 if (
Block *initializer = op.getInitializerBlock()) {
1175 llvm::IRBuilder<llvm::TargetFolder> builder(
1176 llvmModule->getContext(),
1177 llvm::TargetFolder(llvmModule->getDataLayout()));
1179 [[maybe_unused]]
int numConstantsHit = 0;
1180 [[maybe_unused]]
int numConstantsErased = 0;
1183 for (
auto &op : initializer->without_terminator()) {
1184 if (failed(convertOperation(op, builder)))
1197 if (
auto *agg = dyn_cast<llvm::ConstantAggregate>(cst)) {
1201 auto [iterator, inserted] =
1202 constantAggregateUseMap.try_emplace(agg, numUsers);
1205 iterator->second += numUsers;
1211 auto cst = dyn_cast<llvm::ConstantAggregate>(
lookupValue(v));
1214 auto iter = constantAggregateUseMap.find(cst);
1215 assert(iter != constantAggregateUseMap.end() &&
"constant not found");
1217 if (iter->second == 0) {
1220 if (cst->user_empty()) {
1221 cst->destroyConstant();
1222 numConstantsErased++;
1224 constantAggregateUseMap.erase(iter);
1229 ReturnOp ret = cast<ReturnOp>(initializer->getTerminator());
1230 llvm::Constant *cst =
1231 cast<llvm::Constant>(
lookupValue(ret.getOperand(0)));
1232 auto *global = cast<llvm::GlobalVariable>(
lookupGlobal(op));
1234 global->setInitializer(cst);
1238 for (
auto it : constantAggregateUseMap) {
1239 auto cst = it.first;
1240 cst->removeDeadConstantUsers();
1241 if (cst->user_empty()) {
1242 cst->destroyConstant();
1243 numConstantsErased++;
1247 LLVM_DEBUG(llvm::dbgs()
1248 <<
"Convert initializer for " << op.
getName() <<
"\n";
1249 llvm::dbgs() << numConstantsHit <<
" new constants hit\n";
1251 << numConstantsErased <<
" dangling constants erased\n";);
1257 auto ctorOp = dyn_cast<GlobalCtorsOp>(op);
1258 auto dtorOp = dyn_cast<GlobalDtorsOp>(op);
1259 if (!ctorOp && !dtorOp)
1261 auto range = ctorOp ? llvm::zip(ctorOp.getCtors(), ctorOp.getPriorities())
1262 :
llvm::zip(dtorOp.getDtors(), dtorOp.getPriorities());
1263 auto appendGlobalFn =
1264 ctorOp ? llvm::appendToGlobalCtors : llvm::appendToGlobalDtors;
1265 for (
auto symbolAndPriority : range) {
1267 cast<FlatSymbolRefAttr>(std::get<0>(symbolAndPriority)).getValue());
1268 appendGlobalFn(*llvmModule, f,
1269 cast<IntegerAttr>(std::get<1>(symbolAndPriority)).getInt(),
1274 for (
auto op :
getModuleBody(mlirModule).getOps<LLVM::GlobalOp>())
1275 if (failed(convertDialectAttributes(op, {})))
1280 for (
const auto &[compileUnit, globals] : allGVars) {
1281 compileUnit->replaceGlobalVariables(
1286 for (
auto op :
getModuleBody(mlirModule).getOps<LLVM::AliasOp>()) {
1287 Block &initializer = op.getInitializerBlock();
1288 llvm::IRBuilder<llvm::TargetFolder> builder(
1289 llvmModule->getContext(),
1290 llvm::TargetFolder(llvmModule->getDataLayout()));
1293 if (failed(convertOperation(op, builder)))
1300 auto *cst = cast<llvm::Constant>(
lookupValue(ret.getOperand(0)));
1301 assert(aliasesMapping.count(op));
1302 auto *alias = cast<llvm::GlobalAlias>(aliasesMapping[op]);
1303 alias->setAliasee(cst);
1306 for (
auto op :
getModuleBody(mlirModule).getOps<LLVM::AliasOp>())
1307 if (failed(convertDialectAttributes(op, {})))
1320 llvm::Function *llvmFunc,
1322 StringRef value = StringRef()) {
1323 auto kind = llvm::Attribute::getAttrKindFromName(key);
1325 llvmFunc->addFnAttr(key, value);
1329 if (llvm::Attribute::isIntAttrKind(kind)) {
1331 return emitError(loc) <<
"LLVM attribute '" << key <<
"' expects a value";
1334 if (!value.getAsInteger(0, result))
1335 llvmFunc->addFnAttr(
1338 llvmFunc->addFnAttr(key, value);
1343 return emitError(loc) <<
"LLVM attribute '" << key
1344 <<
"' does not expect a value, found '" << value
1347 llvmFunc->addFnAttr(kind);
1353 const llvm::APInt &value) {
1360 const llvm::APInt &value) {
1368 llvm::Metadata *typeMD =
1370 llvm::Metadata *isSignedMD =
1380 values, std::back_inserter(mdValues), [&context](int32_t value) {
1393 static LogicalResult
1395 llvm::Function *llvmFunc) {
1400 if (
auto stringAttr = dyn_cast<StringAttr>(attr)) {
1407 auto arrayAttr = dyn_cast<ArrayAttr>(attr);
1408 if (!arrayAttr || arrayAttr.size() != 2)
1410 <<
"expected 'passthrough' to contain string or array attributes";
1412 auto keyAttr = dyn_cast<StringAttr>(arrayAttr[0]);
1413 auto valueAttr = dyn_cast<StringAttr>(arrayAttr[1]);
1414 if (!keyAttr || !valueAttr)
1416 <<
"expected arrays within 'passthrough' to contain two strings";
1419 valueAttr.getValue())))
1425 LogicalResult ModuleTranslation::convertOneFunction(LLVMFuncOp func) {
1428 blockMapping.clear();
1429 valueMapping.clear();
1430 branchMapping.clear();
1434 for (
auto [mlirArg, llvmArg] :
1435 llvm::zip(func.getArguments(), llvmFunc->args()))
1439 if (func.getPersonality()) {
1440 llvm::Type *ty = llvm::PointerType::getUnqual(llvmFunc->getContext());
1441 if (llvm::Constant *pfunc =
getLLVMConstant(ty, func.getPersonalityAttr(),
1442 func.getLoc(), *
this))
1443 llvmFunc->setPersonalityFn(pfunc);
1446 if (std::optional<StringRef> section = func.getSection())
1447 llvmFunc->setSection(*section);
1449 if (func.getArmStreaming())
1450 llvmFunc->addFnAttr(
"aarch64_pstate_sm_enabled");
1451 else if (func.getArmLocallyStreaming())
1452 llvmFunc->addFnAttr(
"aarch64_pstate_sm_body");
1453 else if (func.getArmStreamingCompatible())
1454 llvmFunc->addFnAttr(
"aarch64_pstate_sm_compatible");
1456 if (func.getArmNewZa())
1457 llvmFunc->addFnAttr(
"aarch64_new_za");
1458 else if (func.getArmInZa())
1459 llvmFunc->addFnAttr(
"aarch64_in_za");
1460 else if (func.getArmOutZa())
1461 llvmFunc->addFnAttr(
"aarch64_out_za");
1462 else if (func.getArmInoutZa())
1463 llvmFunc->addFnAttr(
"aarch64_inout_za");
1464 else if (func.getArmPreservesZa())
1465 llvmFunc->addFnAttr(
"aarch64_preserves_za");
1467 if (
auto targetCpu = func.getTargetCpu())
1468 llvmFunc->addFnAttr(
"target-cpu", *targetCpu);
1470 if (
auto tuneCpu = func.getTuneCpu())
1471 llvmFunc->addFnAttr(
"tune-cpu", *tuneCpu);
1473 if (
auto targetFeatures = func.getTargetFeatures())
1474 llvmFunc->addFnAttr(
"target-features", targetFeatures->getFeaturesString());
1476 if (
auto attr = func.getVscaleRange())
1477 llvmFunc->addFnAttr(llvm::Attribute::getWithVScaleRangeArgs(
1479 attr->getMaxRange().getInt()));
1481 if (
auto unsafeFpMath = func.getUnsafeFpMath())
1482 llvmFunc->addFnAttr(
"unsafe-fp-math", llvm::toStringRef(*unsafeFpMath));
1484 if (
auto noInfsFpMath = func.getNoInfsFpMath())
1485 llvmFunc->addFnAttr(
"no-infs-fp-math", llvm::toStringRef(*noInfsFpMath));
1487 if (
auto noNansFpMath = func.getNoNansFpMath())
1488 llvmFunc->addFnAttr(
"no-nans-fp-math", llvm::toStringRef(*noNansFpMath));
1490 if (
auto approxFuncFpMath = func.getApproxFuncFpMath())
1491 llvmFunc->addFnAttr(
"approx-func-fp-math",
1492 llvm::toStringRef(*approxFuncFpMath));
1494 if (
auto noSignedZerosFpMath = func.getNoSignedZerosFpMath())
1495 llvmFunc->addFnAttr(
"no-signed-zeros-fp-math",
1496 llvm::toStringRef(*noSignedZerosFpMath));
1498 if (
auto denormalFpMath = func.getDenormalFpMath())
1499 llvmFunc->addFnAttr(
"denormal-fp-math", *denormalFpMath);
1501 if (
auto denormalFpMathF32 = func.getDenormalFpMathF32())
1502 llvmFunc->addFnAttr(
"denormal-fp-math-f32", *denormalFpMathF32);
1504 if (
auto fpContract = func.getFpContract())
1505 llvmFunc->addFnAttr(
"fp-contract", *fpContract);
1508 if (FramePointerKindAttr attr = func.getFramePointerAttr())
1509 llvmFunc->addFnAttr(
"frame-pointer",
1510 LLVM::framePointerKind::stringifyFramePointerKind(
1511 (attr.getFramePointerKind())));
1514 llvm::LLVMContext &llvmContext = llvmFunc->getContext();
1515 for (
auto &bb : func) {
1516 auto *llvmBB = llvm::BasicBlock::Create(llvmContext);
1517 llvmBB->insertInto(llvmFunc);
1524 for (
Block *bb : blocks) {
1525 CapturingIRBuilder builder(llvmContext,
1526 llvm::TargetFolder(llvmModule->getDataLayout()));
1527 if (failed(convertBlockImpl(*bb, bb->isEntryBlock(), builder,
1537 return convertDialectAttributes(func, {});
1540 LogicalResult ModuleTranslation::convertDialectAttributes(
1543 if (failed(iface.
amendOperation(op, instructions, attribute, *
this)))
1551 llvm::Function *llvmFunc) {
1552 if (!func.getMemoryEffects())
1555 MemoryEffectsAttr memEffects = func.getMemoryEffectsAttr();
1558 llvm::MemoryEffects newMemEffects =
1559 llvm::MemoryEffects(llvm::MemoryEffects::Location::ArgMem,
1560 convertModRefInfoToLLVM(memEffects.getArgMem()));
1561 newMemEffects |= llvm::MemoryEffects(
1562 llvm::MemoryEffects::Location::InaccessibleMem,
1563 convertModRefInfoToLLVM(memEffects.getInaccessibleMem()));
1565 llvm::MemoryEffects(llvm::MemoryEffects::Location::Other,
1566 convertModRefInfoToLLVM(memEffects.getOther()));
1567 llvmFunc->setMemoryEffects(newMemEffects);
1572 llvm::Function *llvmFunc) {
1573 if (func.getNoInlineAttr())
1574 llvmFunc->addFnAttr(llvm::Attribute::NoInline);
1575 if (func.getAlwaysInlineAttr())
1576 llvmFunc->addFnAttr(llvm::Attribute::AlwaysInline);
1577 if (func.getOptimizeNoneAttr())
1578 llvmFunc->addFnAttr(llvm::Attribute::OptimizeNone);
1579 if (func.getConvergentAttr())
1580 llvmFunc->addFnAttr(llvm::Attribute::Convergent);
1581 if (func.getNoUnwindAttr())
1582 llvmFunc->addFnAttr(llvm::Attribute::NoUnwind);
1583 if (func.getWillReturnAttr())
1584 llvmFunc->addFnAttr(llvm::Attribute::WillReturn);
1590 llvm::Function *llvmFunc,
1592 llvm::LLVMContext &llvmContext = llvmFunc->getContext();
1594 if (VecTypeHintAttr vecTypeHint = func.getVecTypeHintAttr()) {
1595 Type type = vecTypeHint.getHint().getValue();
1596 llvm::Type *llvmType = translation.
convertType(type);
1597 bool isSigned = vecTypeHint.getIsSigned();
1598 llvmFunc->setMetadata(
1599 func.getVecTypeHintAttrName(),
1604 func.getWorkGroupSizeHint()) {
1605 llvmFunc->setMetadata(
1606 func.getWorkGroupSizeHintAttrName(),
1611 func.getReqdWorkGroupSize()) {
1612 llvmFunc->setMetadata(
1613 func.getReqdWorkGroupSizeAttrName(),
1617 if (std::optional<uint32_t> intelReqdSubGroupSize =
1618 func.getIntelReqdSubGroupSize()) {
1619 llvmFunc->setMetadata(
1620 func.getIntelReqdSubGroupSizeAttrName(),
1622 llvm::APInt(32, *intelReqdSubGroupSize)));
1627 llvm::Attribute::AttrKind llvmKind,
1632 .Case<TypeAttr>([&](
auto typeAttr) {
1633 attrBuilder.addTypeAttr(
1634 llvmKind, moduleTranslation.
convertType(typeAttr.getValue()));
1637 .Case<IntegerAttr>([&](
auto intAttr) {
1638 attrBuilder.addRawIntAttr(llvmKind, intAttr.getInt());
1641 .Case<UnitAttr>([&](
auto) {
1642 attrBuilder.addAttribute(llvmKind);
1645 .Case<LLVM::ConstantRangeAttr>([&](
auto rangeAttr) {
1646 attrBuilder.addConstantRangeAttr(
1648 llvm::ConstantRange(rangeAttr.getLower(), rangeAttr.getUpper()));
1651 .Default([loc](
auto) {
1652 return emitError(loc,
"unsupported parameter attribute type");
1656 FailureOr<llvm::AttrBuilder>
1658 DictionaryAttr paramAttrs) {
1659 llvm::AttrBuilder attrBuilder(llvmModule->getContext());
1663 for (
auto namedAttr : paramAttrs) {
1664 auto it = attrNameToKindMapping.find(namedAttr.getName());
1665 if (it != attrNameToKindMapping.end()) {
1666 llvm::Attribute::AttrKind llvmKind = it->second;
1670 }
else if (namedAttr.getNameDialect()) {
1679 FailureOr<llvm::AttrBuilder>
1681 DictionaryAttr paramAttrs) {
1682 llvm::AttrBuilder attrBuilder(llvmModule->getContext());
1686 for (
auto namedAttr : paramAttrs) {
1687 auto it = attrNameToKindMapping.find(namedAttr.getName());
1688 if (it != attrNameToKindMapping.end()) {
1689 llvm::Attribute::AttrKind llvmKind = it->second;
1699 LogicalResult ModuleTranslation::convertFunctionSignatures() {
1702 for (
auto function :
getModuleBody(mlirModule).getOps<LLVMFuncOp>()) {
1703 llvm::FunctionCallee llvmFuncCst = llvmModule->getOrInsertFunction(
1705 cast<llvm::FunctionType>(
convertType(
function.getFunctionType())));
1706 llvm::Function *llvmFunc = cast<llvm::Function>(llvmFuncCst.getCallee());
1707 llvmFunc->setLinkage(convertLinkageToLLVM(
function.getLinkage()));
1708 llvmFunc->setCallingConv(convertCConvToLLVM(
function.getCConv()));
1719 if (std::optional<uint64_t> entryCount =
function.getFunctionEntryCount())
1720 llvmFunc->setEntryCount(entryCount.value());
1723 if (ArrayAttr allResultAttrs =
function.getAllResultAttrs()) {
1724 DictionaryAttr resultAttrs = cast<DictionaryAttr>(allResultAttrs[0]);
1725 FailureOr<llvm::AttrBuilder> attrBuilder =
1727 if (failed(attrBuilder))
1729 llvmFunc->addRetAttrs(*attrBuilder);
1735 FailureOr<llvm::AttrBuilder> attrBuilder =
1737 if (failed(attrBuilder))
1739 llvmArg.addAttrs(*attrBuilder);
1745 function.getLoc(),
function.getPassthrough(), llvmFunc)))
1749 llvmFunc->setVisibility(convertVisibilityToLLVM(
function.getVisibility_()));
1752 if (std::optional<mlir::SymbolRefAttr> comdat =
function.getComdat()) {
1753 auto selectorOp = cast<ComdatSelectorOp>(
1755 llvmFunc->setComdat(comdatMapping.lookup(selectorOp));
1758 if (
auto gc =
function.getGarbageCollector())
1759 llvmFunc->setGC(gc->str());
1761 if (
auto unnamedAddr =
function.getUnnamedAddr())
1762 llvmFunc->setUnnamedAddr(convertUnnamedAddrToLLVM(*unnamedAddr));
1764 if (
auto alignment =
function.getAlignment())
1765 llvmFunc->setAlignment(llvm::MaybeAlign(*alignment));
1768 debugTranslation->translate(
function, *llvmFunc);
1774 LogicalResult ModuleTranslation::convertFunctions() {
1776 for (
auto function :
getModuleBody(mlirModule).getOps<LLVMFuncOp>()) {
1779 if (
function.isExternal()) {
1780 if (failed(convertDialectAttributes(
function, {})))
1785 if (failed(convertOneFunction(
function)))
1792 LogicalResult ModuleTranslation::convertComdats() {
1793 for (
auto comdatOp :
getModuleBody(mlirModule).getOps<ComdatOp>()) {
1794 for (
auto selectorOp : comdatOp.getOps<ComdatSelectorOp>()) {
1796 if (module->getComdatSymbolTable().contains(selectorOp.getSymName()))
1798 <<
"comdat selection symbols must be unique even in different "
1800 llvm::Comdat *comdat = module->getOrInsertComdat(selectorOp.getSymName());
1801 comdat->setSelectionKind(convertComdatToLLVM(selectorOp.getComdat()));
1802 comdatMapping.try_emplace(selectorOp, comdat);
1809 llvm::Instruction *inst) {
1810 if (llvm::MDNode *node = loopAnnotationTranslation->getAccessGroups(op))
1811 inst->setMetadata(llvm::LLVMContext::MD_access_group, node);
1816 auto [scopeIt, scopeInserted] =
1817 aliasScopeMetadataMapping.try_emplace(aliasScopeAttr,
nullptr);
1819 return scopeIt->second;
1820 llvm::LLVMContext &ctx = llvmModule->getContext();
1821 auto dummy = llvm::MDNode::getTemporary(ctx, std::nullopt);
1823 auto [domainIt, insertedDomain] = aliasDomainMetadataMapping.try_emplace(
1824 aliasScopeAttr.getDomain(),
nullptr);
1825 if (insertedDomain) {
1828 operands.push_back(dummy.get());
1829 if (StringAttr description = aliasScopeAttr.getDomain().getDescription())
1833 llvm::Metadata *replacement;
1834 if (
auto stringAttr =
1835 dyn_cast<StringAttr>(aliasScopeAttr.getDomain().getId()))
1838 replacement = domainIt->second;
1839 domainIt->second->replaceOperandWith(0, replacement);
1842 assert(domainIt->second &&
"Scope's domain should already be valid");
1845 operands.push_back(dummy.get());
1846 operands.push_back(domainIt->second);
1847 if (StringAttr description = aliasScopeAttr.getDescription())
1851 llvm::Metadata *replacement;
1852 if (
auto stringAttr = dyn_cast<StringAttr>(aliasScopeAttr.getId()))
1855 replacement = scopeIt->second;
1856 scopeIt->second->replaceOperandWith(0, replacement);
1857 return scopeIt->second;
1863 nodes.reserve(aliasScopeAttrs.size());
1864 for (AliasScopeAttr aliasScopeAttr : aliasScopeAttrs)
1870 llvm::Instruction *inst) {
1871 auto populateScopeMetadata = [&](ArrayAttr aliasScopeAttrs,
unsigned kind) {
1872 if (!aliasScopeAttrs || aliasScopeAttrs.empty())
1875 llvm::to_vector(aliasScopeAttrs.getAsRange<AliasScopeAttr>()));
1876 inst->setMetadata(kind, node);
1879 populateScopeMetadata(op.getAliasScopesOrNull(),
1880 llvm::LLVMContext::MD_alias_scope);
1881 populateScopeMetadata(op.getNoAliasScopesOrNull(),
1882 llvm::LLVMContext::MD_noalias);
1885 llvm::MDNode *ModuleTranslation::getTBAANode(TBAATagAttr tbaaAttr)
const {
1886 return tbaaMetadataMapping.lookup(tbaaAttr);
1890 llvm::Instruction *inst) {
1891 ArrayAttr tagRefs = op.getTBAATagsOrNull();
1892 if (!tagRefs || tagRefs.empty())
1899 if (tagRefs.size() > 1) {
1900 op.emitWarning() <<
"TBAA access tags were not translated, because LLVM "
1901 "IR only supports a single tag per instruction";
1905 llvm::MDNode *node = getTBAANode(cast<TBAATagAttr>(tagRefs[0]));
1906 inst->setMetadata(llvm::LLVMContext::MD_tbaa, node);
1915 assert(inst &&
"expected the operation to have a mapping to an instruction");
1918 llvm::LLVMContext::MD_prof,
1922 LogicalResult ModuleTranslation::createTBAAMetadata() {
1923 llvm::LLVMContext &ctx = llvmModule->getContext();
1935 walker.
addWalk([&](TBAARootAttr root) {
1936 tbaaMetadataMapping.insert(
1940 walker.
addWalk([&](TBAATypeDescriptorAttr descriptor) {
1943 for (TBAAMemberAttr member : descriptor.getMembers()) {
1944 operands.push_back(tbaaMetadataMapping.lookup(member.getTypeDesc()));
1952 walker.
addWalk([&](TBAATagAttr tag) {
1955 operands.push_back(tbaaMetadataMapping.lookup(tag.getBaseType()));
1956 operands.push_back(tbaaMetadataMapping.lookup(tag.getAccessType()));
1960 if (tag.getConstant())
1967 mlirModule->
walk([&](AliasAnalysisOpInterface analysisOpInterface) {
1968 if (
auto attr = analysisOpInterface.getTBAATagsOrNull())
1975 LogicalResult ModuleTranslation::createIdentMetadata() {
1977 LLVMDialect::getIdentAttrName())) {
1978 StringRef ident = attr;
1979 llvm::LLVMContext &ctx = llvmModule->getContext();
1980 llvm::NamedMDNode *namedMd =
1981 llvmModule->getOrInsertNamedMetadata(LLVMDialect::getIdentAttrName());
1983 namedMd->addOperand(md);
1989 LogicalResult ModuleTranslation::createCommandlineMetadata() {
1991 LLVMDialect::getCommandlineAttrName())) {
1992 StringRef cmdLine = attr;
1993 llvm::LLVMContext &ctx = llvmModule->getContext();
1994 llvm::NamedMDNode *nmd = llvmModule->getOrInsertNamedMetadata(
1995 LLVMDialect::getCommandlineAttrName());
1998 nmd->addOperand(md);
2005 llvm::Instruction *inst) {
2006 LoopAnnotationAttr attr =
2008 .Case<LLVM::BrOp, LLVM::CondBrOp>(
2009 [](
auto branchOp) {
return branchOp.getLoopAnnotationAttr(); });
2012 llvm::MDNode *loopMD =
2013 loopAnnotationTranslation->translateLoopAnnotation(attr, op);
2014 inst->setMetadata(llvm::LLVMContext::MD_loop, loopMD);
2018 auto iface = cast<DisjointFlagInterface>(op);
2020 if (
auto disjointInst = dyn_cast<llvm::PossiblyDisjointInst>(value))
2021 disjointInst->setIsDisjoint(iface.getIsDisjoint());
2031 remapped.reserve(values.size());
2032 for (
Value v : values)
2039 ompBuilder = std::make_unique<llvm::OpenMPIRBuilder>(*llvmModule);
2040 ompBuilder->initialize();
2045 ompBuilder->setConfig(llvm::OpenMPIRBuilderConfig(
2053 return ompBuilder.get();
2057 llvm::DILocalScope *scope) {
2058 return debugTranslation->translateLoc(loc, scope);
2061 llvm::DIExpression *
2063 return debugTranslation->translateExpression(attr);
2066 llvm::DIGlobalVariableExpression *
2068 LLVM::DIGlobalVariableExpressionAttr attr) {
2069 return debugTranslation->translateGlobalVariableExpression(attr);
2073 return debugTranslation->translate(attr);
2078 return convertRoundingModeToLLVM(rounding);
2082 LLVM::FPExceptionBehavior exceptionBehavior) {
2083 return convertFPExceptionBehaviorToLLVM(exceptionBehavior);
2088 return llvmModule->getOrInsertNamedMetadata(name);
2091 void ModuleTranslation::StackFrame::anchor() {}
2093 static std::unique_ptr<llvm::Module>
2096 m->getContext()->getOrLoadDialect<LLVM::LLVMDialect>();
2097 auto llvmModule = std::make_unique<llvm::Module>(name, llvmContext);
2100 llvmModule->setNewDbgInfoFormatFlag(
false);
2101 if (
auto dataLayoutAttr =
2102 m->getDiscardableAttr(LLVM::LLVMDialect::getDataLayoutAttrName())) {
2103 llvmModule->setDataLayout(cast<StringAttr>(dataLayoutAttr).getValue());
2105 FailureOr<llvm::DataLayout> llvmDataLayout(llvm::DataLayout(
""));
2106 if (
auto iface = dyn_cast<DataLayoutOpInterface>(m)) {
2107 if (DataLayoutSpecInterface spec = iface.getDataLayoutSpec()) {
2111 }
else if (
auto mod = dyn_cast<ModuleOp>(m)) {
2112 if (DataLayoutSpecInterface spec = mod.getDataLayoutSpec()) {
2117 if (failed(llvmDataLayout))
2119 llvmModule->setDataLayout(*llvmDataLayout);
2121 if (
auto targetTripleAttr =
2122 m->getDiscardableAttr(LLVM::LLVMDialect::getTargetTripleAttrName()))
2123 llvmModule->setTargetTriple(cast<StringAttr>(targetTripleAttr).getValue());
2128 std::unique_ptr<llvm::Module>
2130 StringRef name,
bool disableVerification) {
2132 module->
emitOpError(
"can not be translated to an LLVMIR module");
2136 std::unique_ptr<llvm::Module> llvmModule =
2145 llvm::IRBuilder<llvm::TargetFolder> llvmBuilder(
2147 llvm::TargetFolder(translator.getLLVMModule()->getDataLayout()));
2153 if (failed(translator.convertOperation(*module, llvmBuilder)))
2156 if (failed(translator.convertComdats()))
2158 if (failed(translator.convertFunctionSignatures()))
2160 if (failed(translator.convertGlobalsAndAliases()))
2162 if (failed(translator.createTBAAMetadata()))
2164 if (failed(translator.createIdentMetadata()))
2166 if (failed(translator.createCommandlineMetadata()))
2171 if (!isa<LLVM::LLVMFuncOp, LLVM::AliasOp, LLVM::GlobalOp,
2172 LLVM::GlobalCtorsOp, LLVM::GlobalDtorsOp, LLVM::ComdatOp>(&o) &&
2174 failed(translator.convertOperation(o, llvmBuilder))) {
2182 if (failed(translator.convertFunctions()))
2190 if (!disableVerification &&
2191 llvm::verifyModule(*translator.llvmModule, &llvm::errs()))
2194 return std::move(translator.llvmModule);
static MLIRContext * getContext(OpFoldResult val)
static Value getPHISourceValue(Block *current, Block *pred, unsigned numArguments, unsigned index)
Get the SSA value passed to the current block from the terminator operation of its predecessor.
static llvm::MDNode * convertIntegerToMDNode(llvm::LLVMContext &context, const llvm::APInt &value)
Return a representation of value as an MDNode.
static llvm::Constant * convertDenseElementsAttr(Location loc, DenseElementsAttr denseElementsAttr, llvm::Type *llvmType, const ModuleTranslation &moduleTranslation)
Convert a dense elements attribute to an LLVM IR constant using its raw data storage if possible.
static llvm::MDNode * convertVecTypeHintToMDNode(llvm::LLVMContext &context, llvm::Type *type, bool isSigned)
Return an MDNode encoding vec_type_hint metadata.
static Block & getModuleBody(Operation *module)
A helper method to get the single Block in an operation honoring LLVM's module requirements.
static llvm::MDNode * convertIntegerArrayToMDNode(llvm::LLVMContext &context, ArrayRef< int32_t > values)
Return an MDNode with a tuple given by the values in values.
static llvm::Metadata * convertIntegerToMetadata(llvm::LLVMContext &context, const llvm::APInt &value)
Return a representation of value as metadata.
static void addRuntimePreemptionSpecifier(bool dsoLocalRequested, llvm::GlobalValue *gv)
Sets the runtime preemption specifier of gv to dso_local if dsoLocalRequested is true,...
static LogicalResult checkedAddLLVMFnAttribute(Location loc, llvm::Function *llvmFunc, StringRef key, StringRef value=StringRef())
Attempts to add an attribute identified by key, optionally with the given value to LLVM function llvm...
static void convertFunctionAttributes(LLVMFuncOp func, llvm::Function *llvmFunc)
Converts function attributes from func and attaches them to llvmFunc.
llvm::cl::opt< bool > UseNewDbgInfoFormat
static bool shouldDropGlobalInitializer(llvm::GlobalValue::LinkageTypes linkage, llvm::Constant *cst)
A helper method to decide if a constant must not be set as a global variable initializer.
static llvm::Type * getInnermostElementType(llvm::Type *type)
Returns the first non-sequential type nested in sequential types.
static void convertFunctionKernelAttributes(LLVMFuncOp func, llvm::Function *llvmFunc, ModuleTranslation &translation)
Converts function attributes from func and attaches them to llvmFunc.
static std::unique_ptr< llvm::Module > prepareLLVMModule(Operation *m, llvm::LLVMContext &llvmContext, StringRef name)
static llvm::Constant * convertDenseResourceElementsAttr(Location loc, DenseResourceElementsAttr denseResourceAttr, llvm::Type *llvmType, const ModuleTranslation &moduleTranslation)
Convert a dense resource elements attribute to an LLVM IR constant using its raw data storage if poss...
static LogicalResult convertParameterAttr(llvm::AttrBuilder &attrBuilder, llvm::Attribute::AttrKind llvmKind, NamedAttribute namedAttr, ModuleTranslation &moduleTranslation, Location loc)
static void convertFunctionMemoryAttributes(LLVMFuncOp func, llvm::Function *llvmFunc)
Converts memory effect attributes from func and attaches them to llvmFunc.
static LogicalResult forwardPassthroughAttributes(Location loc, std::optional< ArrayAttr > attributes, llvm::Function *llvmFunc)
Attaches the attributes listed in the given array attribute to llvmFunc.
static llvm::Constant * buildSequentialConstant(ArrayRef< llvm::Constant * > &constants, ArrayRef< int64_t > shape, llvm::Type *type, Location loc)
Builds a constant of a sequential LLVM type type, potentially containing other sequential types recur...
This class represents a processed binary blob of data.
ArrayRef< char > getData() const
Return the raw underlying data of this blob.
void addWalk(WalkFn< Attribute > &&fn)
Register a walk function for a given attribute or type.
WalkResult walk(T element)
Walk the given attribute/type, and recursively walk any sub elements.
Attributes are known-constant values of operations.
Block represents an ordered list of Operations.
Operation * getTerminator()
Get the terminator operation of this block.
iterator_range< pred_iterator > getPredecessors()
BlockArgListType getArguments()
iterator_range< iterator > without_terminator()
Return an iterator range over the operation within this block excluding the terminator operation at t...
The main mechanism for performing data layout queries.
std::optional< uint64_t > getTypeIndexBitwidth(Type t) const
Returns the bitwidth that should be used when performing index computations for the given pointer-lik...
uint64_t getTypePreferredAlignment(Type t) const
Returns the preferred of the given type in the current scope.
uint64_t getTypeABIAlignment(Type t) const
Returns the required alignment of the given type in the current scope.
llvm::TypeSize getTypeSizeInBits(Type t) const
Returns the size in bits of the given type in the current scope.
An attribute that represents a reference to a dense vector or tensor object.
std::enable_if_t<!std::is_base_of< Attribute, T >::value||std::is_same< Attribute, T >::value, T > getSplatValue() const
Return the splat value for this attribute.
int64_t getNumElements() const
Returns the number of elements held by this attribute.
bool isSplat() const
Returns true if this attribute corresponds to a splat, i.e.
ArrayRef< char > getRawData() const
Return the raw storage data held by this attribute.
ShapedType getType() const
Return the type of this ElementsAttr, guaranteed to be a vector or tensor with static shape.
const InterfaceType * getInterfaceFor(Object *obj) const
Get the interface for a given object, or null if one is not registered.
Base class for dialect interfaces providing translation to LLVM IR.
virtual LogicalResult convertOperation(Operation *op, llvm::IRBuilderBase &builder, LLVM::ModuleTranslation &moduleTranslation) const
Hook for derived dialect interface to provide translation of the operations to LLVM IR.
virtual LogicalResult convertParameterAttr(LLVM::LLVMFuncOp function, int argIdx, NamedAttribute attribute, LLVM::ModuleTranslation &moduleTranslation) const
Acts on the given function operation using the interface implemented by the dialect of one of the fun...
virtual LogicalResult amendOperation(Operation *op, ArrayRef< llvm::Instruction * > instructions, NamedAttribute attribute, LLVM::ModuleTranslation &moduleTranslation) const
Acts on the given operation using the interface implemented by the dialect of one of the operation's ...
This class represents the base attribute for all debug info attributes.
Implementation class for module translation.
llvm::fp::ExceptionBehavior translateFPExceptionBehavior(LLVM::FPExceptionBehavior exceptionBehavior)
Translates the given LLVM FP exception behavior metadata.
llvm::Value * lookupValue(Value value) const
Finds an LLVM IR value corresponding to the given MLIR value.
llvm::DIGlobalVariableExpression * translateGlobalVariableExpression(LLVM::DIGlobalVariableExpressionAttr attr)
Translates the given LLVM global variable expression metadata.
llvm::NamedMDNode * getOrInsertNamedModuleMetadata(StringRef name)
Gets the named metadata in the LLVM IR module being constructed, creating it if it does not exist.
llvm::Instruction * lookupBranch(Operation *op) const
Finds an LLVM IR instruction that corresponds to the given MLIR operation with successors.
SmallVector< llvm::Value * > lookupValues(ValueRange values)
Looks up remapped a list of remapped values.
void mapFunction(StringRef name, llvm::Function *func)
Stores the mapping between a function name and its LLVM IR representation.
llvm::DILocation * translateLoc(Location loc, llvm::DILocalScope *scope)
Translates the given location.
llvm::BasicBlock * lookupBlock(Block *block) const
Finds an LLVM IR basic block that corresponds to the given MLIR block.
void setBranchWeightsMetadata(BranchWeightOpInterface op)
Sets LLVM profiling metadata for operations that have branch weights.
llvm::Type * convertType(Type type)
Converts the type from MLIR LLVM dialect to LLVM.
llvm::RoundingMode translateRoundingMode(LLVM::RoundingMode rounding)
Translates the given LLVM rounding mode metadata.
void setTBAAMetadata(AliasAnalysisOpInterface op, llvm::Instruction *inst)
Sets LLVM TBAA metadata for memory operations that have TBAA attributes.
llvm::DIExpression * translateExpression(LLVM::DIExpressionAttr attr)
Translates the given LLVM DWARF expression metadata.
llvm::OpenMPIRBuilder * getOpenMPBuilder()
Returns the OpenMP IR builder associated with the LLVM IR module being constructed.
llvm::CallInst * lookupCall(Operation *op) const
Finds an LLVM call instruction that corresponds to the given MLIR call operation.
llvm::Metadata * translateDebugInfo(LLVM::DINodeAttr attr)
Translates the given LLVM debug info metadata.
void setDisjointFlag(Operation *op, llvm::Value *value)
Sets the disjoint flag attribute for the exported instruction value given the original operation op.
llvm::LLVMContext & getLLVMContext() const
Returns the LLVM context in which the IR is being constructed.
llvm::GlobalValue * lookupGlobal(Operation *op)
Finds an LLVM IR global value that corresponds to the given MLIR operation defining a global value.
llvm::Module * getLLVMModule()
Returns the LLVM module in which the IR is being constructed.
llvm::Function * lookupFunction(StringRef name) const
Finds an LLVM IR function by its name.
llvm::MDNode * getOrCreateAliasScopes(ArrayRef< AliasScopeAttr > aliasScopeAttrs)
Returns the LLVM metadata corresponding to an array of mlir LLVM dialect alias scope attributes.
void mapBlock(Block *mlir, llvm::BasicBlock *llvm)
Stores the mapping between an MLIR block and LLVM IR basic block.
llvm::MDNode * getOrCreateAliasScope(AliasScopeAttr aliasScopeAttr)
Returns the LLVM metadata corresponding to a mlir LLVM dialect alias scope attribute.
void forgetMapping(Region ®ion)
Removes the mapping for blocks contained in the region and values defined in these blocks.
void setAliasScopeMetadata(AliasAnalysisOpInterface op, llvm::Instruction *inst)
void setAccessGroupsMetadata(AccessGroupOpInterface op, llvm::Instruction *inst)
void mapValue(Value mlir, llvm::Value *llvm)
Stores the mapping between an MLIR value and its LLVM IR counterpart.
FailureOr< llvm::AttrBuilder > convertParameterAttrs(CallOpInterface callOp, DictionaryAttr paramAttrs)
Translates parameter attributes of a call and adds them to the returned AttrBuilder.
void setLoopMetadata(Operation *op, llvm::Instruction *inst)
Sets LLVM loop metadata for branch operations that have a loop annotation attribute.
llvm::Type * translateType(Type type)
Translates the given MLIR LLVM dialect type to LLVM IR.
A helper class that converts LoopAnnotationAttrs and AccessGroupAttrs into corresponding llvm::MDNode...
This class defines the main interface for locations in MLIR and acts as a non-nullable wrapper around...
NamedAttribute represents a combination of a name and an Attribute value.
Attribute getValue() const
Return the value of the attribute.
This class provides the API for ops that are known to be terminators.
Operation is the basic unit of execution within MLIR.
Value getOperand(unsigned idx)
AttrClass getAttrOfType(StringAttr name)
Attribute getAttr(StringAttr name)
Return the specified attribute if present, null otherwise.
Block * getSuccessor(unsigned index)
unsigned getNumSuccessors()
OpResult getResult(unsigned idx)
Get the 'idx'th result of this operation.
std::enable_if_t< llvm::function_traits< std::decay_t< FnT > >::num_args==1, RetT > walk(FnT &&callback)
Walk the operation by calling the callback for each nested operation (including this one),...
Location getLoc()
The source location the operation was defined or derived from.
InFlightDiagnostic emitError(const Twine &message={})
Emit an error about fatal conditions with this operation, reporting up to any diagnostic handlers tha...
Region & getRegion(unsigned index)
Returns the region held by this operation at position 'index'.
OperationName getName()
The name of an operation is the key identifier for it.
dialect_attr_range getDialectAttrs()
Return a range corresponding to the dialect attributes for this operation.
operand_range getOperands()
Returns an iterator on the underlying Value's.
InFlightDiagnostic emitOpError(const Twine &message={})
Emit an error with the op name prefixed, like "'dim' op " which is convenient for verifiers.
This class contains a list of basic blocks and a link to the parent operation it is attached to.
This class models how operands are forwarded to block arguments in control flow.
bool empty() const
Returns true if there are no successor operands.
static Operation * lookupNearestSymbolFrom(Operation *from, StringAttr symbol)
Returns the operation registered with the given symbol name within the closest parent operation of,...
Instances of the Type class are uniqued, have an immutable identifier and an optional mutable compone...
This class provides an abstraction over the different types of ranges over Values.
This class represents an instance of an SSA value in the MLIR system, representing a computable value...
use_iterator use_end() const
Type getType() const
Return the type of this value.
use_iterator use_begin() const
ArrayRef< T > asArrayRef() const
The OpAsmOpInterface, see OpAsmInterface.td for more details.
void connectPHINodes(Region ®ion, const ModuleTranslation &state)
For all blocks in the region that were converted to LLVM IR using the given ModuleTranslation,...
llvm::CallInst * createIntrinsicCall(llvm::IRBuilderBase &builder, llvm::Intrinsic::ID intrinsic, ArrayRef< llvm::Value * > args={}, ArrayRef< llvm::Type * > tys={})
Creates a call to an LLVM IR intrinsic function with the given arguments.
static llvm::DenseMap< llvm::StringRef, llvm::Attribute::AttrKind > getAttrNameToKindMapping()
Returns a dense map from LLVM attribute name to their kind in LLVM IR dialect.
llvm::Constant * getLLVMConstant(llvm::Type *llvmType, Attribute attr, Location loc, const ModuleTranslation &moduleTranslation)
Create an LLVM IR constant of llvmType from the MLIR attribute attr.
bool satisfiesLLVMModule(Operation *op)
LLVM requires some operations to be inside of a Module operation.
void legalizeDIExpressionsRecursively(Operation *op)
Register all known legalization patterns declared here and apply them to all ops in op.
bool isCompatibleType(Type type)
Returns true if the given type is compatible with the LLVM dialect.
void ensureDistinctSuccessors(Operation *op)
Make argument-taking successors of each block distinct.
constexpr void enumerate(std::tuple< Tys... > &tuple, CallbackT &&callback)
DictionaryAttr getArgAttrDict(FunctionOpInterface op, unsigned index)
Returns the dictionary attribute corresponding to the argument at 'index'.
Include the generated interface declarations.
std::unique_ptr< llvm::Module > translateModuleToLLVMIR(Operation *module, llvm::LLVMContext &llvmContext, llvm::StringRef name="LLVMDialectModule", bool disableVerification=false)
Translates a given LLVM dialect module into an LLVM IR module living in the given context.
SetVector< Block * > getBlocksSortedByDominance(Region ®ion)
Gets a list of blocks that is sorted according to dominance.
DataLayoutSpecInterface translateDataLayout(const llvm::DataLayout &dataLayout, MLIRContext *context)
Translate the given LLVM data layout into an MLIR equivalent using the DLTI dialect.
Type getType(OpFoldResult ofr)
Returns the int type of the integer in ofr.
InFlightDiagnostic emitError(Location loc)
Utility method to emit an error message using this location.
auto get(MLIRContext *context, Ts &&...params)
Helper method that injects context only if needed, this helps unify some of the attribute constructio...