25 #include "llvm/ADT/ArrayRef.h"
26 #include "llvm/ADT/SetVector.h"
27 #include "llvm/ADT/TypeSwitch.h"
28 #include "llvm/Frontend/OpenMP/OMPConstants.h"
29 #include "llvm/Frontend/OpenMP/OMPIRBuilder.h"
30 #include "llvm/IR/DebugInfoMetadata.h"
31 #include "llvm/IR/IRBuilder.h"
32 #include "llvm/IR/ReplaceConstant.h"
33 #include "llvm/Support/FileSystem.h"
34 #include "llvm/TargetParser/Triple.h"
35 #include "llvm/Transforms/Utils/ModuleUtils.h"
47 static llvm::omp::ScheduleKind
48 convertToScheduleKind(std::optional<omp::ClauseScheduleKind> schedKind) {
49 if (!schedKind.has_value())
50 return llvm::omp::OMP_SCHEDULE_Default;
51 switch (schedKind.value()) {
52 case omp::ClauseScheduleKind::Static:
53 return llvm::omp::OMP_SCHEDULE_Static;
54 case omp::ClauseScheduleKind::Dynamic:
55 return llvm::omp::OMP_SCHEDULE_Dynamic;
56 case omp::ClauseScheduleKind::Guided:
57 return llvm::omp::OMP_SCHEDULE_Guided;
58 case omp::ClauseScheduleKind::Auto:
59 return llvm::omp::OMP_SCHEDULE_Auto;
61 return llvm::omp::OMP_SCHEDULE_Runtime;
63 llvm_unreachable(
"unhandled schedule clause argument");
68 class OpenMPAllocaStackFrame
73 explicit OpenMPAllocaStackFrame(llvm::OpenMPIRBuilder::InsertPointTy allocaIP)
74 : allocaInsertPoint(allocaIP) {}
75 llvm::OpenMPIRBuilder::InsertPointTy allocaInsertPoint;
80 class OpenMPVarMappingStackFrame
82 OpenMPVarMappingStackFrame> {
86 explicit OpenMPVarMappingStackFrame(
96 static llvm::OpenMPIRBuilder::InsertPointTy
102 llvm::OpenMPIRBuilder::InsertPointTy allocaInsertPoint;
104 [&](
const OpenMPAllocaStackFrame &frame) {
105 allocaInsertPoint = frame.allocaInsertPoint;
109 return allocaInsertPoint;
118 if (builder.GetInsertBlock() ==
119 &builder.GetInsertBlock()->getParent()->getEntryBlock()) {
120 assert(builder.GetInsertPoint() == builder.GetInsertBlock()->end() &&
121 "Assuming end of basic block");
122 llvm::BasicBlock *entryBB = llvm::BasicBlock::Create(
123 builder.getContext(),
"entry", builder.GetInsertBlock()->getParent(),
124 builder.GetInsertBlock()->getNextNode());
125 builder.CreateBr(entryBB);
126 builder.SetInsertPoint(entryBB);
129 llvm::BasicBlock &funcEntryBlock =
130 builder.GetInsertBlock()->getParent()->getEntryBlock();
131 return llvm::OpenMPIRBuilder::InsertPointTy(
132 &funcEntryBlock, funcEntryBlock.getFirstInsertionPt());
141 Region ®ion, StringRef blockName, llvm::IRBuilderBase &builder,
144 llvm::BasicBlock *continuationBlock =
145 splitBB(builder,
true,
"omp.region.cont");
146 llvm::BasicBlock *sourceBlock = builder.GetInsertBlock();
148 llvm::LLVMContext &llvmContext = builder.getContext();
149 for (
Block &bb : region) {
150 llvm::BasicBlock *llvmBB = llvm::BasicBlock::Create(
151 llvmContext, blockName, builder.GetInsertBlock()->getParent(),
152 builder.GetInsertBlock()->getNextNode());
153 moduleTranslation.
mapBlock(&bb, llvmBB);
156 llvm::Instruction *sourceTerminator = sourceBlock->getTerminator();
162 bool operandsProcessed =
false;
163 unsigned numYields = 0;
165 if (omp::YieldOp yield = dyn_cast<omp::YieldOp>(bb.getTerminator())) {
166 if (!operandsProcessed) {
167 for (
unsigned i = 0, e = yield->getNumOperands(); i < e; ++i) {
168 continuationBlockPHITypes.push_back(
169 moduleTranslation.
convertType(yield->getOperand(i).getType()));
171 operandsProcessed =
true;
173 assert(continuationBlockPHITypes.size() == yield->getNumOperands() &&
174 "mismatching number of values yielded from the region");
175 for (
unsigned i = 0, e = yield->getNumOperands(); i < e; ++i) {
176 llvm::Type *operandType =
177 moduleTranslation.
convertType(yield->getOperand(i).getType());
179 assert(continuationBlockPHITypes[i] == operandType &&
180 "values of mismatching types yielded from the region");
189 if (!continuationBlockPHITypes.empty())
191 continuationBlockPHIs &&
192 "expected continuation block PHIs if converted regions yield values");
193 if (continuationBlockPHIs) {
194 llvm::IRBuilderBase::InsertPointGuard guard(builder);
195 continuationBlockPHIs->reserve(continuationBlockPHITypes.size());
196 builder.SetInsertPoint(continuationBlock, continuationBlock->begin());
197 for (llvm::Type *ty : continuationBlockPHITypes)
198 continuationBlockPHIs->push_back(builder.CreatePHI(ty, numYields));
204 for (
Block *bb : blocks) {
205 llvm::BasicBlock *llvmBB = moduleTranslation.
lookupBlock(bb);
208 if (bb->isEntryBlock()) {
209 assert(sourceTerminator->getNumSuccessors() == 1 &&
210 "provided entry block has multiple successors");
211 assert(sourceTerminator->getSuccessor(0) == continuationBlock &&
212 "ContinuationBlock is not the successor of the entry block");
213 sourceTerminator->setSuccessor(0, llvmBB);
216 llvm::IRBuilderBase::InsertPointGuard guard(builder);
218 moduleTranslation.
convertBlock(*bb, bb->isEntryBlock(), builder))) {
219 bodyGenStatus = failure();
220 return continuationBlock;
230 Operation *terminator = bb->getTerminator();
231 if (isa<omp::TerminatorOp, omp::YieldOp>(terminator)) {
232 builder.CreateBr(continuationBlock);
234 for (
unsigned i = 0, e = terminator->
getNumOperands(); i < e; ++i)
235 (*continuationBlockPHIs)[i]->addIncoming(
249 return continuationBlock;
255 case omp::ClauseProcBindKind::Close:
256 return llvm::omp::ProcBindKind::OMP_PROC_BIND_close;
257 case omp::ClauseProcBindKind::Master:
258 return llvm::omp::ProcBindKind::OMP_PROC_BIND_master;
259 case omp::ClauseProcBindKind::Primary:
260 return llvm::omp::ProcBindKind::OMP_PROC_BIND_primary;
261 case omp::ClauseProcBindKind::Spread:
262 return llvm::omp::ProcBindKind::OMP_PROC_BIND_spread;
264 llvm_unreachable(
"Unknown ClauseProcBindKind kind");
271 auto maskedOp = cast<omp::MaskedOp>(opInst);
272 using InsertPointTy = llvm::OpenMPIRBuilder::InsertPointTy;
275 LogicalResult bodyGenStatus = success();
277 auto bodyGenCB = [&](InsertPointTy allocaIP, InsertPointTy codeGenIP) {
279 auto ®ion = maskedOp.getRegion();
280 builder.restoreIP(codeGenIP);
287 auto finiCB = [&](InsertPointTy codeGenIP) {};
289 llvm::Value *filterVal =
nullptr;
290 if (
auto filterVar = maskedOp.getFilteredThreadId()) {
291 filterVal = moduleTranslation.
lookupValue(filterVar);
293 llvm::LLVMContext &llvmContext = builder.getContext();
297 assert(filterVal !=
nullptr);
298 llvm::OpenMPIRBuilder::LocationDescription ompLoc(builder);
300 ompLoc, bodyGenCB, finiCB, filterVal));
308 using InsertPointTy = llvm::OpenMPIRBuilder::InsertPointTy;
311 LogicalResult bodyGenStatus = success();
313 auto bodyGenCB = [&](InsertPointTy allocaIP, InsertPointTy codeGenIP) {
315 auto ®ion = cast<omp::MasterOp>(opInst).getRegion();
316 builder.restoreIP(codeGenIP);
323 auto finiCB = [&](InsertPointTy codeGenIP) {};
325 llvm::OpenMPIRBuilder::LocationDescription ompLoc(builder);
327 ompLoc, bodyGenCB, finiCB));
335 using InsertPointTy = llvm::OpenMPIRBuilder::InsertPointTy;
336 auto criticalOp = cast<omp::CriticalOp>(opInst);
339 LogicalResult bodyGenStatus = success();
341 auto bodyGenCB = [&](InsertPointTy allocaIP, InsertPointTy codeGenIP) {
343 auto ®ion = cast<omp::CriticalOp>(opInst).getRegion();
344 builder.restoreIP(codeGenIP);
346 moduleTranslation, bodyGenStatus);
351 auto finiCB = [&](InsertPointTy codeGenIP) {};
353 llvm::OpenMPIRBuilder::LocationDescription ompLoc(builder);
354 llvm::LLVMContext &llvmContext = moduleTranslation.
getLLVMContext();
355 llvm::Constant *hint =
nullptr;
358 if (criticalOp.getNameAttr()) {
361 auto symbolRef = cast<SymbolRefAttr>(criticalOp.getNameAttr());
362 auto criticalDeclareOp =
363 SymbolTable::lookupNearestSymbolFrom<omp::CriticalDeclareOp>(criticalOp,
367 static_cast<int>(criticalDeclareOp.getHint()));
370 ompLoc, bodyGenCB, finiCB, criticalOp.getName().value_or(
""), hint));
377 SymbolRefAttr symbolName) {
378 omp::PrivateClauseOp privatizer =
379 SymbolTable::lookupNearestSymbolFrom<omp::PrivateClauseOp>(from,
381 assert(privatizer &&
"privatizer not found in the symbol table");
390 std::optional<ArrayAttr> attr = op.getPrivateSyms();
394 privatizations.reserve(privatizations.size() + attr->size());
395 for (
auto symbolRef : attr->getAsRange<SymbolRefAttr>()) {
401 template <
typename T>
405 std::optional<ArrayAttr> attr = op.getReductionSyms();
409 reductions.reserve(reductions.size() + op.getNumReductionVars());
410 for (
auto symbolRef : attr->getAsRange<SymbolRefAttr>()) {
411 reductions.push_back(
412 SymbolTable::lookupNearestSymbolFrom<omp::DeclareReductionOp>(
423 Region ®ion, StringRef blockName, llvm::IRBuilderBase &builder,
431 if (llvm::hasSingleElement(region)) {
432 llvm::Instruction *potentialTerminator =
433 builder.GetInsertBlock()->empty() ? nullptr
434 : &builder.GetInsertBlock()->back();
436 if (potentialTerminator && potentialTerminator->isTerminator())
437 potentialTerminator->removeFromParent();
438 moduleTranslation.
mapBlock(®ion.
front(), builder.GetInsertBlock());
441 region.
front(),
true, builder)))
445 if (continuationBlockArgs)
447 *continuationBlockArgs,
454 if (potentialTerminator && potentialTerminator->isTerminator()) {
455 llvm::BasicBlock *block = builder.GetInsertBlock();
456 if (block->empty()) {
462 potentialTerminator->insertInto(block, block->begin());
464 potentialTerminator->insertAfter(&block->back());
471 LogicalResult bodyGenStatus = success();
474 region, blockName, builder, moduleTranslation, bodyGenStatus, &phis);
475 if (failed(bodyGenStatus))
477 if (continuationBlockArgs)
478 llvm::append_range(*continuationBlockArgs, phis);
479 builder.SetInsertPoint(continuationBlock,
480 continuationBlock->getFirstInsertionPt());
487 using OwningReductionGen = std::function<llvm::OpenMPIRBuilder::InsertPointTy(
488 llvm::OpenMPIRBuilder::InsertPointTy, llvm::Value *, llvm::Value *,
490 using OwningAtomicReductionGen =
491 std::function<llvm::OpenMPIRBuilder::InsertPointTy(
492 llvm::OpenMPIRBuilder::InsertPointTy, llvm::Type *, llvm::Value *,
499 static OwningReductionGen
505 OwningReductionGen gen =
506 [&, decl](llvm::OpenMPIRBuilder::InsertPointTy insertPoint,
507 llvm::Value *lhs, llvm::Value *rhs,
508 llvm::Value *&result)
mutable {
509 moduleTranslation.
mapValue(decl.getReductionLhsArg(), lhs);
510 moduleTranslation.
mapValue(decl.getReductionRhsArg(), rhs);
511 builder.restoreIP(insertPoint);
514 "omp.reduction.nonatomic.body",
515 builder, moduleTranslation, &phis)))
516 return llvm::OpenMPIRBuilder::InsertPointTy();
517 assert(phis.size() == 1);
519 return builder.saveIP();
528 static OwningAtomicReductionGen
530 llvm::IRBuilderBase &builder,
532 if (decl.getAtomicReductionRegion().empty())
533 return OwningAtomicReductionGen();
538 OwningAtomicReductionGen atomicGen =
539 [&, decl](llvm::OpenMPIRBuilder::InsertPointTy insertPoint, llvm::Type *,
540 llvm::Value *lhs, llvm::Value *rhs)
mutable {
541 moduleTranslation.
mapValue(decl.getAtomicReductionLhsArg(), lhs);
542 moduleTranslation.
mapValue(decl.getAtomicReductionRhsArg(), rhs);
543 builder.restoreIP(insertPoint);
546 "omp.reduction.atomic.body", builder,
547 moduleTranslation, &phis)))
548 return llvm::OpenMPIRBuilder::InsertPointTy();
549 assert(phis.empty());
550 return builder.saveIP();
559 auto orderedOp = cast<omp::OrderedOp>(opInst);
561 omp::ClauseDepend dependType = *orderedOp.getDoacrossDependType();
562 bool isDependSource = dependType == omp::ClauseDepend::dependsource;
563 unsigned numLoops = *orderedOp.getDoacrossNumLoops();
565 moduleTranslation.
lookupValues(orderedOp.getDoacrossDependVars());
567 size_t indexVecValues = 0;
568 while (indexVecValues < vecValues.size()) {
570 storeValues.reserve(numLoops);
571 for (
unsigned i = 0; i < numLoops; i++) {
572 storeValues.push_back(vecValues[indexVecValues]);
575 llvm::OpenMPIRBuilder::InsertPointTy allocaIP =
577 llvm::OpenMPIRBuilder::LocationDescription ompLoc(builder);
578 builder.restoreIP(moduleTranslation.
getOpenMPBuilder()->createOrderedDepend(
579 ompLoc, allocaIP, numLoops, storeValues,
".cnt.addr", isDependSource));
589 using InsertPointTy = llvm::OpenMPIRBuilder::InsertPointTy;
590 auto orderedRegionOp = cast<omp::OrderedRegionOp>(opInst);
593 if (orderedRegionOp.getParLevelSimd())
598 LogicalResult bodyGenStatus = success();
600 auto bodyGenCB = [&](InsertPointTy allocaIP, InsertPointTy codeGenIP) {
602 auto ®ion = cast<omp::OrderedRegionOp>(opInst).getRegion();
603 builder.restoreIP(codeGenIP);
605 moduleTranslation, bodyGenStatus);
610 auto finiCB = [&](InsertPointTy codeGenIP) {};
612 llvm::OpenMPIRBuilder::LocationDescription ompLoc(builder);
615 ompLoc, bodyGenCB, finiCB, !orderedRegionOp.getParLevelSimd()));
616 return bodyGenStatus;
621 struct DeferredStore {
622 DeferredStore(llvm::Value *value, llvm::Value *address)
623 : value(value), address(address) {}
626 llvm::Value *address;
633 template <
typename T>
636 llvm::IRBuilderBase &builder,
638 const llvm::OpenMPIRBuilder::InsertPointTy &allocaIP,
644 llvm::IRBuilderBase::InsertPointGuard guard(builder);
645 builder.SetInsertPoint(allocaIP.getBlock()->getTerminator());
648 deferredStores.reserve(loop.getNumReductionVars());
650 for (std::size_t i = 0; i < loop.getNumReductionVars(); ++i) {
651 Region &allocRegion = reductionDecls[i].getAllocRegion();
653 if (allocRegion.
empty())
658 builder, moduleTranslation, &phis)))
660 assert(phis.size() == 1 &&
"expected one allocation to be yielded");
662 builder.SetInsertPoint(allocaIP.getBlock()->getTerminator());
666 llvm::Value *var = builder.CreateAlloca(
667 moduleTranslation.
convertType(reductionDecls[i].getType()));
668 deferredStores.emplace_back(phis[0], var);
670 privateReductionVariables[i] = var;
671 moduleTranslation.
mapValue(reductionArgs[i], phis[0]);
672 reductionVariableMap.try_emplace(loop.getReductionVars()[i], phis[0]);
674 assert(allocRegion.
empty() &&
675 "allocaction is implicit for by-val reduction");
676 llvm::Value *var = builder.CreateAlloca(
677 moduleTranslation.
convertType(reductionDecls[i].getType()));
678 moduleTranslation.
mapValue(reductionArgs[i], var);
679 privateReductionVariables[i] = var;
680 reductionVariableMap.try_emplace(loop.getReductionVars()[i], var);
688 template <
typename T>
695 mlir::omp::DeclareReductionOp &reduction = reductionDecls[i];
696 Region &initializerRegion = reduction.getInitializerRegion();
699 mlir::Value mlirSource = loop.getReductionVars()[i];
700 llvm::Value *llvmSource = moduleTranslation.
lookupValue(mlirSource);
701 assert(llvmSource &&
"lookup reduction var");
702 moduleTranslation.
mapValue(reduction.getInitializerMoldArg(), llvmSource);
705 llvm::Value *allocation =
706 reductionVariableMap.lookup(loop.getReductionVars()[i]);
707 moduleTranslation.
mapValue(reduction.getInitializerAllocArg(), allocation);
712 template <
typename T>
714 T loop, llvm::IRBuilderBase &builder,
721 unsigned numReductions = loop.getNumReductionVars();
723 for (
unsigned i = 0; i < numReductions; ++i) {
724 owningReductionGens.push_back(
726 owningAtomicReductionGens.push_back(
731 reductionInfos.reserve(numReductions);
732 for (
unsigned i = 0; i < numReductions; ++i) {
733 llvm::OpenMPIRBuilder::ReductionGenAtomicCBTy atomicGen =
nullptr;
734 if (owningAtomicReductionGens[i])
735 atomicGen = owningAtomicReductionGens[i];
736 llvm::Value *variable =
737 moduleTranslation.
lookupValue(loop.getReductionVars()[i]);
738 reductionInfos.push_back(
740 privateReductionVariables[i],
741 llvm::OpenMPIRBuilder::EvalKind::Scalar,
742 owningReductionGens[i],
743 nullptr, atomicGen});
752 llvm::IRBuilderBase &builder, StringRef regionName,
753 bool shouldLoadCleanupRegionArg =
true) {
755 if (cleanupRegion->empty())
761 llvm::Instruction *potentialTerminator =
762 builder.GetInsertBlock()->empty() ? nullptr
763 : &builder.GetInsertBlock()->back();
764 if (potentialTerminator && potentialTerminator->isTerminator())
765 builder.SetInsertPoint(potentialTerminator);
766 llvm::Value *privateVarValue =
767 shouldLoadCleanupRegionArg
768 ? builder.CreateLoad(
771 : privateVariables[i];
789 OP op, llvm::IRBuilderBase &builder,
791 llvm::OpenMPIRBuilder::InsertPointTy &allocaIP,
795 if (op.getNumReductionVars() == 0)
806 owningReductionGens, owningAtomicReductionGens,
807 privateReductionVariables, reductionInfos);
812 llvm::UnreachableInst *tempTerminator = builder.CreateUnreachable();
813 builder.SetInsertPoint(tempTerminator);
814 llvm::OpenMPIRBuilder::InsertPointTy contInsertPoint =
815 ompBuilder->createReductions(builder.saveIP(), allocaIP, reductionInfos,
816 isByRef, op.getNowait());
817 if (!contInsertPoint.getBlock())
818 return op->
emitOpError() <<
"failed to convert reductions";
819 auto nextInsertionPoint =
820 ompBuilder->createBarrier(contInsertPoint, llvm::omp::OMPD_for);
821 tempTerminator->eraseFromParent();
822 builder.restoreIP(nextInsertionPoint);
826 llvm::transform(reductionDecls, std::back_inserter(reductionRegions),
827 [](omp::DeclareReductionOp reductionDecl) {
828 return &reductionDecl.getCleanupRegion();
831 moduleTranslation, builder,
832 "omp.reduction.cleanup");
843 template <
typename OP>
847 llvm::OpenMPIRBuilder::InsertPointTy &allocaIP,
852 if (op.getNumReductionVars() == 0)
858 allocaIP, reductionDecls,
859 privateReductionVariables, reductionVariableMap,
860 deferredStores, isByRef)))
865 for (
auto [data, addr] : deferredStores)
866 builder.CreateStore(data, addr);
871 for (
unsigned i = 0; i < op.getNumReductionVars(); ++i) {
876 reductionVariableMap, i);
879 "omp.reduction.neutral", builder,
880 moduleTranslation, &phis)))
882 assert(phis.size() == 1 &&
"expected one value to be yielded from the "
883 "reduction neutral element declaration region");
885 if (!reductionDecls[i].getAllocRegion().empty())
894 llvm::Value *var = builder.CreateAlloca(
895 moduleTranslation.
convertType(reductionDecls[i].getType()));
898 builder.CreateStore(phis[0], var);
900 privateReductionVariables[i] = var;
901 moduleTranslation.
mapValue(reductionArgs[i], phis[0]);
902 reductionVariableMap.try_emplace(op.getReductionVars()[i], phis[0]);
905 builder.CreateStore(phis[0], privateReductionVariables[i]);
912 moduleTranslation.
forgetMapping(reductionDecls[i].getInitializerRegion());
921 using InsertPointTy = llvm::OpenMPIRBuilder::InsertPointTy;
922 using StorableBodyGenCallbackTy =
923 llvm::OpenMPIRBuilder::StorableBodyGenCallbackTy;
925 auto sectionsOp = cast<omp::SectionsOp>(opInst);
929 if (!sectionsOp.getAllocateVars().empty() ||
930 !sectionsOp.getAllocatorVars().empty() ||
931 !sectionsOp.getPrivateVars().empty() || sectionsOp.getPrivateSyms())
932 return opInst.
emitError(
"unhandled clauses for translation to LLVM IR");
935 assert(isByRef.size() == sectionsOp.getNumReductionVars());
939 llvm::OpenMPIRBuilder::InsertPointTy allocaIP =
943 sectionsOp.getNumReductionVars());
947 cast<omp::BlockArgOpenMPOpInterface>(opInst).getReductionBlockArgs();
950 sectionsOp, reductionArgs, builder, moduleTranslation, allocaIP,
951 reductionDecls, privateReductionVariables, reductionVariableMap,
959 moduleTranslation, reductionVariableMap);
961 LogicalResult bodyGenStatus = success();
965 auto sectionOp = dyn_cast<omp::SectionOp>(op);
969 Region ®ion = sectionOp.getRegion();
970 auto sectionCB = [§ionsOp, ®ion, &builder, &moduleTranslation,
971 &bodyGenStatus](InsertPointTy allocaIP,
972 InsertPointTy codeGenIP) {
973 builder.restoreIP(codeGenIP);
980 sectionsOp.getRegion().getNumArguments());
981 for (
auto [sectionsArg, sectionArg] : llvm::zip_equal(
982 sectionsOp.getRegion().getArguments(), region.
getArguments())) {
983 llvm::Value *llvmVal = moduleTranslation.lookupValue(sectionsArg);
985 moduleTranslation.mapValue(sectionArg, llvmVal);
989 moduleTranslation, bodyGenStatus);
991 sectionCBs.push_back(sectionCB);
997 if (sectionCBs.empty())
1000 assert(isa<omp::SectionOp>(*sectionsOp.getRegion().op_begin()));
1005 auto privCB = [&](InsertPointTy, InsertPointTy codeGenIP, llvm::Value &,
1007 llvm::Value *&replacementValue) -> InsertPointTy {
1008 replacementValue = &vPtr;
1014 auto finiCB = [&](InsertPointTy codeGenIP) {};
1017 llvm::OpenMPIRBuilder::LocationDescription ompLoc(builder);
1019 ompLoc, allocaIP, sectionCBs, privCB, finiCB,
false,
1020 sectionsOp.getNowait()));
1022 if (failed(bodyGenStatus))
1023 return bodyGenStatus;
1027 allocaIP, reductionDecls,
1028 privateReductionVariables, isByRef);
1032 static LogicalResult
1035 using InsertPointTy = llvm::OpenMPIRBuilder::InsertPointTy;
1036 llvm::OpenMPIRBuilder::LocationDescription ompLoc(builder);
1037 LogicalResult bodyGenStatus = success();
1038 if (!singleOp.getPrivateVars().empty() || singleOp.getPrivateSyms())
1039 return singleOp.emitError(
"unhandled clauses for translation to LLVM IR");
1041 auto bodyCB = [&](InsertPointTy allocaIP, InsertPointTy codegenIP) {
1042 builder.restoreIP(codegenIP);
1044 moduleTranslation, bodyGenStatus);
1046 auto finiCB = [&](InsertPointTy codeGenIP) {};
1050 std::optional<ArrayAttr> cpFuncs = singleOp.getCopyprivateSyms();
1053 for (
size_t i = 0, e = cpVars.size(); i < e; ++i) {
1054 llvmCPVars.push_back(moduleTranslation.
lookupValue(cpVars[i]));
1055 auto llvmFuncOp = SymbolTable::lookupNearestSymbolFrom<LLVM::LLVMFuncOp>(
1056 singleOp, cast<SymbolRefAttr>((*cpFuncs)[i]));
1057 llvmCPFuncs.push_back(
1062 ompLoc, bodyCB, finiCB, singleOp.getNowait(), llvmCPVars, llvmCPFuncs));
1063 return bodyGenStatus;
1067 static LogicalResult
1070 using InsertPointTy = llvm::OpenMPIRBuilder::InsertPointTy;
1071 LogicalResult bodyGenStatus = success();
1072 if (!op.getAllocatorVars().empty() || op.getReductionSyms() ||
1073 !op.getPrivateVars().empty() || op.getPrivateSyms())
1074 return op.
emitError(
"unhandled clauses for translation to LLVM IR");
1076 auto bodyCB = [&](InsertPointTy allocaIP, InsertPointTy codegenIP) {
1078 moduleTranslation, allocaIP);
1079 builder.restoreIP(codegenIP);
1081 moduleTranslation, bodyGenStatus);
1084 llvm::Value *numTeamsLower =
nullptr;
1085 if (
Value numTeamsLowerVar = op.getNumTeamsLower())
1086 numTeamsLower = moduleTranslation.
lookupValue(numTeamsLowerVar);
1088 llvm::Value *numTeamsUpper =
nullptr;
1089 if (
Value numTeamsUpperVar = op.getNumTeamsUpper())
1090 numTeamsUpper = moduleTranslation.
lookupValue(numTeamsUpperVar);
1092 llvm::Value *threadLimit =
nullptr;
1093 if (
Value threadLimitVar = op.getThreadLimit())
1094 threadLimit = moduleTranslation.
lookupValue(threadLimitVar);
1096 llvm::Value *ifExpr =
nullptr;
1097 if (
Value ifVar = op.getIfExpr())
1100 llvm::OpenMPIRBuilder::LocationDescription ompLoc(builder);
1102 ompLoc, bodyCB, numTeamsLower, numTeamsUpper, threadLimit, ifExpr));
1103 return bodyGenStatus;
1110 if (dependVars.empty())
1112 for (
auto dep : llvm::zip(dependVars, dependKinds->getValue())) {
1113 llvm::omp::RTLDependenceKindTy type;
1115 cast<mlir::omp::ClauseTaskDependAttr>(std::get<1>(dep)).getValue()) {
1116 case mlir::omp::ClauseTaskDepend::taskdependin:
1117 type = llvm::omp::RTLDependenceKindTy::DepIn;
1122 case mlir::omp::ClauseTaskDepend::taskdependout:
1123 case mlir::omp::ClauseTaskDepend::taskdependinout:
1124 type = llvm::omp::RTLDependenceKindTy::DepInOut;
1127 llvm::Value *depVal = moduleTranslation.
lookupValue(std::get<0>(dep));
1128 llvm::OpenMPIRBuilder::DependData dd(type, depVal->getType(), depVal);
1129 dds.emplace_back(dd);
1133 static LogicalResult
1136 using InsertPointTy = llvm::OpenMPIRBuilder::InsertPointTy;
1137 LogicalResult bodyGenStatus = success();
1138 if (taskOp.getUntiedAttr() || taskOp.getMergeableAttr() ||
1139 taskOp.getInReductionSyms() || taskOp.getPriority() ||
1140 !taskOp.getAllocateVars().empty() || !taskOp.getPrivateVars().empty() ||
1141 taskOp.getPrivateSyms()) {
1142 return taskOp.emitError(
"unhandled clauses for translation to LLVM IR");
1144 auto bodyCB = [&](InsertPointTy allocaIP, InsertPointTy codegenIP) {
1148 moduleTranslation, allocaIP);
1150 builder.restoreIP(codegenIP);
1152 moduleTranslation, bodyGenStatus);
1157 moduleTranslation, dds);
1159 llvm::OpenMPIRBuilder::InsertPointTy allocaIP =
1161 llvm::OpenMPIRBuilder::LocationDescription ompLoc(builder);
1163 ompLoc, allocaIP, bodyCB, !taskOp.getUntied(),
1165 moduleTranslation.
lookupValue(taskOp.getIfExpr()), dds));
1166 return bodyGenStatus;
1170 static LogicalResult
1173 using InsertPointTy = llvm::OpenMPIRBuilder::InsertPointTy;
1174 LogicalResult bodyGenStatus = success();
1175 if (!tgOp.getTaskReductionVars().empty() || !tgOp.getAllocateVars().empty()) {
1176 return tgOp.emitError(
"unhandled clauses for translation to LLVM IR");
1178 auto bodyCB = [&](InsertPointTy allocaIP, InsertPointTy codegenIP) {
1179 builder.restoreIP(codegenIP);
1181 moduleTranslation, bodyGenStatus);
1184 llvm::OpenMPIRBuilder::LocationDescription ompLoc(builder);
1186 ompLoc, allocaIP, bodyCB));
1187 return bodyGenStatus;
1190 static LogicalResult
1193 if (!twOp.getDependVars().empty() || twOp.getDependKinds() ||
1195 return twOp.emitError(
"unhandled clauses for translation to LLVM IR");
1202 static LogicalResult
1205 auto wsloopOp = cast<omp::WsloopOp>(opInst);
1206 if (!wsloopOp.getAllocateVars().empty() ||
1207 !wsloopOp.getAllocatorVars().empty() ||
1208 !wsloopOp.getPrivateVars().empty() || wsloopOp.getPrivateSyms())
1209 return opInst.
emitError(
"unhandled clauses for translation to LLVM IR");
1214 auto loopOp = cast<omp::LoopNestOp>(wsloopOp.getWrappedLoop());
1217 assert(isByRef.size() == wsloopOp.getNumReductionVars());
1221 wsloopOp.getScheduleKind().value_or(omp::ClauseScheduleKind::Static);
1224 llvm::Value *step = moduleTranslation.
lookupValue(loopOp.getLoopSteps()[0]);
1225 llvm::Type *ivType = step->getType();
1226 llvm::Value *chunk =
nullptr;
1227 if (wsloopOp.getScheduleChunk()) {
1228 llvm::Value *chunkVar =
1229 moduleTranslation.
lookupValue(wsloopOp.getScheduleChunk());
1230 chunk = builder.CreateSExtOrTrunc(chunkVar, ivType);
1235 llvm::OpenMPIRBuilder::InsertPointTy allocaIP =
1239 wsloopOp.getNumReductionVars());
1243 cast<omp::BlockArgOpenMPOpInterface>(opInst).getReductionBlockArgs();
1246 wsloopOp, reductionArgs, builder, moduleTranslation, allocaIP,
1247 reductionDecls, privateReductionVariables, reductionVariableMap,
1255 moduleTranslation, reductionVariableMap);
1258 llvm::OpenMPIRBuilder::LocationDescription ompLoc(builder);
1265 LogicalResult bodyGenStatus = success();
1266 auto bodyGen = [&](llvm::OpenMPIRBuilder::InsertPointTy ip, llvm::Value *iv) {
1269 loopOp.getRegion().front().getArgument(loopInfos.size()), iv);
1274 bodyInsertPoints.push_back(ip);
1276 if (loopInfos.size() != loopOp.getNumLoops() - 1)
1280 builder.restoreIP(ip);
1282 moduleTranslation, bodyGenStatus);
1291 for (
unsigned i = 0, e = loopOp.getNumLoops(); i < e; ++i) {
1292 llvm::Value *lowerBound =
1293 moduleTranslation.
lookupValue(loopOp.getLoopLowerBounds()[i]);
1294 llvm::Value *upperBound =
1295 moduleTranslation.
lookupValue(loopOp.getLoopUpperBounds()[i]);
1296 llvm::Value *step = moduleTranslation.
lookupValue(loopOp.getLoopSteps()[i]);
1301 llvm::OpenMPIRBuilder::LocationDescription loc = ompLoc;
1302 llvm::OpenMPIRBuilder::InsertPointTy computeIP = ompLoc.IP;
1304 loc = llvm::OpenMPIRBuilder::LocationDescription(bodyInsertPoints.back());
1305 computeIP = loopInfos.front()->getPreheaderIP();
1307 loopInfos.push_back(ompBuilder->createCanonicalLoop(
1308 loc, bodyGen, lowerBound, upperBound, step,
1309 true, loopOp.getLoopInclusive(), computeIP));
1311 if (failed(bodyGenStatus))
1317 llvm::IRBuilderBase::InsertPoint afterIP = loopInfos.front()->getAfterIP();
1318 llvm::CanonicalLoopInfo *loopInfo =
1319 ompBuilder->collapseLoops(ompLoc.DL, loopInfos, {});
1324 bool isOrdered = wsloopOp.getOrdered().has_value();
1325 std::optional<omp::ScheduleModifier> scheduleMod = wsloopOp.getScheduleMod();
1326 bool isSimd = wsloopOp.getScheduleSimd();
1328 ompBuilder->applyWorkshareLoop(
1329 ompLoc.DL, loopInfo, allocaIP, !wsloopOp.getNowait(),
1330 convertToScheduleKind(schedule), chunk, isSimd,
1331 scheduleMod == omp::ScheduleModifier::monotonic,
1332 scheduleMod == omp::ScheduleModifier::nonmonotonic, isOrdered);
1338 builder.restoreIP(afterIP);
1342 allocaIP, reductionDecls,
1343 privateReductionVariables, isByRef);
1347 static LogicalResult
1350 using InsertPointTy = llvm::OpenMPIRBuilder::InsertPointTy;
1352 assert(isByRef.size() == opInst.getNumReductionVars());
1356 LogicalResult bodyGenStatus = success();
1361 cast<omp::BlockArgOpenMPOpInterface>(*opInst).getPrivateBlockArgs();
1364 llvmPrivateVars.reserve(privateBlockArgs.size());
1365 privateDecls.reserve(privateBlockArgs.size());
1372 opInst.getNumReductionVars());
1375 auto bodyGenCB = [&](InsertPointTy allocaIP, InsertPointTy codeGenIP) {
1377 llvm::BranchInst *allocaTerminator =
1378 llvm::cast<llvm::BranchInst>(allocaIP.getBlock()->getTerminator());
1379 builder.SetInsertPoint(allocaTerminator);
1380 assert(allocaTerminator->getNumSuccessors() == 1 &&
1381 "This is an unconditional branch created by OpenMPIRBuilder");
1382 llvm::BasicBlock *afterAllocas = allocaTerminator->getSuccessor(0);
1396 llvm::BasicBlock *privAllocBlock =
nullptr;
1397 if (!privateBlockArgs.empty())
1398 privAllocBlock = splitBB(builder,
true,
"omp.private.latealloc");
1399 for (
unsigned i = 0; i < privateBlockArgs.size(); ++i) {
1400 Region &allocRegion = privateDecls[i].getAllocRegion();
1403 llvm::Value *nonPrivateVar =
1404 moduleTranslation.
lookupValue(opInst.getPrivateVars()[i]);
1405 assert(nonPrivateVar);
1406 moduleTranslation.
mapValue(privateDecls[i].getAllocMoldArg(),
1411 if (privateDecls[i].getAllocMoldArg().getUses().empty()) {
1416 builder.SetInsertPoint(allocaIP.getBlock()->getTerminator());
1418 builder.SetInsertPoint(privAllocBlock->getTerminator());
1421 builder, moduleTranslation, &phis))) {
1422 bodyGenStatus = failure();
1425 assert(phis.size() == 1 &&
"expected one allocation to be yielded");
1427 moduleTranslation.
mapValue(privateBlockArgs[i], phis[0]);
1428 llvmPrivateVars.push_back(phis[0]);
1440 cast<omp::BlockArgOpenMPOpInterface>(*opInst).getReductionBlockArgs();
1443 InsertPointTy(allocaIP.getBlock(),
1444 allocaIP.getBlock()->getTerminator()->getIterator());
1447 opInst, reductionArgs, builder, moduleTranslation, allocaIP,
1448 reductionDecls, privateReductionVariables, reductionVariableMap,
1449 deferredStores, isByRef)))
1450 bodyGenStatus = failure();
1453 bool needsFirstprivate =
1454 llvm::any_of(privateDecls, [](omp::PrivateClauseOp &privOp) {
1455 return privOp.getDataSharingType() ==
1456 omp::DataSharingClauseType::FirstPrivate;
1458 if (needsFirstprivate) {
1460 assert(afterAllocas->getSinglePredecessor());
1461 builder.SetInsertPoint(
1462 afterAllocas->getSinglePredecessor()->getTerminator());
1463 llvm::BasicBlock *copyBlock =
1464 splitBB(builder,
true,
"omp.private.copy");
1465 builder.SetInsertPoint(copyBlock->getFirstNonPHIOrDbgOrAlloca());
1467 for (
unsigned i = 0; i < privateBlockArgs.size(); ++i) {
1468 if (privateDecls[i].getDataSharingType() !=
1469 omp::DataSharingClauseType::FirstPrivate)
1473 Region ©Region = privateDecls[i].getCopyRegion();
1476 llvm::Value *nonPrivateVar =
1477 moduleTranslation.
lookupValue(opInst.getPrivateVars()[i]);
1478 assert(nonPrivateVar);
1479 moduleTranslation.
mapValue(privateDecls[i].getCopyMoldArg(),
1483 moduleTranslation.
mapValue(privateDecls[i].getCopyPrivateArg(),
1484 llvmPrivateVars[i]);
1487 builder.SetInsertPoint(builder.GetInsertBlock()->getTerminator());
1489 builder, moduleTranslation))) {
1490 bodyGenStatus = failure();
1503 builder.SetInsertPoint(builder.GetInsertBlock()->getTerminator());
1504 llvm::BasicBlock *initBlock = splitBB(builder,
true,
"omp.reduction.init");
1506 InsertPointTy(allocaIP.getBlock(),
1507 allocaIP.getBlock()->getTerminator()->getIterator());
1509 builder.restoreIP(allocaIP);
1511 for (
unsigned i = 0; i < opInst.getNumReductionVars(); ++i) {
1513 if (!reductionDecls[i].getAllocRegion().empty())
1519 byRefVars[i] = builder.CreateAlloca(
1520 moduleTranslation.
convertType(reductionDecls[i].getType()));
1524 builder.SetInsertPoint(initBlock->getFirstNonPHIOrDbgOrAlloca());
1529 for (
auto [data, addr] : deferredStores)
1530 builder.CreateStore(data, addr);
1532 for (
unsigned i = 0; i < opInst.getNumReductionVars(); ++i) {
1537 reductionVariableMap, i);
1539 reductionDecls[i].getInitializerRegion(),
"omp.reduction.neutral",
1540 builder, moduleTranslation, &phis)))
1541 bodyGenStatus = failure();
1542 assert(phis.size() == 1 &&
1543 "expected one value to be yielded from the "
1544 "reduction neutral element declaration region");
1546 builder.SetInsertPoint(builder.GetInsertBlock()->getTerminator());
1549 if (!reductionDecls[i].getAllocRegion().empty())
1556 builder.CreateStore(phis[0], byRefVars[i]);
1558 privateReductionVariables[i] = byRefVars[i];
1559 moduleTranslation.
mapValue(reductionArgs[i], phis[0]);
1560 reductionVariableMap.try_emplace(opInst.getReductionVars()[i], phis[0]);
1563 builder.CreateStore(phis[0], privateReductionVariables[i]);
1569 moduleTranslation.
forgetMapping(reductionDecls[i].getInitializerRegion());
1576 moduleTranslation, reductionVariableMap);
1581 moduleTranslation, allocaIP);
1584 builder.restoreIP(codeGenIP);
1587 moduleTranslation, bodyGenStatus);
1590 if (opInst.getNumReductionVars() > 0) {
1596 owningReductionGens, owningAtomicReductionGens,
1597 privateReductionVariables, reductionInfos);
1600 builder.SetInsertPoint(regionBlock->getTerminator());
1603 llvm::UnreachableInst *tempTerminator = builder.CreateUnreachable();
1604 builder.SetInsertPoint(tempTerminator);
1606 llvm::OpenMPIRBuilder::InsertPointTy contInsertPoint =
1607 ompBuilder->createReductions(builder.saveIP(), allocaIP,
1608 reductionInfos, isByRef,
false);
1609 if (!contInsertPoint.getBlock()) {
1610 bodyGenStatus = opInst->emitOpError() <<
"failed to convert reductions";
1614 tempTerminator->eraseFromParent();
1615 builder.restoreIP(contInsertPoint);
1619 auto privCB = [](InsertPointTy allocaIP, InsertPointTy codeGenIP,
1620 llvm::Value &, llvm::Value &val, llvm::Value *&replVal) {
1629 auto finiCB = [&](InsertPointTy codeGenIP) {
1630 InsertPointTy oldIP = builder.saveIP();
1631 builder.restoreIP(codeGenIP);
1636 llvm::transform(reductionDecls, std::back_inserter(reductionCleanupRegions),
1637 [](omp::DeclareReductionOp reductionDecl) {
1638 return &reductionDecl.getCleanupRegion();
1641 reductionCleanupRegions, privateReductionVariables,
1642 moduleTranslation, builder,
"omp.reduction.cleanup")))
1643 bodyGenStatus = failure();
1646 llvm::transform(privateDecls, std::back_inserter(privateCleanupRegions),
1647 [](omp::PrivateClauseOp privatizer) {
1648 return &privatizer.getDeallocRegion();
1652 privateCleanupRegions, llvmPrivateVars, moduleTranslation, builder,
1653 "omp.private.dealloc",
false)))
1654 bodyGenStatus = failure();
1656 builder.restoreIP(oldIP);
1659 llvm::Value *ifCond =
nullptr;
1660 if (
auto ifVar = opInst.getIfExpr())
1662 llvm::Value *numThreads =
nullptr;
1663 if (
auto numThreadsVar = opInst.getNumThreads())
1664 numThreads = moduleTranslation.
lookupValue(numThreadsVar);
1665 auto pbKind = llvm::omp::OMP_PROC_BIND_default;
1666 if (
auto bind = opInst.getProcBindKind())
1669 bool isCancellable =
false;
1671 llvm::OpenMPIRBuilder::InsertPointTy allocaIP =
1673 llvm::OpenMPIRBuilder::LocationDescription ompLoc(builder);
1676 ompBuilder->createParallel(ompLoc, allocaIP, bodyGenCB, privCB, finiCB,
1677 ifCond, numThreads, pbKind, isCancellable));
1679 return bodyGenStatus;
1683 static llvm::omp::OrderKind
1686 return llvm::omp::OrderKind::OMP_ORDER_unknown;
1688 case omp::ClauseOrderKind::Concurrent:
1689 return llvm::omp::OrderKind::OMP_ORDER_concurrent;
1691 llvm_unreachable(
"Unknown ClauseOrderKind kind");
1695 if (!op.getLinearVars().empty() || !op.getLinearStepVars().empty())
1696 return op.
emitError(
"linear clause not yet supported");
1698 if (!op.getPrivateVars().empty() || op.getPrivateSyms())
1699 return op.
emitError(
"privatization clauses not yet supported");
1701 if (!op.getReductionVars().empty() || op.getReductionByref() ||
1702 op.getReductionSyms())
1703 return op.
emitError(
"reduction clause not yet supported");
1709 static LogicalResult
1712 auto simdOp = cast<omp::SimdOp>(opInst);
1713 auto loopOp = cast<omp::LoopNestOp>(simdOp.getWrappedLoop());
1718 llvm::OpenMPIRBuilder::LocationDescription ompLoc(builder);
1725 LogicalResult bodyGenStatus = success();
1726 auto bodyGen = [&](llvm::OpenMPIRBuilder::InsertPointTy ip, llvm::Value *iv) {
1729 loopOp.getRegion().front().getArgument(loopInfos.size()), iv);
1734 bodyInsertPoints.push_back(ip);
1736 if (loopInfos.size() != loopOp.getNumLoops() - 1)
1740 builder.restoreIP(ip);
1742 moduleTranslation, bodyGenStatus);
1751 for (
unsigned i = 0, e = loopOp.getNumLoops(); i < e; ++i) {
1752 llvm::Value *lowerBound =
1753 moduleTranslation.
lookupValue(loopOp.getLoopLowerBounds()[i]);
1754 llvm::Value *upperBound =
1755 moduleTranslation.
lookupValue(loopOp.getLoopUpperBounds()[i]);
1756 llvm::Value *step = moduleTranslation.
lookupValue(loopOp.getLoopSteps()[i]);
1761 llvm::OpenMPIRBuilder::LocationDescription loc = ompLoc;
1762 llvm::OpenMPIRBuilder::InsertPointTy computeIP = ompLoc.IP;
1764 loc = llvm::OpenMPIRBuilder::LocationDescription(bodyInsertPoints.back(),
1766 computeIP = loopInfos.front()->getPreheaderIP();
1768 loopInfos.push_back(ompBuilder->createCanonicalLoop(
1769 loc, bodyGen, lowerBound, upperBound, step,
1770 true,
true, computeIP));
1772 if (failed(bodyGenStatus))
1777 llvm::IRBuilderBase::InsertPoint afterIP = loopInfos.front()->getAfterIP();
1778 llvm::CanonicalLoopInfo *loopInfo =
1779 ompBuilder->collapseLoops(ompLoc.DL, loopInfos, {});
1781 llvm::ConstantInt *simdlen =
nullptr;
1782 if (std::optional<uint64_t> simdlenVar = simdOp.getSimdlen())
1783 simdlen = builder.getInt64(simdlenVar.value());
1785 llvm::ConstantInt *safelen =
nullptr;
1786 if (std::optional<uint64_t> safelenVar = simdOp.getSafelen())
1787 safelen = builder.getInt64(safelenVar.value());
1789 llvm::MapVector<llvm::Value *, llvm::Value *> alignedVars;
1791 ompBuilder->applySimd(loopInfo, alignedVars,
1793 ? moduleTranslation.
lookupValue(simdOp.getIfExpr())
1795 order, simdlen, safelen);
1797 builder.restoreIP(afterIP);
1802 static llvm::AtomicOrdering
1805 return llvm::AtomicOrdering::Monotonic;
1808 case omp::ClauseMemoryOrderKind::Seq_cst:
1809 return llvm::AtomicOrdering::SequentiallyConsistent;
1810 case omp::ClauseMemoryOrderKind::Acq_rel:
1811 return llvm::AtomicOrdering::AcquireRelease;
1812 case omp::ClauseMemoryOrderKind::Acquire:
1813 return llvm::AtomicOrdering::Acquire;
1814 case omp::ClauseMemoryOrderKind::Release:
1815 return llvm::AtomicOrdering::Release;
1816 case omp::ClauseMemoryOrderKind::Relaxed:
1817 return llvm::AtomicOrdering::Monotonic;
1819 llvm_unreachable(
"Unknown ClauseMemoryOrderKind kind");
1823 static LogicalResult
1827 auto readOp = cast<omp::AtomicReadOp>(opInst);
1830 llvm::OpenMPIRBuilder::LocationDescription ompLoc(builder);
1833 llvm::Value *x = moduleTranslation.
lookupValue(readOp.getX());
1834 llvm::Value *v = moduleTranslation.
lookupValue(readOp.getV());
1836 llvm::Type *elementType =
1837 moduleTranslation.
convertType(readOp.getElementType());
1839 llvm::OpenMPIRBuilder::AtomicOpValue V = {v, elementType,
false,
false};
1840 llvm::OpenMPIRBuilder::AtomicOpValue X = {x, elementType,
false,
false};
1841 builder.restoreIP(ompBuilder->createAtomicRead(ompLoc, X, V, AO));
1846 static LogicalResult
1849 auto writeOp = cast<omp::AtomicWriteOp>(opInst);
1852 llvm::OpenMPIRBuilder::LocationDescription ompLoc(builder);
1854 llvm::Value *expr = moduleTranslation.
lookupValue(writeOp.getExpr());
1855 llvm::Value *dest = moduleTranslation.
lookupValue(writeOp.getX());
1856 llvm::Type *ty = moduleTranslation.
convertType(writeOp.getExpr().getType());
1857 llvm::OpenMPIRBuilder::AtomicOpValue x = {dest, ty,
false,
1859 builder.restoreIP(ompBuilder->createAtomicWrite(ompLoc, x, expr, ao));
1867 .Case([&](LLVM::AddOp) {
return llvm::AtomicRMWInst::BinOp::Add; })
1868 .Case([&](LLVM::SubOp) {
return llvm::AtomicRMWInst::BinOp::Sub; })
1869 .Case([&](LLVM::AndOp) {
return llvm::AtomicRMWInst::BinOp::And; })
1870 .Case([&](LLVM::OrOp) {
return llvm::AtomicRMWInst::BinOp::Or; })
1871 .Case([&](LLVM::XOrOp) {
return llvm::AtomicRMWInst::BinOp::Xor; })
1872 .Case([&](LLVM::UMaxOp) {
return llvm::AtomicRMWInst::BinOp::UMax; })
1873 .Case([&](LLVM::UMinOp) {
return llvm::AtomicRMWInst::BinOp::UMin; })
1874 .Case([&](LLVM::FAddOp) {
return llvm::AtomicRMWInst::BinOp::FAdd; })
1875 .Case([&](LLVM::FSubOp) {
return llvm::AtomicRMWInst::BinOp::FSub; })
1876 .Default(llvm::AtomicRMWInst::BinOp::BAD_BINOP);
1880 static LogicalResult
1882 llvm::IRBuilderBase &builder,
1887 auto &innerOpList = opInst.getRegion().front().getOperations();
1888 bool isXBinopExpr{
false};
1889 llvm::AtomicRMWInst::BinOp binop;
1891 llvm::Value *llvmExpr =
nullptr;
1892 llvm::Value *llvmX =
nullptr;
1893 llvm::Type *llvmXElementType =
nullptr;
1894 if (innerOpList.size() == 2) {
1900 opInst.getRegion().getArgument(0))) {
1901 return opInst.emitError(
"no atomic update operation with region argument"
1902 " as operand found inside atomic.update region");
1905 isXBinopExpr = innerOp.
getOperand(0) == opInst.getRegion().getArgument(0);
1907 llvmExpr = moduleTranslation.
lookupValue(mlirExpr);
1911 binop = llvm::AtomicRMWInst::BinOp::BAD_BINOP;
1913 llvmX = moduleTranslation.
lookupValue(opInst.getX());
1915 opInst.getRegion().getArgument(0).getType());
1916 llvm::OpenMPIRBuilder::AtomicOpValue llvmAtomicX = {llvmX, llvmXElementType,
1920 llvm::AtomicOrdering atomicOrdering =
1924 LogicalResult updateGenStatus = success();
1925 auto updateFn = [&opInst, &moduleTranslation, &updateGenStatus](
1926 llvm::Value *atomicx,
1927 llvm::IRBuilder<> &builder) -> llvm::Value * {
1929 moduleTranslation.
mapValue(*opInst.getRegion().args_begin(), atomicx);
1930 moduleTranslation.
mapBlock(&bb, builder.GetInsertBlock());
1931 if (failed(moduleTranslation.
convertBlock(bb,
true, builder))) {
1932 updateGenStatus = (opInst.emitError()
1933 <<
"unable to convert update operation to llvm IR");
1936 omp::YieldOp yieldop = dyn_cast<omp::YieldOp>(bb.
getTerminator());
1937 assert(yieldop && yieldop.getResults().size() == 1 &&
1938 "terminator must be omp.yield op and it must have exactly one "
1940 return moduleTranslation.
lookupValue(yieldop.getResults()[0]);
1945 llvm::OpenMPIRBuilder::LocationDescription ompLoc(builder);
1946 builder.restoreIP(ompBuilder->createAtomicUpdate(
1947 ompLoc, allocaIP, llvmAtomicX, llvmExpr, atomicOrdering, binop, updateFn,
1949 return updateGenStatus;
1952 static LogicalResult
1954 llvm::IRBuilderBase &builder,
1958 bool isXBinopExpr =
false, isPostfixUpdate =
false;
1959 llvm::AtomicRMWInst::BinOp binop = llvm::AtomicRMWInst::BinOp::BAD_BINOP;
1961 omp::AtomicUpdateOp atomicUpdateOp = atomicCaptureOp.getAtomicUpdateOp();
1962 omp::AtomicWriteOp atomicWriteOp = atomicCaptureOp.getAtomicWriteOp();
1964 assert((atomicUpdateOp || atomicWriteOp) &&
1965 "internal op must be an atomic.update or atomic.write op");
1967 if (atomicWriteOp) {
1968 isPostfixUpdate =
true;
1969 mlirExpr = atomicWriteOp.getExpr();
1971 isPostfixUpdate = atomicCaptureOp.getSecondOp() ==
1972 atomicCaptureOp.getAtomicUpdateOp().getOperation();
1973 auto &innerOpList = atomicUpdateOp.getRegion().front().getOperations();
1976 if (innerOpList.size() == 2) {
1979 atomicUpdateOp.getRegion().getArgument(0))) {
1980 return atomicUpdateOp.emitError(
1981 "no atomic update operation with region argument"
1982 " as operand found inside atomic.update region");
1986 innerOp.
getOperand(0) == atomicUpdateOp.getRegion().getArgument(0);
1989 binop = llvm::AtomicRMWInst::BinOp::BAD_BINOP;
1993 llvm::Value *llvmExpr = moduleTranslation.
lookupValue(mlirExpr);
1994 llvm::Value *llvmX =
1995 moduleTranslation.
lookupValue(atomicCaptureOp.getAtomicReadOp().getX());
1996 llvm::Value *llvmV =
1997 moduleTranslation.
lookupValue(atomicCaptureOp.getAtomicReadOp().getV());
1998 llvm::Type *llvmXElementType = moduleTranslation.
convertType(
1999 atomicCaptureOp.getAtomicReadOp().getElementType());
2000 llvm::OpenMPIRBuilder::AtomicOpValue llvmAtomicX = {llvmX, llvmXElementType,
2003 llvm::OpenMPIRBuilder::AtomicOpValue llvmAtomicV = {llvmV, llvmXElementType,
2007 llvm::AtomicOrdering atomicOrdering =
2010 LogicalResult updateGenStatus = success();
2011 auto updateFn = [&](llvm::Value *atomicx,
2012 llvm::IRBuilder<> &builder) -> llvm::Value * {
2014 return moduleTranslation.
lookupValue(atomicWriteOp.getExpr());
2015 Block &bb = *atomicUpdateOp.getRegion().
begin();
2016 moduleTranslation.
mapValue(*atomicUpdateOp.getRegion().args_begin(),
2018 moduleTranslation.
mapBlock(&bb, builder.GetInsertBlock());
2019 if (failed(moduleTranslation.
convertBlock(bb,
true, builder))) {
2020 updateGenStatus = (atomicUpdateOp.emitError()
2021 <<
"unable to convert update operation to llvm IR");
2024 omp::YieldOp yieldop = dyn_cast<omp::YieldOp>(bb.
getTerminator());
2025 assert(yieldop && yieldop.getResults().size() == 1 &&
2026 "terminator must be omp.yield op and it must have exactly one "
2028 return moduleTranslation.
lookupValue(yieldop.getResults()[0]);
2033 llvm::OpenMPIRBuilder::LocationDescription ompLoc(builder);
2034 builder.restoreIP(ompBuilder->createAtomicCapture(
2035 ompLoc, allocaIP, llvmAtomicX, llvmAtomicV, llvmExpr, atomicOrdering,
2036 binop, updateFn, atomicUpdateOp, isPostfixUpdate, isXBinopExpr));
2037 return updateGenStatus;
2042 static LogicalResult
2045 llvm::OpenMPIRBuilder::LocationDescription ompLoc(builder);
2046 auto threadprivateOp = cast<omp::ThreadprivateOp>(opInst);
2048 Value symAddr = threadprivateOp.getSymAddr();
2050 if (!isa<LLVM::AddressOfOp>(symOp))
2051 return opInst.
emitError(
"Addressing symbol not found");
2052 LLVM::AddressOfOp addressOfOp = dyn_cast<LLVM::AddressOfOp>(symOp);
2054 LLVM::GlobalOp global =
2055 addressOfOp.getGlobal(moduleTranslation.
symbolTable());
2056 llvm::GlobalValue *globalValue = moduleTranslation.
lookupGlobal(global);
2057 llvm::Type *type = globalValue->getValueType();
2058 llvm::TypeSize typeSize =
2059 builder.GetInsertBlock()->getModule()->getDataLayout().getTypeStoreSize(
2061 llvm::ConstantInt *size = builder.getInt64(typeSize.getFixedValue());
2062 llvm::StringRef suffix = llvm::StringRef(
".cache", 6);
2063 std::string cacheName = (Twine(global.getSymName()).concat(suffix)).str();
2064 llvm::Value *callInst =
2066 ompLoc, globalValue, size, cacheName);
2071 static llvm::OffloadEntriesInfoManager::OMPTargetDeviceClauseKind
2073 switch (deviceClause) {
2074 case mlir::omp::DeclareTargetDeviceType::host:
2075 return llvm::OffloadEntriesInfoManager::OMPTargetDeviceClauseHost;
2077 case mlir::omp::DeclareTargetDeviceType::nohost:
2078 return llvm::OffloadEntriesInfoManager::OMPTargetDeviceClauseNoHost;
2080 case mlir::omp::DeclareTargetDeviceType::any:
2081 return llvm::OffloadEntriesInfoManager::OMPTargetDeviceClauseAny;
2084 llvm_unreachable(
"unhandled device clause");
2087 static llvm::OffloadEntriesInfoManager::OMPTargetGlobalVarEntryKind
2089 mlir::omp::DeclareTargetCaptureClause captureClause) {
2090 switch (captureClause) {
2091 case mlir::omp::DeclareTargetCaptureClause::to:
2092 return llvm::OffloadEntriesInfoManager::OMPTargetGlobalVarEntryTo;
2093 case mlir::omp::DeclareTargetCaptureClause::link:
2094 return llvm::OffloadEntriesInfoManager::OMPTargetGlobalVarEntryLink;
2095 case mlir::omp::DeclareTargetCaptureClause::enter:
2096 return llvm::OffloadEntriesInfoManager::OMPTargetGlobalVarEntryEnter;
2098 llvm_unreachable(
"unhandled capture clause");
2103 llvm::OpenMPIRBuilder &ompBuilder) {
2105 llvm::raw_svector_ostream os(suffix);
2107 auto loc = globalOp->getLoc()->findInstanceOf<FileLineColLoc>();
2108 auto fileInfoCallBack = [&loc]() {
2109 return std::pair<std::string, uint64_t>(
2110 llvm::StringRef(loc.getFilename()), loc.getLine());
2114 "_%x", ompBuilder.getTargetEntryUniqueInfo(fileInfoCallBack).FileID);
2116 os <<
"_decl_tgt_ref_ptr";
2122 if (
auto addressOfOp =
2123 llvm::dyn_cast_if_present<LLVM::AddressOfOp>(value.
getDefiningOp())) {
2124 auto modOp = addressOfOp->getParentOfType<mlir::ModuleOp>();
2125 Operation *gOp = modOp.lookupSymbol(addressOfOp.getGlobalName());
2126 if (
auto declareTargetGlobal =
2127 llvm::dyn_cast<mlir::omp::DeclareTargetInterface>(gOp))
2128 if (declareTargetGlobal.getDeclareTargetCaptureClause() ==
2129 mlir::omp::DeclareTargetCaptureClause::link)
2138 static llvm::Value *
2145 if (
auto addressOfOp =
2146 llvm::dyn_cast_if_present<LLVM::AddressOfOp>(value.
getDefiningOp())) {
2147 if (
auto gOp = llvm::dyn_cast_or_null<LLVM::GlobalOp>(
2148 addressOfOp->getParentOfType<mlir::ModuleOp>().lookupSymbol(
2149 addressOfOp.getGlobalName()))) {
2151 if (
auto declareTargetGlobal =
2152 llvm::dyn_cast<mlir::omp::DeclareTargetInterface>(
2153 gOp.getOperation())) {
2157 if ((declareTargetGlobal.getDeclareTargetCaptureClause() ==
2158 mlir::omp::DeclareTargetCaptureClause::link) ||
2159 (declareTargetGlobal.getDeclareTargetCaptureClause() ==
2160 mlir::omp::DeclareTargetCaptureClause::to &&
2161 ompBuilder->Config.hasRequiresUnifiedSharedMemory())) {
2165 if (gOp.getSymName().contains(suffix))
2170 (gOp.getSymName().str() + suffix.str()).str());
2186 struct MapInfoData : llvm::OpenMPIRBuilder::MapInfosTy {
2198 void append(MapInfoData &CurInfo) {
2199 IsDeclareTarget.append(CurInfo.IsDeclareTarget.begin(),
2200 CurInfo.IsDeclareTarget.end());
2201 MapClause.append(CurInfo.MapClause.begin(), CurInfo.MapClause.end());
2202 OriginalValue.append(CurInfo.OriginalValue.begin(),
2203 CurInfo.OriginalValue.end());
2204 BaseType.append(CurInfo.BaseType.begin(), CurInfo.BaseType.end());
2205 llvm::OpenMPIRBuilder::MapInfosTy::append(CurInfo);
2211 if (
auto nestedArrTy = llvm::dyn_cast_if_present<LLVM::LLVMArrayType>(
2212 arrTy.getElementType()))
2228 Operation *clauseOp, llvm::Value *basePointer,
2229 llvm::Type *baseType, llvm::IRBuilderBase &builder,
2231 if (
auto memberClause =
2232 mlir::dyn_cast_if_present<mlir::omp::MapInfoOp>(clauseOp)) {
2237 if (!memberClause.getBounds().empty()) {
2238 llvm::Value *elementCount = builder.getInt64(1);
2239 for (
auto bounds : memberClause.getBounds()) {
2240 if (
auto boundOp = mlir::dyn_cast_if_present<mlir::omp::MapBoundsOp>(
2241 bounds.getDefiningOp())) {
2246 elementCount = builder.CreateMul(
2250 moduleTranslation.
lookupValue(boundOp.getUpperBound()),
2251 moduleTranslation.
lookupValue(boundOp.getLowerBound())),
2252 builder.getInt64(1)));
2259 if (
auto arrTy = llvm::dyn_cast_if_present<LLVM::LLVMArrayType>(type))
2267 return builder.CreateMul(elementCount,
2268 builder.getInt64(underlyingTypeSzInBits / 8));
2278 llvm::IRBuilderBase &builder,
const ArrayRef<Value> &useDevPtrOperands = {},
2280 auto checkIsAMember = [](
const auto &mapVars,
auto mapOp) {
2288 for (
Value mapValue : mapVars) {
2289 auto map = cast<omp::MapInfoOp>(mapValue.getDefiningOp());
2290 for (
auto member : map.getMembers())
2291 if (member == mapOp)
2298 for (
Value mapValue : mapVars) {
2299 auto mapOp = cast<omp::MapInfoOp>(mapValue.getDefiningOp());
2301 mapOp.getVarPtrPtr() ? mapOp.getVarPtrPtr() : mapOp.getVarPtr();
2302 mapData.OriginalValue.push_back(moduleTranslation.
lookupValue(offloadPtr));
2303 mapData.Pointers.push_back(mapData.OriginalValue.back());
2305 if (llvm::Value *refPtr =
2307 moduleTranslation)) {
2308 mapData.IsDeclareTarget.push_back(
true);
2309 mapData.BasePointers.push_back(refPtr);
2311 mapData.IsDeclareTarget.push_back(
false);
2312 mapData.BasePointers.push_back(mapData.OriginalValue.back());
2315 mapData.BaseType.push_back(
2316 moduleTranslation.
convertType(mapOp.getVarType()));
2317 mapData.Sizes.push_back(
2318 getSizeInBytes(dl, mapOp.getVarType(), mapOp, mapData.Pointers.back(),
2319 mapData.BaseType.back(), builder, moduleTranslation));
2320 mapData.MapClause.push_back(mapOp.getOperation());
2321 mapData.Types.push_back(
2322 llvm::omp::OpenMPOffloadMappingFlags(mapOp.getMapType().value()));
2326 mapData.IsAMapping.push_back(
true);
2327 mapData.IsAMember.push_back(checkIsAMember(mapVars, mapOp));
2330 auto findMapInfo = [&mapData](llvm::Value *val,
2331 llvm::OpenMPIRBuilder::DeviceInfoTy devInfoTy) {
2334 for (llvm::Value *basePtr : mapData.OriginalValue) {
2335 if (basePtr == val && mapData.IsAMapping[index]) {
2337 mapData.Types[index] |=
2338 llvm::omp::OpenMPOffloadMappingFlags::OMP_MAP_RETURN_PARAM;
2339 mapData.DevicePointers[index] = devInfoTy;
2348 llvm::OpenMPIRBuilder::DeviceInfoTy devInfoTy) {
2349 for (
Value mapValue : useDevOperands) {
2350 auto mapOp = cast<omp::MapInfoOp>(mapValue.getDefiningOp());
2352 mapOp.getVarPtrPtr() ? mapOp.getVarPtrPtr() : mapOp.getVarPtr();
2353 llvm::Value *origValue = moduleTranslation.
lookupValue(offloadPtr);
2356 if (!findMapInfo(origValue, devInfoTy)) {
2357 mapData.OriginalValue.push_back(origValue);
2358 mapData.Pointers.push_back(mapData.OriginalValue.back());
2359 mapData.IsDeclareTarget.push_back(
false);
2360 mapData.BasePointers.push_back(mapData.OriginalValue.back());
2361 mapData.BaseType.push_back(
2362 moduleTranslation.
convertType(mapOp.getVarType()));
2363 mapData.Sizes.push_back(builder.getInt64(0));
2364 mapData.MapClause.push_back(mapOp.getOperation());
2365 mapData.Types.push_back(
2366 llvm::omp::OpenMPOffloadMappingFlags::OMP_MAP_RETURN_PARAM);
2369 mapData.DevicePointers.push_back(devInfoTy);
2370 mapData.IsAMapping.push_back(
false);
2371 mapData.IsAMember.push_back(checkIsAMember(useDevOperands, mapOp));
2376 addDevInfos(useDevAddrOperands, llvm::OpenMPIRBuilder::DeviceInfoTy::Address);
2377 addDevInfos(useDevPtrOperands, llvm::OpenMPIRBuilder::DeviceInfoTy::Pointer);
2381 auto *res = llvm::find(mapData.MapClause, memberOp);
2382 assert(res != mapData.MapClause.end() &&
2383 "MapInfoOp for member not found in MapData, cannot return index");
2384 return std::distance(mapData.MapClause.begin(), res);
2392 if (indexAttr.size() == 1)
2394 dyn_cast<omp::MapInfoOp>(mapInfo.getMembers()[0].getDefiningOp()))
2399 std::iota(indices.begin(), indices.end(), 0);
2401 llvm::sort(indices.begin(), indices.end(),
2402 [&](
const size_t a,
const size_t b) {
2403 auto indexValues = indexAttr.getValues<int32_t>();
2404 for (int i = 0; i < shape[1]; ++i) {
2405 int aIndex = indexValues[a * shape[1] + i];
2406 int bIndex = indexValues[b * shape[1] + i];
2408 if (aIndex == bIndex)
2411 if (aIndex != -1 && bIndex == -1)
2414 if (aIndex == -1 && bIndex != -1)
2418 if (aIndex < bIndex)
2421 if (bIndex < aIndex)
2431 return llvm::cast<omp::MapInfoOp>(
2432 mapInfo.getMembers()[indices.front()].getDefiningOp());
2454 std::vector<llvm::Value *>
2456 llvm::IRBuilderBase &builder,
bool isArrayTy,
2458 std::vector<llvm::Value *> idx;
2469 idx.push_back(builder.getInt64(0));
2470 for (
int i = bounds.size() - 1; i >= 0; --i) {
2471 if (
auto boundOp = dyn_cast_if_present<omp::MapBoundsOp>(
2472 bounds[i].getDefiningOp())) {
2473 idx.push_back(moduleTranslation.
lookupValue(boundOp.getLowerBound()));
2495 std::vector<llvm::Value *> dimensionIndexSizeOffset{builder.getInt64(1)};
2496 for (
size_t i = 1; i < bounds.size(); ++i) {
2497 if (
auto boundOp = dyn_cast_if_present<omp::MapBoundsOp>(
2498 bounds[i].getDefiningOp())) {
2499 dimensionIndexSizeOffset.push_back(builder.CreateMul(
2500 moduleTranslation.
lookupValue(boundOp.getExtent()),
2501 dimensionIndexSizeOffset[i - 1]));
2509 for (
int i = bounds.size() - 1; i >= 0; --i) {
2510 if (
auto boundOp = dyn_cast_if_present<omp::MapBoundsOp>(
2511 bounds[i].getDefiningOp())) {
2513 idx.emplace_back(builder.CreateMul(
2514 moduleTranslation.
lookupValue(boundOp.getLowerBound()),
2515 dimensionIndexSizeOffset[i]));
2517 idx.back() = builder.CreateAdd(
2518 idx.back(), builder.CreateMul(moduleTranslation.
lookupValue(
2519 boundOp.getLowerBound()),
2520 dimensionIndexSizeOffset[i]));
2545 llvm::OpenMPIRBuilder &ompBuilder,
DataLayout &dl,
2546 llvm::OpenMPIRBuilder::MapInfosTy &combinedInfo, MapInfoData &mapData,
2547 uint64_t mapDataIndex,
bool isTargetParams) {
2549 combinedInfo.Types.emplace_back(
2551 ? llvm::omp::OpenMPOffloadMappingFlags::OMP_MAP_TARGET_PARAM
2552 : llvm::omp::OpenMPOffloadMappingFlags::OMP_MAP_NONE);
2553 combinedInfo.DevicePointers.emplace_back(
2554 mapData.DevicePointers[mapDataIndex]);
2556 mapData.MapClause[mapDataIndex]->getLoc(), ompBuilder));
2557 combinedInfo.BasePointers.emplace_back(mapData.BasePointers[mapDataIndex]);
2567 llvm::cast<omp::MapInfoOp>(mapData.MapClause[mapDataIndex]);
2569 llvm::Value *lowAddr, *highAddr;
2570 if (!parentClause.getPartialMap()) {
2571 lowAddr = builder.CreatePointerCast(mapData.Pointers[mapDataIndex],
2572 builder.getPtrTy());
2573 highAddr = builder.CreatePointerCast(
2574 builder.CreateConstGEP1_32(mapData.BaseType[mapDataIndex],
2575 mapData.Pointers[mapDataIndex], 1),
2576 builder.getPtrTy());
2577 combinedInfo.Pointers.emplace_back(mapData.Pointers[mapDataIndex]);
2579 auto mapOp = dyn_cast<omp::MapInfoOp>(mapData.MapClause[mapDataIndex]);
2582 lowAddr = builder.CreatePointerCast(mapData.Pointers[firstMemberIdx],
2583 builder.getPtrTy());
2586 highAddr = builder.CreatePointerCast(
2587 builder.CreateGEP(mapData.BaseType[lastMemberIdx],
2588 mapData.Pointers[lastMemberIdx], builder.getInt64(1)),
2589 builder.getPtrTy());
2590 combinedInfo.Pointers.emplace_back(mapData.Pointers[firstMemberIdx]);
2593 llvm::Value *size = builder.CreateIntCast(
2594 builder.CreatePtrDiff(builder.getInt8Ty(), highAddr, lowAddr),
2595 builder.getInt64Ty(),
2597 combinedInfo.Sizes.push_back(size);
2604 llvm::omp::OpenMPOffloadMappingFlags mapFlag =
2605 llvm::omp::OpenMPOffloadMappingFlags::OMP_MAP_TO;
2607 llvm::omp::OpenMPOffloadMappingFlags memberOfFlag =
2608 ompBuilder.getMemberOfFlag(combinedInfo.BasePointers.size() - 1);
2609 ompBuilder.setCorrectMemberOfFlag(mapFlag, memberOfFlag);
2617 if (!parentClause.getPartialMap()) {
2618 combinedInfo.Types.emplace_back(mapFlag);
2619 combinedInfo.DevicePointers.emplace_back(
2622 mapData.MapClause[mapDataIndex]->getLoc(), ompBuilder));
2623 combinedInfo.BasePointers.emplace_back(mapData.BasePointers[mapDataIndex]);
2624 combinedInfo.Pointers.emplace_back(mapData.Pointers[mapDataIndex]);
2625 combinedInfo.Sizes.emplace_back(mapData.Sizes[mapDataIndex]);
2627 return memberOfFlag;
2639 if (mapOp.getVarPtrPtr())
2654 llvm::OpenMPIRBuilder &ompBuilder,
DataLayout &dl,
2655 llvm::OpenMPIRBuilder::MapInfosTy &combinedInfo, MapInfoData &mapData,
2656 uint64_t mapDataIndex, llvm::omp::OpenMPOffloadMappingFlags memberOfFlag) {
2659 llvm::cast<omp::MapInfoOp>(mapData.MapClause[mapDataIndex]);
2661 for (
auto mappedMembers : parentClause.getMembers()) {
2663 llvm::cast<omp::MapInfoOp>(mappedMembers.getDefiningOp());
2666 assert(memberDataIdx >= 0 &&
"could not find mapped member of structure");
2671 llvm::omp::OpenMPOffloadMappingFlags(memberClause.getMapType().value());
2672 mapFlag &= ~
llvm::omp::OpenMPOffloadMappingFlags::OMP_MAP_TARGET_PARAM;
2673 mapFlag |= llvm::omp::OpenMPOffloadMappingFlags::OMP_MAP_MEMBER_OF;
2674 ompBuilder.setCorrectMemberOfFlag(mapFlag, memberOfFlag);
2676 mapFlag |= llvm::omp::OpenMPOffloadMappingFlags::OMP_MAP_PTR_AND_OBJ;
2678 combinedInfo.Types.emplace_back(mapFlag);
2679 combinedInfo.DevicePointers.emplace_back(
2680 mapData.DevicePointers[memberDataIdx]);
2681 combinedInfo.Names.emplace_back(
2683 combinedInfo.BasePointers.emplace_back(mapData.BasePointers[mapDataIndex]);
2684 combinedInfo.Pointers.emplace_back(mapData.Pointers[memberDataIdx]);
2685 combinedInfo.Sizes.emplace_back(mapData.Sizes[memberDataIdx]);
2691 llvm::OpenMPIRBuilder::MapInfosTy &combinedInfo,
2692 bool isTargetParams,
int mapDataParentIdx = -1) {
2696 auto mapFlag = mapData.Types[mapDataIdx];
2697 auto mapInfoOp = llvm::cast<omp::MapInfoOp>(mapData.MapClause[mapDataIdx]);
2701 mapFlag |= llvm::omp::OpenMPOffloadMappingFlags::OMP_MAP_PTR_AND_OBJ;
2703 if (isTargetParams && !mapData.IsDeclareTarget[mapDataIdx])
2704 mapFlag |= llvm::omp::OpenMPOffloadMappingFlags::OMP_MAP_TARGET_PARAM;
2706 if (mapInfoOp.getMapCaptureType().value() ==
2707 omp::VariableCaptureKind::ByCopy &&
2709 mapFlag |= llvm::omp::OpenMPOffloadMappingFlags::OMP_MAP_LITERAL;
2714 if (mapDataParentIdx >= 0)
2715 combinedInfo.BasePointers.emplace_back(
2716 mapData.BasePointers[mapDataParentIdx]);
2718 combinedInfo.BasePointers.emplace_back(mapData.BasePointers[mapDataIdx]);
2720 combinedInfo.Pointers.emplace_back(mapData.Pointers[mapDataIdx]);
2721 combinedInfo.DevicePointers.emplace_back(mapData.DevicePointers[mapDataIdx]);
2722 combinedInfo.Names.emplace_back(mapData.Names[mapDataIdx]);
2723 combinedInfo.Types.emplace_back(mapFlag);
2724 combinedInfo.Sizes.emplace_back(mapData.Sizes[mapDataIdx]);
2729 llvm::OpenMPIRBuilder &ompBuilder,
DataLayout &dl,
2730 llvm::OpenMPIRBuilder::MapInfosTy &combinedInfo, MapInfoData &mapData,
2731 uint64_t mapDataIndex,
bool isTargetParams) {
2733 llvm::cast<omp::MapInfoOp>(mapData.MapClause[mapDataIndex]);
2738 if (parentClause.getMembers().size() == 1 && parentClause.getPartialMap()) {
2739 auto memberClause = llvm::cast<omp::MapInfoOp>(
2740 parentClause.getMembers()[0].getDefiningOp());
2757 llvm::omp::OpenMPOffloadMappingFlags memberOfParentFlag =
2759 combinedInfo, mapData, mapDataIndex, isTargetParams);
2761 combinedInfo, mapData, mapDataIndex,
2762 memberOfParentFlag);
2772 llvm::IRBuilderBase &builder) {
2773 for (
size_t i = 0; i < mapData.MapClause.size(); ++i) {
2775 if (!mapData.IsDeclareTarget[i]) {
2776 auto mapOp = cast<omp::MapInfoOp>(mapData.MapClause[i]);
2777 omp::VariableCaptureKind captureKind =
2778 mapOp.getMapCaptureType().value_or(omp::VariableCaptureKind::ByRef);
2788 switch (captureKind) {
2789 case omp::VariableCaptureKind::ByRef: {
2790 llvm::Value *newV = mapData.Pointers[i];
2792 moduleTranslation, builder, mapData.BaseType[i]->isArrayTy(),
2795 newV = builder.CreateLoad(builder.getPtrTy(), newV);
2797 if (!offsetIdx.empty())
2798 newV = builder.CreateInBoundsGEP(mapData.BaseType[i], newV, offsetIdx,
2800 mapData.Pointers[i] = newV;
2802 case omp::VariableCaptureKind::ByCopy: {
2803 llvm::Type *type = mapData.BaseType[i];
2805 if (mapData.Pointers[i]->getType()->isPointerTy())
2806 newV = builder.CreateLoad(type, mapData.Pointers[i]);
2808 newV = mapData.Pointers[i];
2811 auto curInsert = builder.saveIP();
2813 auto *memTempAlloc =
2814 builder.CreateAlloca(builder.getPtrTy(),
nullptr,
".casted");
2815 builder.restoreIP(curInsert);
2817 builder.CreateStore(newV, memTempAlloc);
2818 newV = builder.CreateLoad(builder.getPtrTy(), memTempAlloc);
2821 mapData.Pointers[i] = newV;
2822 mapData.BasePointers[i] = newV;
2824 case omp::VariableCaptureKind::This:
2825 case omp::VariableCaptureKind::VLAType:
2826 mapData.MapClause[i]->emitOpError(
"Unhandled capture kind");
2837 llvm::OpenMPIRBuilder::MapInfosTy &combinedInfo,
2838 MapInfoData &mapData,
bool isTargetParams =
false) {
2860 for (
size_t i = 0; i < mapData.MapClause.size(); ++i) {
2863 if (mapData.IsAMember[i])
2866 auto mapInfoOp = dyn_cast<omp::MapInfoOp>(mapData.MapClause[i]);
2867 if (!mapInfoOp.getMembers().empty()) {
2869 combinedInfo, mapData, i, isTargetParams);
2877 static LogicalResult
2880 llvm::Value *ifCond =
nullptr;
2881 int64_t deviceID = llvm::omp::OMP_DEVICEID_UNDEF;
2885 llvm::omp::RuntimeFunction RTLFn;
2889 llvm::OpenMPIRBuilder::TargetDataInfo info(
true,
2892 LogicalResult result =
2894 .Case([&](omp::TargetDataOp dataOp) {
2895 if (
auto ifVar = dataOp.getIfExpr())
2898 if (
auto devId = dataOp.getDevice())
2900 dyn_cast<LLVM::ConstantOp>(devId.getDefiningOp()))
2901 if (
auto intAttr = dyn_cast<IntegerAttr>(constOp.getValue()))
2902 deviceID = intAttr.getInt();
2904 mapVars = dataOp.getMapVars();
2905 useDevicePtrVars = dataOp.getUseDevicePtrVars();
2906 useDeviceAddrVars = dataOp.getUseDeviceAddrVars();
2909 .Case([&](omp::TargetEnterDataOp enterDataOp) {
2910 if (!enterDataOp.getDependVars().empty())
2911 return (LogicalResult)(enterDataOp.emitError(
2912 "`depend` is not supported yet"));
2914 if (
auto ifVar = enterDataOp.getIfExpr())
2917 if (
auto devId = enterDataOp.getDevice())
2919 dyn_cast<LLVM::ConstantOp>(devId.getDefiningOp()))
2920 if (
auto intAttr = dyn_cast<IntegerAttr>(constOp.getValue()))
2921 deviceID = intAttr.getInt();
2923 enterDataOp.getNowait()
2924 ? llvm::omp::OMPRTL___tgt_target_data_begin_nowait_mapper
2925 : llvm::omp::OMPRTL___tgt_target_data_begin_mapper;
2926 mapVars = enterDataOp.getMapVars();
2927 info.HasNoWait = enterDataOp.getNowait();
2930 .Case([&](omp::TargetExitDataOp exitDataOp) {
2931 if (!exitDataOp.getDependVars().empty())
2932 return (LogicalResult)(exitDataOp.emitError(
2933 "`depend` is not supported yet"));
2935 if (
auto ifVar = exitDataOp.getIfExpr())
2938 if (
auto devId = exitDataOp.getDevice())
2940 dyn_cast<LLVM::ConstantOp>(devId.getDefiningOp()))
2941 if (
auto intAttr = dyn_cast<IntegerAttr>(constOp.getValue()))
2942 deviceID = intAttr.getInt();
2944 RTLFn = exitDataOp.getNowait()
2945 ? llvm::omp::OMPRTL___tgt_target_data_end_nowait_mapper
2946 : llvm::omp::OMPRTL___tgt_target_data_end_mapper;
2947 mapVars = exitDataOp.getMapVars();
2948 info.HasNoWait = exitDataOp.getNowait();
2951 .Case([&](omp::TargetUpdateOp updateDataOp) {
2952 if (!updateDataOp.getDependVars().empty())
2953 return (LogicalResult)(updateDataOp.emitError(
2954 "`depend` is not supported yet"));
2956 if (
auto ifVar = updateDataOp.getIfExpr())
2959 if (
auto devId = updateDataOp.getDevice())
2961 dyn_cast<LLVM::ConstantOp>(devId.getDefiningOp()))
2962 if (
auto intAttr = dyn_cast<IntegerAttr>(constOp.getValue()))
2963 deviceID = intAttr.getInt();
2966 updateDataOp.getNowait()
2967 ? llvm::omp::OMPRTL___tgt_target_data_update_nowait_mapper
2968 : llvm::omp::OMPRTL___tgt_target_data_update_mapper;
2969 mapVars = updateDataOp.getMapVars();
2970 info.HasNoWait = updateDataOp.getNowait();
2974 return op->
emitError(
"unsupported OpenMP operation: ")
2981 using InsertPointTy = llvm::OpenMPIRBuilder::InsertPointTy;
2983 MapInfoData mapData;
2985 builder, useDevicePtrVars, useDeviceAddrVars);
2988 llvm::OpenMPIRBuilder::MapInfosTy combinedInfo;
2990 [&](InsertPointTy codeGenIP) -> llvm::OpenMPIRBuilder::MapInfosTy & {
2991 builder.restoreIP(codeGenIP);
2992 genMapInfos(builder, moduleTranslation, DL, combinedInfo, mapData);
2993 return combinedInfo;
2999 [&moduleTranslation](
3000 llvm::OpenMPIRBuilder::DeviceInfoTy type,
3002 llvm::OpenMPIRBuilder::MapValuesArrayTy &basePointers,
3003 llvm::OpenMPIRBuilder::MapDeviceInfoArrayTy &devicePointers,
3007 auto basePtrRange = llvm::map_range(
3008 llvm::make_filter_range(
3009 llvm::zip_equal(basePointers, devicePointers),
3010 [type](
auto x) {
return std::get<1>(x) == type; }),
3011 [](
auto x) {
return std::get<0>(x); });
3016 for (
auto [arg, basePointer] : llvm::zip_equal(blockArgs, basePtrRange))
3017 moduleTranslation.
mapValue(arg, mapper ? mapper(basePointer)
3021 using BodyGenTy = llvm::OpenMPIRBuilder::BodyGenTy;
3022 LogicalResult bodyGenStatus = success();
3023 auto bodyGenCB = [&](InsertPointTy codeGenIP, BodyGenTy bodyGenType) {
3024 assert(isa<omp::TargetDataOp>(op) &&
3025 "BodyGen requested for non TargetDataOp");
3026 auto blockArgIface = cast<omp::BlockArgOpenMPOpInterface>(op);
3027 Region ®ion = cast<omp::TargetDataOp>(op).getRegion();
3028 switch (bodyGenType) {
3029 case BodyGenTy::Priv:
3031 if (!info.DevicePtrInfoMap.empty()) {
3032 builder.restoreIP(codeGenIP);
3034 mapUseDevice(llvm::OpenMPIRBuilder::DeviceInfoTy::Address,
3035 blockArgIface.getUseDeviceAddrBlockArgs(),
3036 combinedInfo.BasePointers, combinedInfo.DevicePointers,
3037 [&](llvm::Value *basePointer) -> llvm::Value * {
3038 return builder.CreateLoad(
3040 info.DevicePtrInfoMap[basePointer].second);
3042 mapUseDevice(llvm::OpenMPIRBuilder::DeviceInfoTy::Pointer,
3043 blockArgIface.getUseDevicePtrBlockArgs(),
3044 combinedInfo.BasePointers, combinedInfo.DevicePointers,
3045 [&](llvm::Value *basePointer) {
3046 return info.DevicePtrInfoMap[basePointer].second;
3050 builder, moduleTranslation);
3053 case BodyGenTy::DupNoPriv:
3055 case BodyGenTy::NoPriv:
3057 if (info.DevicePtrInfoMap.empty()) {
3058 builder.restoreIP(codeGenIP);
3061 if (ompBuilder->Config.IsTargetDevice.value_or(
false)) {
3062 mapUseDevice(llvm::OpenMPIRBuilder::DeviceInfoTy::Address,
3063 blockArgIface.getUseDeviceAddrBlockArgs(),
3064 mapData.BasePointers, mapData.DevicePointers);
3065 mapUseDevice(llvm::OpenMPIRBuilder::DeviceInfoTy::Pointer,
3066 blockArgIface.getUseDevicePtrBlockArgs(),
3067 mapData.BasePointers, mapData.DevicePointers);
3071 builder, moduleTranslation);
3075 return builder.saveIP();
3078 llvm::OpenMPIRBuilder::LocationDescription ompLoc(builder);
3079 llvm::OpenMPIRBuilder::InsertPointTy allocaIP =
3081 if (isa<omp::TargetDataOp>(op)) {
3082 builder.restoreIP(ompBuilder->createTargetData(
3083 ompLoc, allocaIP, builder.saveIP(), builder.getInt64(deviceID), ifCond,
3084 info, genMapInfoCB,
nullptr, bodyGenCB));
3086 builder.restoreIP(ompBuilder->createTargetData(
3087 ompLoc, allocaIP, builder.saveIP(), builder.getInt64(deviceID), ifCond,
3088 info, genMapInfoCB, &RTLFn));
3091 return bodyGenStatus;
3099 if (!cast<mlir::ModuleOp>(op))
3104 ompBuilder->M.addModuleFlag(llvm::Module::Max,
"openmp-device",
3105 attribute.getOpenmpDeviceVersion());
3107 if (attribute.getNoGpuLib())
3110 ompBuilder->createGlobalFlag(
3111 attribute.getDebugKind() ,
3112 "__omp_rtl_debug_kind");
3113 ompBuilder->createGlobalFlag(
3115 .getAssumeTeamsOversubscription()
3117 "__omp_rtl_assume_teams_oversubscription");
3118 ompBuilder->createGlobalFlag(
3120 .getAssumeThreadsOversubscription()
3122 "__omp_rtl_assume_threads_oversubscription");
3123 ompBuilder->createGlobalFlag(
3124 attribute.getAssumeNoThreadState() ,
3125 "__omp_rtl_assume_no_thread_state");
3126 ompBuilder->createGlobalFlag(
3128 .getAssumeNoNestedParallelism()
3130 "__omp_rtl_assume_no_nested_parallelism");
3135 omp::TargetOp targetOp,
3136 llvm::StringRef parentName =
"") {
3137 auto fileLoc = targetOp.getLoc()->findInstanceOf<FileLineColLoc>();
3139 assert(fileLoc &&
"No file found from location");
3140 StringRef fileName = fileLoc.getFilename().getValue();
3142 llvm::sys::fs::UniqueID id;
3143 if (
auto ec = llvm::sys::fs::getUniqueID(fileName,
id)) {
3144 targetOp.emitError(
"Unable to get unique ID for file");
3148 uint64_t line = fileLoc.getLine();
3149 targetInfo = llvm::TargetRegionEntryInfo(parentName,
id.getDevice(),
3150 id.getFile(), line);
3155 auto targetOp = cast<omp::TargetOp>(opInst);
3156 if (targetOp.getIfExpr()) {
3157 opInst.
emitError(
"If clause not yet supported");
3161 if (targetOp.getDevice()) {
3162 opInst.
emitError(
"Device clause not yet supported");
3166 if (targetOp.getThreadLimit()) {
3167 opInst.
emitError(
"Thread limit clause not yet supported");
3171 if (!targetOp.getAllocateVars().empty() ||
3172 !targetOp.getAllocatorVars().empty()) {
3173 opInst.
emitError(
"Allocate clause not yet supported");
3177 if (!targetOp.getInReductionVars().empty() ||
3178 targetOp.getInReductionByref() || targetOp.getInReductionSyms()) {
3179 opInst.
emitError(
"In reduction clause not yet supported");
3189 llvm::IRBuilderBase &builder, llvm::Function *func) {
3190 for (
size_t i = 0; i < mapData.MapClause.size(); ++i) {
3203 if (mapData.IsDeclareTarget[i]) {
3210 if (
auto *constant = dyn_cast<llvm::Constant>(mapData.OriginalValue[i]))
3211 convertUsersOfConstantsToInstructions(constant, func,
false);
3218 for (llvm::User *user : mapData.OriginalValue[i]->users())
3219 userVec.push_back(user);
3221 for (llvm::User *user : userVec) {
3222 if (
auto *insn = dyn_cast<llvm::Instruction>(user)) {
3223 if (insn->getFunction() == func) {
3224 auto *load = builder.CreateLoad(mapData.BasePointers[i]->getType(),
3225 mapData.BasePointers[i]);
3226 load->moveBefore(insn);
3227 user->replaceUsesOfWith(mapData.OriginalValue[i], load);
3274 static llvm::IRBuilderBase::InsertPoint
3276 llvm::Value *input, llvm::Value *&retVal,
3277 llvm::IRBuilderBase &builder,
3278 llvm::OpenMPIRBuilder &ompBuilder,
3280 llvm::IRBuilderBase::InsertPoint allocaIP,
3281 llvm::IRBuilderBase::InsertPoint codeGenIP) {
3282 builder.restoreIP(allocaIP);
3284 omp::VariableCaptureKind capture = omp::VariableCaptureKind::ByRef;
3287 for (
size_t i = 0; i < mapData.MapClause.size(); ++i)
3288 if (mapData.OriginalValue[i] == input) {
3289 auto mapOp = cast<omp::MapInfoOp>(mapData.MapClause[i]);
3291 mapOp.getMapCaptureType().value_or(omp::VariableCaptureKind::ByRef);
3296 unsigned int allocaAS = ompBuilder.M.getDataLayout().getAllocaAddrSpace();
3297 unsigned int defaultAS =
3298 ompBuilder.M.getDataLayout().getProgramAddressSpace();
3301 llvm::Value *v = builder.CreateAlloca(arg.getType(), allocaAS);
3303 if (allocaAS != defaultAS && arg.getType()->isPointerTy())
3304 v = builder.CreateAddrSpaceCast(v, builder.getPtrTy(defaultAS));
3306 builder.CreateStore(&arg, v);
3308 builder.restoreIP(codeGenIP);
3311 case omp::VariableCaptureKind::ByCopy: {
3315 case omp::VariableCaptureKind::ByRef: {
3316 retVal = builder.CreateAlignedLoad(
3318 ompBuilder.M.getDataLayout().getPrefTypeAlign(v->getType()));
3321 case omp::VariableCaptureKind::This:
3322 case omp::VariableCaptureKind::VLAType:
3323 assert(
false &&
"Currently unsupported capture kind");
3327 return builder.saveIP();
3330 static LogicalResult
3338 bool isTargetDevice = ompBuilder->Config.isTargetDevice();
3340 auto targetOp = cast<omp::TargetOp>(opInst);
3341 auto &targetRegion = targetOp.getRegion();
3345 cast<omp::BlockArgOpenMPOpInterface>(opInst).getMapBlockArgs();
3346 llvm::Function *llvmOutlinedFn =
nullptr;
3350 bool isOffloadEntry =
3351 isTargetDevice || !ompBuilder->Config.TargetTriples.empty();
3353 LogicalResult bodyGenStatus = success();
3354 using InsertPointTy = llvm::OpenMPIRBuilder::InsertPointTy;
3355 auto bodyCB = [&](InsertPointTy allocaIP,
3356 InsertPointTy codeGenIP) -> InsertPointTy {
3359 llvm::Function *llvmParentFn =
3361 llvmOutlinedFn = codeGenIP.getBlock()->getParent();
3362 assert(llvmParentFn && llvmOutlinedFn &&
3363 "Both parent and outlined functions must exist at this point");
3365 if (
auto attr = llvmParentFn->getFnAttribute(
"target-cpu");
3366 attr.isStringAttribute())
3367 llvmOutlinedFn->addFnAttr(attr);
3369 if (
auto attr = llvmParentFn->getFnAttribute(
"target-features");
3370 attr.isStringAttribute())
3371 llvmOutlinedFn->addFnAttr(attr);
3373 builder.restoreIP(codeGenIP);
3374 for (
auto [arg, mapOp] : llvm::zip_equal(mapBlockArgs, mapVars)) {
3375 auto mapInfoOp = cast<omp::MapInfoOp>(mapOp.getDefiningOp());
3376 llvm::Value *mapOpValue =
3377 moduleTranslation.
lookupValue(mapInfoOp.getVarPtr());
3378 moduleTranslation.
mapValue(arg, mapOpValue);
3383 if (!targetOp.getPrivateVars().empty()) {
3384 builder.restoreIP(allocaIP);
3387 std::optional<ArrayAttr> privateSyms = targetOp.getPrivateSyms();
3389 cast<omp::BlockArgOpenMPOpInterface>(opInst).getPrivateBlockArgs();
3391 for (
auto [privVar, privatizerNameAttr, privBlockArg] :
3392 llvm::zip_equal(privateVars, *privateSyms, privateBlockArgs)) {
3394 SymbolRefAttr privSym = cast<SymbolRefAttr>(privatizerNameAttr);
3395 omp::PrivateClauseOp privatizer =
findPrivatizer(&opInst, privSym);
3396 if (privatizer.getDataSharingType() ==
3397 omp::DataSharingClauseType::FirstPrivate ||
3398 !privatizer.getDeallocRegion().empty()) {
3399 opInst.
emitError(
"Translation of omp.target from MLIR to LLVMIR "
3400 "failed because translation of firstprivate and "
3401 " private allocatables is not supported yet");
3402 bodyGenStatus = failure();
3404 moduleTranslation.
mapValue(privatizer.getAllocMoldArg(),
3406 Region &allocRegion = privatizer.getAllocRegion();
3409 allocRegion,
"omp.targetop.privatizer", builder,
3410 moduleTranslation, &yieldedValues))) {
3412 "failed to inline `alloc` region of an `omp.private` "
3413 "op in the target region");
3414 bodyGenStatus = failure();
3416 assert(yieldedValues.size() == 1);
3417 moduleTranslation.
mapValue(privBlockArg, yieldedValues.front());
3420 builder.restoreIP(builder.saveIP());
3425 targetRegion,
"omp.target", builder, moduleTranslation, bodyGenStatus);
3426 builder.SetInsertPoint(exitBlock);
3427 return builder.saveIP();
3430 llvm::OpenMPIRBuilder::LocationDescription ompLoc(builder);
3431 StringRef parentName = parentFn.getName();
3433 llvm::TargetRegionEntryInfo entryInfo;
3438 int32_t defaultValTeams = -1;
3439 int32_t defaultValThreads = 0;
3441 llvm::OpenMPIRBuilder::InsertPointTy allocaIP =
3444 MapInfoData mapData;
3448 llvm::OpenMPIRBuilder::MapInfosTy combinedInfos;
3449 auto genMapInfoCB = [&](llvm::OpenMPIRBuilder::InsertPointTy codeGenIP)
3450 -> llvm::OpenMPIRBuilder::MapInfosTy & {
3451 builder.restoreIP(codeGenIP);
3452 genMapInfos(builder, moduleTranslation, dl, combinedInfos, mapData,
true);
3453 return combinedInfos;
3456 auto argAccessorCB = [&](llvm::Argument &arg, llvm::Value *input,
3457 llvm::Value *&retVal, InsertPointTy allocaIP,
3458 InsertPointTy codeGenIP) {
3464 if (!isTargetDevice) {
3465 retVal = cast<llvm::Value>(&arg);
3470 *ompBuilder, moduleTranslation,
3471 allocaIP, codeGenIP);
3475 for (
size_t i = 0; i < mapVars.size(); ++i) {
3482 if (!mapData.IsDeclareTarget[i] && !mapData.IsAMember[i])
3483 kernelInput.push_back(mapData.OriginalValue[i]);
3487 moduleTranslation, dds);
3490 ompLoc, isOffloadEntry, allocaIP, builder.saveIP(), entryInfo,
3491 defaultValTeams, defaultValThreads, kernelInput, genMapInfoCB, bodyCB,
3492 argAccessorCB, dds, targetOp.getNowait()));
3500 return bodyGenStatus;
3503 static LogicalResult
3513 if (FunctionOpInterface funcOp = dyn_cast<FunctionOpInterface>(op)) {
3514 if (
auto offloadMod = dyn_cast<omp::OffloadModuleInterface>(
3516 if (!offloadMod.getIsTargetDevice())
3519 omp::DeclareTargetDeviceType declareType =
3520 attribute.getDeviceType().getValue();
3522 if (declareType == omp::DeclareTargetDeviceType::host) {
3523 llvm::Function *llvmFunc =
3525 llvmFunc->dropAllReferences();
3526 llvmFunc->eraseFromParent();
3532 if (LLVM::GlobalOp gOp = dyn_cast<LLVM::GlobalOp>(op)) {
3533 llvm::Module *llvmModule = moduleTranslation.
getLLVMModule();
3534 if (
auto *gVal = llvmModule->getNamedValue(gOp.getSymName())) {
3536 bool isDeclaration = gOp.isDeclaration();
3537 bool isExternallyVisible =
3540 llvm::StringRef mangledName = gOp.getSymName();
3541 auto captureClause =
3547 std::vector<llvm::GlobalVariable *> generatedRefs;
3549 std::vector<llvm::Triple> targetTriple;
3550 auto targetTripleAttr = dyn_cast_or_null<mlir::StringAttr>(
3552 LLVM::LLVMDialect::getTargetTripleAttrName()));
3553 if (targetTripleAttr)
3554 targetTriple.emplace_back(targetTripleAttr.data());
3556 auto fileInfoCallBack = [&loc]() {
3557 std::string filename =
"";
3558 std::uint64_t lineNo = 0;
3561 filename = loc.getFilename().str();
3562 lineNo = loc.getLine();
3565 return std::pair<std::string, std::uint64_t>(llvm::StringRef(filename),
3569 ompBuilder->registerTargetGlobalVariable(
3570 captureClause, deviceClause, isDeclaration, isExternallyVisible,
3571 ompBuilder->getTargetEntryUniqueInfo(fileInfoCallBack), mangledName,
3572 generatedRefs,
false, targetTriple,
3574 gVal->getType(), gVal);
3576 if (ompBuilder->Config.isTargetDevice() &&
3577 (attribute.getCaptureClause().getValue() !=
3578 mlir::omp::DeclareTargetCaptureClause::to ||
3579 ompBuilder->Config.hasRequiresUnifiedSharedMemory())) {
3580 ompBuilder->getAddrOfDeclareTargetVar(
3581 captureClause, deviceClause, isDeclaration, isExternallyVisible,
3582 ompBuilder->getTargetEntryUniqueInfo(fileInfoCallBack), mangledName,
3583 generatedRefs,
false, targetTriple, gVal->getType(),
3601 if (
auto declareTargetIface =
3602 llvm::dyn_cast<mlir::omp::DeclareTargetInterface>(
3603 parentFn.getOperation()))
3604 if (declareTargetIface.isDeclareTarget() &&
3605 declareTargetIface.getDeclareTargetDeviceType() !=
3606 mlir::omp::DeclareTargetDeviceType::host)
3614 static LogicalResult
3621 .Case([&](omp::BarrierOp) {
3622 ompBuilder->createBarrier(builder.saveIP(), llvm::omp::OMPD_barrier);
3625 .Case([&](omp::TaskyieldOp) {
3626 ompBuilder->createTaskyield(builder.saveIP());
3629 .Case([&](omp::FlushOp) {
3638 ompBuilder->createFlush(builder.saveIP());
3641 .Case([&](omp::ParallelOp op) {
3644 .Case([&](omp::MaskedOp) {
3647 .Case([&](omp::MasterOp) {
3650 .Case([&](omp::CriticalOp) {
3653 .Case([&](omp::OrderedRegionOp) {
3656 .Case([&](omp::OrderedOp) {
3659 .Case([&](omp::WsloopOp) {
3662 .Case([&](omp::SimdOp) {
3665 .Case([&](omp::AtomicReadOp) {
3668 .Case([&](omp::AtomicWriteOp) {
3671 .Case([&](omp::AtomicUpdateOp op) {
3674 .Case([&](omp::AtomicCaptureOp op) {
3677 .Case([&](omp::SectionsOp) {
3680 .Case([&](omp::SingleOp op) {
3683 .Case([&](omp::TeamsOp op) {
3686 .Case([&](omp::TaskOp op) {
3689 .Case([&](omp::TaskgroupOp op) {
3692 .Case([&](omp::TaskwaitOp op) {
3695 .Case<omp::YieldOp, omp::TerminatorOp, omp::DeclareReductionOp,
3696 omp::CriticalDeclareOp>([](
auto op) {
3707 .Case([&](omp::ThreadprivateOp) {
3710 .Case<omp::TargetDataOp, omp::TargetEnterDataOp, omp::TargetExitDataOp,
3711 omp::TargetUpdateOp>([&](
auto op) {
3714 .Case([&](omp::TargetOp) {
3717 .Case<omp::MapInfoOp, omp::MapBoundsOp, omp::PrivateClauseOp>(
3725 return inst->
emitError(
"unsupported OpenMP operation: ")
3730 static LogicalResult
3736 static LogicalResult
3739 if (isa<omp::TargetOp>(op))
3741 if (isa<omp::TargetDataOp>(op))
3745 if (isa<omp::TargetOp>(oper)) {
3747 return WalkResult::interrupt();
3748 return WalkResult::skip();
3750 if (isa<omp::TargetDataOp>(oper)) {
3752 return WalkResult::interrupt();
3753 return WalkResult::skip();
3755 return WalkResult::advance();
3756 }).wasInterrupted();
3757 return failure(interrupted);
3764 class OpenMPDialectLLVMIRTranslationInterface
3785 LogicalResult OpenMPDialectLLVMIRTranslationInterface::amendOperation(
3791 .Case(
"omp.is_target_device",
3793 if (
auto deviceAttr = dyn_cast<BoolAttr>(attr)) {
3794 llvm::OpenMPIRBuilderConfig &config =
3796 config.setIsTargetDevice(deviceAttr.getValue());
3803 if (
auto gpuAttr = dyn_cast<BoolAttr>(attr)) {
3804 llvm::OpenMPIRBuilderConfig &config =
3806 config.setIsGPU(gpuAttr.getValue());
3811 .Case(
"omp.host_ir_filepath",
3813 if (
auto filepathAttr = dyn_cast<StringAttr>(attr)) {
3814 llvm::OpenMPIRBuilder *ompBuilder =
3816 ompBuilder->loadOffloadInfoMetadata(filepathAttr.getValue());
3823 if (
auto rtlAttr = dyn_cast<omp::FlagsAttr>(attr))
3827 .Case(
"omp.version",
3829 if (
auto versionAttr = dyn_cast<omp::VersionAttr>(attr)) {
3830 llvm::OpenMPIRBuilder *ompBuilder =
3832 ompBuilder->M.addModuleFlag(llvm::Module::Max,
"openmp",
3833 versionAttr.getVersion());
3838 .Case(
"omp.declare_target",
3840 if (
auto declareTargetAttr =
3841 dyn_cast<omp::DeclareTargetAttr>(attr))
3846 .Case(
"omp.requires",
3848 if (
auto requiresAttr = dyn_cast<omp::ClauseRequiresAttr>(attr)) {
3849 using Requires = omp::ClauseRequires;
3850 Requires flags = requiresAttr.getValue();
3851 llvm::OpenMPIRBuilderConfig &config =
3853 config.setHasRequiresReverseOffload(
3854 bitEnumContainsAll(flags, Requires::reverse_offload));
3855 config.setHasRequiresUnifiedAddress(
3856 bitEnumContainsAll(flags, Requires::unified_address));
3857 config.setHasRequiresUnifiedSharedMemory(
3858 bitEnumContainsAll(flags, Requires::unified_shared_memory));
3859 config.setHasRequiresDynamicAllocators(
3860 bitEnumContainsAll(flags, Requires::dynamic_allocators));
3865 .Case(
"omp.target_triples",
3867 if (
auto triplesAttr = dyn_cast<ArrayAttr>(attr)) {
3868 llvm::OpenMPIRBuilderConfig &config =
3870 config.TargetTriples.clear();
3871 config.TargetTriples.reserve(triplesAttr.size());
3872 for (
Attribute tripleAttr : triplesAttr) {
3873 if (
auto tripleStrAttr = dyn_cast<StringAttr>(tripleAttr))
3874 config.TargetTriples.emplace_back(tripleStrAttr.getValue());
3892 LogicalResult OpenMPDialectLLVMIRTranslationInterface::convertOperation(
3893 Operation *op, llvm::IRBuilderBase &builder,
3897 if (ompBuilder->Config.isTargetDevice()) {
3908 registry.
insert<omp::OpenMPDialect>();
3910 dialect->addInterfaces<OpenMPDialectLLVMIRTranslationInterface>();
static llvm::Value * getRefPtrIfDeclareTarget(mlir::Value value, LLVM::ModuleTranslation &moduleTranslation)
static void handleDeclareTargetMapVar(MapInfoData &mapData, LLVM::ModuleTranslation &moduleTranslation, llvm::IRBuilderBase &builder, llvm::Function *func)
static LogicalResult convertOmpAtomicUpdate(omp::AtomicUpdateOp &opInst, llvm::IRBuilderBase &builder, LLVM::ModuleTranslation &moduleTranslation)
Converts an OpenMP atomic update operation using OpenMPIRBuilder.
static llvm::omp::OrderKind convertOrderKind(std::optional< omp::ClauseOrderKind > o)
Convert Order attribute to llvm::omp::OrderKind.
static LogicalResult convertOmpAtomicCapture(omp::AtomicCaptureOp atomicCaptureOp, llvm::IRBuilderBase &builder, LLVM::ModuleTranslation &moduleTranslation)
static llvm::OffloadEntriesInfoManager::OMPTargetGlobalVarEntryKind convertToCaptureClauseKind(mlir::omp::DeclareTargetCaptureClause captureClause)
static omp::MapInfoOp getFirstOrLastMappedMemberPtr(omp::MapInfoOp mapInfo, bool first)
static LogicalResult convertOmpOrderedRegion(Operation &opInst, llvm::IRBuilderBase &builder, LLVM::ModuleTranslation &moduleTranslation)
Converts an OpenMP 'ordered_region' operation into LLVM IR using OpenMPIRBuilder.
static llvm::OpenMPIRBuilder::InsertPointTy findAllocaInsertPoint(llvm::IRBuilderBase &builder, const LLVM::ModuleTranslation &moduleTranslation)
Find the insertion point for allocas given the current insertion point for normal operations in the b...
static LogicalResult convertOmpAtomicWrite(Operation &opInst, llvm::IRBuilderBase &builder, LLVM::ModuleTranslation &moduleTranslation)
Converts an omp.atomic.write operation to LLVM IR.
static OwningAtomicReductionGen makeAtomicReductionGen(omp::DeclareReductionOp decl, llvm::IRBuilderBase &builder, LLVM::ModuleTranslation &moduleTranslation)
Create an OpenMPIRBuilder-compatible atomic reduction generator for the given reduction declaration.
static LogicalResult simdOpSupported(omp::SimdOp op)
static LogicalResult convertOmpWsloop(Operation &opInst, llvm::IRBuilderBase &builder, LLVM::ModuleTranslation &moduleTranslation)
Converts an OpenMP workshare loop into LLVM IR using OpenMPIRBuilder.
static LogicalResult convertOmpMaster(Operation &opInst, llvm::IRBuilderBase &builder, LLVM::ModuleTranslation &moduleTranslation)
Converts an OpenMP 'master' operation into LLVM IR using OpenMPIRBuilder.
static void collectPrivatizationDecls(omp::ParallelOp op, SmallVectorImpl< omp::PrivateClauseOp > &privatizations)
Populates privatizations with privatization declarations used for the given op.
static LogicalResult convertOmpTaskwaitOp(omp::TaskwaitOp twOp, llvm::IRBuilderBase &builder, LLVM::ModuleTranslation &moduleTranslation)
static void createAlteredByCaptureMap(MapInfoData &mapData, LLVM::ModuleTranslation &moduleTranslation, llvm::IRBuilderBase &builder)
static LogicalResult convertOmpTaskOp(omp::TaskOp taskOp, llvm::IRBuilderBase &builder, LLVM::ModuleTranslation &moduleTranslation)
Converts an OpenMP task construct into LLVM IR using OpenMPIRBuilder.
llvm::AtomicRMWInst::BinOp convertBinOpToAtomic(Operation &op)
Converts an LLVM dialect binary operation to the corresponding enum value for atomicrmw supported bin...
static llvm::AtomicOrdering convertAtomicOrdering(std::optional< omp::ClauseMemoryOrderKind > ao)
Convert an Atomic Ordering attribute to llvm::AtomicOrdering.
static LogicalResult convertOmpSections(Operation &opInst, llvm::IRBuilderBase &builder, LLVM::ModuleTranslation &moduleTranslation)
static LogicalResult convertOmpCritical(Operation &opInst, llvm::IRBuilderBase &builder, LLVM::ModuleTranslation &moduleTranslation)
Converts an OpenMP 'critical' operation into LLVM IR using OpenMPIRBuilder.
static LogicalResult convertOmpTargetData(Operation *op, llvm::IRBuilderBase &builder, LLVM::ModuleTranslation &moduleTranslation)
static LogicalResult allocAndInitializeReductionVars(OP op, ArrayRef< BlockArgument > reductionArgs, llvm::IRBuilderBase &builder, LLVM::ModuleTranslation &moduleTranslation, llvm::OpenMPIRBuilder::InsertPointTy &allocaIP, SmallVectorImpl< omp::DeclareReductionOp > &reductionDecls, SmallVectorImpl< llvm::Value * > &privateReductionVariables, DenseMap< Value, llvm::Value * > &reductionVariableMap, llvm::ArrayRef< bool > isByRef)
static LogicalResult convertOmpSimd(Operation &opInst, llvm::IRBuilderBase &builder, LLVM::ModuleTranslation &moduleTranslation)
Converts an OpenMP simd loop into LLVM IR using OpenMPIRBuilder.
static LogicalResult convertOmpOrdered(Operation &opInst, llvm::IRBuilderBase &builder, LLVM::ModuleTranslation &moduleTranslation)
Converts an OpenMP 'ordered' operation into LLVM IR using OpenMPIRBuilder.
static LogicalResult convertOmpMasked(Operation &opInst, llvm::IRBuilderBase &builder, LLVM::ModuleTranslation &moduleTranslation)
Converts an OpenMP 'masked' operation into LLVM IR using OpenMPIRBuilder.
static bool isTargetDeviceOp(Operation *op)
static LogicalResult inlineOmpRegionCleanup(llvm::SmallVectorImpl< Region * > &cleanupRegions, llvm::ArrayRef< llvm::Value * > privateVariables, LLVM::ModuleTranslation &moduleTranslation, llvm::IRBuilderBase &builder, StringRef regionName, bool shouldLoadCleanupRegionArg=true)
handling of DeclareReductionOp's cleanup region
static void mapInitializationArgs(T loop, LLVM::ModuleTranslation &moduleTranslation, SmallVectorImpl< omp::DeclareReductionOp > &reductionDecls, DenseMap< Value, llvm::Value * > &reductionVariableMap, unsigned i)
Map input arguments to reduction initialization region.
static int getMapDataMemberIdx(MapInfoData &mapData, omp::MapInfoOp memberOp)
static llvm::SmallString< 64 > getDeclareTargetRefPtrSuffix(LLVM::GlobalOp globalOp, llvm::OpenMPIRBuilder &ompBuilder)
static OwningReductionGen makeReductionGen(omp::DeclareReductionOp decl, llvm::IRBuilderBase &builder, LLVM::ModuleTranslation &moduleTranslation)
Create an OpenMPIRBuilder-compatible reduction generator for the given reduction declaration.
static LogicalResult convertOmpParallel(omp::ParallelOp opInst, llvm::IRBuilderBase &builder, LLVM::ModuleTranslation &moduleTranslation)
Converts the OpenMP parallel operation to LLVM IR.
static llvm::omp::OpenMPOffloadMappingFlags mapParentWithMembers(LLVM::ModuleTranslation &moduleTranslation, llvm::IRBuilderBase &builder, llvm::OpenMPIRBuilder &ompBuilder, DataLayout &dl, llvm::OpenMPIRBuilder::MapInfosTy &combinedInfo, MapInfoData &mapData, uint64_t mapDataIndex, bool isTargetParams)
static LogicalResult inlineConvertOmpRegions(Region ®ion, StringRef blockName, llvm::IRBuilderBase &builder, LLVM::ModuleTranslation &moduleTranslation, SmallVectorImpl< llvm::Value * > *continuationBlockArgs=nullptr)
Translates the blocks contained in the given region and appends them to at the current insertion poin...
static LogicalResult convertOmpThreadprivate(Operation &opInst, llvm::IRBuilderBase &builder, LLVM::ModuleTranslation &moduleTranslation)
Converts an OpenMP Threadprivate operation into LLVM IR using OpenMPIRBuilder.
static omp::PrivateClauseOp findPrivatizer(Operation *from, SymbolRefAttr symbolName)
Looks up from the operation from and returns the PrivateClauseOp with name symbolName.
static LogicalResult convertDeclareTargetAttr(Operation *op, mlir::omp::DeclareTargetAttr attribute, LLVM::ModuleTranslation &moduleTranslation)
static LogicalResult createReductionsAndCleanup(OP op, llvm::IRBuilderBase &builder, LLVM::ModuleTranslation &moduleTranslation, llvm::OpenMPIRBuilder::InsertPointTy &allocaIP, SmallVectorImpl< omp::DeclareReductionOp > &reductionDecls, ArrayRef< llvm::Value * > privateReductionVariables, ArrayRef< bool > isByRef)
LogicalResult convertFlagsAttr(Operation *op, mlir::omp::FlagsAttr attribute, LLVM::ModuleTranslation &moduleTranslation)
Lowers the FlagsAttr which is applied to the module on the device pass when offloading,...
uint64_t getArrayElementSizeInBits(LLVM::LLVMArrayType arrTy, DataLayout &dl)
std::vector< llvm::Value * > calculateBoundsOffset(LLVM::ModuleTranslation &moduleTranslation, llvm::IRBuilderBase &builder, bool isArrayTy, OperandRange bounds)
This function calculates the array/pointer offset for map data provided with bounds operations,...
static void processIndividualMap(MapInfoData &mapData, size_t mapDataIdx, llvm::OpenMPIRBuilder::MapInfosTy &combinedInfo, bool isTargetParams, int mapDataParentIdx=-1)
static LogicalResult allocReductionVars(T loop, ArrayRef< BlockArgument > reductionArgs, llvm::IRBuilderBase &builder, LLVM::ModuleTranslation &moduleTranslation, const llvm::OpenMPIRBuilder::InsertPointTy &allocaIP, SmallVectorImpl< omp::DeclareReductionOp > &reductionDecls, SmallVectorImpl< llvm::Value * > &privateReductionVariables, DenseMap< Value, llvm::Value * > &reductionVariableMap, SmallVectorImpl< DeferredStore > &deferredStores, llvm::ArrayRef< bool > isByRefs)
Allocate space for privatized reduction variables.
static ArrayRef< bool > getIsByRef(std::optional< ArrayRef< bool >> attr)
static bool targetOpSupported(Operation &opInst)
static void genMapInfos(llvm::IRBuilderBase &builder, LLVM::ModuleTranslation &moduleTranslation, DataLayout &dl, llvm::OpenMPIRBuilder::MapInfosTy &combinedInfo, MapInfoData &mapData, bool isTargetParams=false)
static llvm::IRBuilderBase::InsertPoint createDeviceArgumentAccessor(MapInfoData &mapData, llvm::Argument &arg, llvm::Value *input, llvm::Value *&retVal, llvm::IRBuilderBase &builder, llvm::OpenMPIRBuilder &ompBuilder, LLVM::ModuleTranslation &moduleTranslation, llvm::IRBuilderBase::InsertPoint allocaIP, llvm::IRBuilderBase::InsertPoint codeGenIP)
static llvm::omp::ProcBindKind getProcBindKind(omp::ClauseProcBindKind kind)
Convert ProcBindKind from MLIR-generated enum to LLVM enum.
static LogicalResult convertTargetOpsInNest(Operation *op, llvm::IRBuilderBase &builder, LLVM::ModuleTranslation &moduleTranslation)
static void processMapMembersWithParent(LLVM::ModuleTranslation &moduleTranslation, llvm::IRBuilderBase &builder, llvm::OpenMPIRBuilder &ompBuilder, DataLayout &dl, llvm::OpenMPIRBuilder::MapInfosTy &combinedInfo, MapInfoData &mapData, uint64_t mapDataIndex, llvm::omp::OpenMPOffloadMappingFlags memberOfFlag)
static LogicalResult convertOmpSingle(omp::SingleOp &singleOp, llvm::IRBuilderBase &builder, LLVM::ModuleTranslation &moduleTranslation)
Converts an OpenMP single construct into LLVM IR using OpenMPIRBuilder.
static void collectReductionDecls(T op, SmallVectorImpl< omp::DeclareReductionOp > &reductions)
Populates reductions with reduction declarations used in the given op.
static void processMapWithMembersOf(LLVM::ModuleTranslation &moduleTranslation, llvm::IRBuilderBase &builder, llvm::OpenMPIRBuilder &ompBuilder, DataLayout &dl, llvm::OpenMPIRBuilder::MapInfosTy &combinedInfo, MapInfoData &mapData, uint64_t mapDataIndex, bool isTargetParams)
static LogicalResult convertOmpTarget(Operation &opInst, llvm::IRBuilderBase &builder, LLVM::ModuleTranslation &moduleTranslation)
static llvm::OffloadEntriesInfoManager::OMPTargetDeviceClauseKind convertToDeviceClauseKind(mlir::omp::DeclareTargetDeviceType deviceClause)
static LogicalResult convertOmpAtomicRead(Operation &opInst, llvm::IRBuilderBase &builder, LLVM::ModuleTranslation &moduleTranslation)
Convert omp.atomic.read operation to LLVM IR.
static void collectMapDataFromMapOperands(MapInfoData &mapData, SmallVectorImpl< Value > &mapVars, LLVM::ModuleTranslation &moduleTranslation, DataLayout &dl, llvm::IRBuilderBase &builder, const ArrayRef< Value > &useDevPtrOperands={}, const ArrayRef< Value > &useDevAddrOperands={})
static bool getTargetEntryUniqueInfo(llvm::TargetRegionEntryInfo &targetInfo, omp::TargetOp targetOp, llvm::StringRef parentName="")
static void collectReductionInfo(T loop, llvm::IRBuilderBase &builder, LLVM::ModuleTranslation &moduleTranslation, SmallVectorImpl< omp::DeclareReductionOp > &reductionDecls, SmallVectorImpl< OwningReductionGen > &owningReductionGens, SmallVectorImpl< OwningAtomicReductionGen > &owningAtomicReductionGens, const ArrayRef< llvm::Value * > privateReductionVariables, SmallVectorImpl< llvm::OpenMPIRBuilder::ReductionInfo > &reductionInfos)
Collect reduction info.
static LogicalResult convertOmpTeams(omp::TeamsOp op, llvm::IRBuilderBase &builder, LLVM::ModuleTranslation &moduleTranslation)
static LogicalResult convertHostOrTargetOperation(Operation *op, llvm::IRBuilderBase &builder, LLVM::ModuleTranslation &moduleTranslation)
Given an OpenMP MLIR operation, create the corresponding LLVM IR (including OpenMP runtime calls).
static bool checkIfPointerMap(omp::MapInfoOp mapOp)
static LogicalResult convertTargetDeviceOp(Operation *op, llvm::IRBuilderBase &builder, LLVM::ModuleTranslation &moduleTranslation)
static void buildDependData(std::optional< ArrayAttr > dependKinds, OperandRange dependVars, LLVM::ModuleTranslation &moduleTranslation, SmallVectorImpl< llvm::OpenMPIRBuilder::DependData > &dds)
static llvm::BasicBlock * convertOmpOpRegions(Region ®ion, StringRef blockName, llvm::IRBuilderBase &builder, LLVM::ModuleTranslation &moduleTranslation, LogicalResult &bodyGenStatus, SmallVectorImpl< llvm::PHINode * > *continuationBlockPHIs=nullptr)
Converts the given region that appears within an OpenMP dialect operation to LLVM IR,...
static LogicalResult convertOmpTaskgroupOp(omp::TaskgroupOp tgOp, llvm::IRBuilderBase &builder, LLVM::ModuleTranslation &moduleTranslation)
Converts an OpenMP taskgroup construct into LLVM IR using OpenMPIRBuilder.
llvm::Value * getSizeInBytes(DataLayout &dl, const mlir::Type &type, Operation *clauseOp, llvm::Value *basePointer, llvm::Type *baseType, llvm::IRBuilderBase &builder, LLVM::ModuleTranslation &moduleTranslation)
static bool isDeclareTargetLink(mlir::Value value)
#define MLIR_DEFINE_EXPLICIT_INTERNAL_INLINE_TYPE_ID(CLASS_NAME)
Attributes are known-constant values of operations.
Block represents an ordered list of Operations.
BlockArgument getArgument(unsigned i)
unsigned getNumArguments()
Operation * getTerminator()
Get the terminator operation of this block.
The main mechanism for performing data layout queries.
llvm::TypeSize getTypeSizeInBits(Type t) const
Returns the size in bits of the given type in the current scope.
An attribute that represents a reference to a dense integer vector or tensor object.
The DialectRegistry maps a dialect namespace to a constructor for the matching dialect.
bool addExtension(TypeID extensionID, std::unique_ptr< DialectExtensionBase > extension)
Add the given extension to the registry.
Base class for dialect interfaces providing translation to LLVM IR.
virtual LogicalResult amendOperation(Operation *op, ArrayRef< llvm::Instruction * > instructions, NamedAttribute attribute, LLVM::ModuleTranslation &moduleTranslation) const
Hook for derived dialect interface to act on an operation that has dialect attributes from the derive...
virtual LogicalResult convertOperation(Operation *op, llvm::IRBuilderBase &builder, LLVM::ModuleTranslation &moduleTranslation) const
Hook for derived dialect interface to provide translation of the operations to LLVM IR.
LLVMTranslationDialectInterface(Dialect *dialect)
Concrete CRTP base class for ModuleTranslation stack frames.
Implementation class for module translation.
llvm::Value * lookupValue(Value value) const
Finds an LLVM IR value corresponding to the given MLIR value.
LogicalResult convertBlock(Block &bb, bool ignoreArguments, llvm::IRBuilderBase &builder)
Translates the contents of the given block to LLVM IR using this translator.
SmallVector< llvm::Value * > lookupValues(ValueRange values)
Looks up remapped a list of remapped values.
llvm::BasicBlock * lookupBlock(Block *block) const
Finds an LLVM IR basic block that corresponds to the given MLIR block.
SymbolTableCollection & symbolTable()
llvm::Type * convertType(Type type)
Converts the type from MLIR LLVM dialect to LLVM.
llvm::OpenMPIRBuilder * getOpenMPBuilder()
Returns the OpenMP IR builder associated with the LLVM IR module being constructed.
llvm::LLVMContext & getLLVMContext() const
Returns the LLVM context in which the IR is being constructed.
llvm::GlobalValue * lookupGlobal(Operation *op)
Finds an LLVM IR global value that corresponds to the given MLIR operation defining a global value.
llvm::Module * getLLVMModule()
Returns the LLVM module in which the IR is being constructed.
llvm::Function * lookupFunction(StringRef name) const
Finds an LLVM IR function by its name.
void mapBlock(Block *mlir, llvm::BasicBlock *llvm)
Stores the mapping between an MLIR block and LLVM IR basic block.
WalkResult stackWalk(llvm::function_ref< WalkResult(const T &)> callback) const
Calls callback for every ModuleTranslation stack frame of type T starting from the top of the stack.
void forgetMapping(Region ®ion)
Removes the mapping for blocks contained in the region and values defined in these blocks.
void mapValue(Value mlir, llvm::Value *llvm)
Stores the mapping between an MLIR value and its LLVM IR counterpart.
T findInstanceOf()
Return an instance of the given location type if one is nested under the current location.
MLIRContext is the top-level object for a collection of MLIR operations.
void appendDialectRegistry(const DialectRegistry ®istry)
Append the contents of the given dialect registry to the registry associated with this context.
NamedAttribute represents a combination of a name and an Attribute value.
StringAttr getName() const
Return the name of the attribute.
Attribute getValue() const
Return the value of the attribute.
This class implements the operand iterators for the Operation class.
Operation is the basic unit of execution within MLIR.
Value getOperand(unsigned idx)
OpResult getResult(unsigned idx)
Get the 'idx'th result of this operation.
std::enable_if_t< llvm::function_traits< std::decay_t< FnT > >::num_args==1, RetT > walk(FnT &&callback)
Walk the operation by calling the callback for each nested operation (including this one),...
Location getLoc()
The source location the operation was defined or derived from.
unsigned getNumOperands()
InFlightDiagnostic emitError(const Twine &message={})
Emit an error about fatal conditions with this operation, reporting up to any diagnostic handlers tha...
OpTy getParentOfType()
Return the closest surrounding parent operation that is of type 'OpTy'.
Region & getRegion(unsigned index)
Returns the region held by this operation at position 'index'.
OperationName getName()
The name of an operation is the key identifier for it.
operand_range getOperands()
Returns an iterator on the underlying Value's.
InFlightDiagnostic emitOpError(const Twine &message={})
Emit an error with the op name prefixed, like "'dim' op " which is convenient for verifiers.
This class contains a list of basic blocks and a link to the parent operation it is attached to.
BlockArgListType getArguments()
unsigned getNumArguments()
BlockListType & getBlocks()
@ Private
The symbol is private and may only be referenced by SymbolRefAttrs local to the operations within the...
Instances of the Type class are uniqued, have an immutable identifier and an optional mutable compone...
This class represents an instance of an SSA value in the MLIR system, representing a computable value...
Type getType() const
Return the type of this value.
Operation * getDefiningOp() const
If this value is the result of an operation, return the operation that defines it.
A utility result that is used to signal how to proceed with an ongoing walk:
bool wasInterrupted() const
Returns true if the walk was interrupted.
static WalkResult interrupt()
The OpAsmOpInterface, see OpAsmInterface.td for more details.
void connectPHINodes(Region ®ion, const ModuleTranslation &state)
For all blocks in the region that were converted to LLVM IR using the given ModuleTranslation,...
llvm::Constant * createMappingInformation(Location loc, llvm::OpenMPIRBuilder &builder)
Create a constant string representing the mapping information extracted from the MLIR location inform...
constexpr void enumerate(std::tuple< Tys... > &tuple, CallbackT &&callback)
Runtime
Potential runtimes for AMD GPU kernels.
Include the generated interface declarations.
SetVector< Block * > getBlocksSortedByDominance(Region ®ion)
Gets a list of blocks that is sorted according to dominance.
Type getType(OpFoldResult ofr)
Returns the int type of the integer in ofr.
void registerOpenMPDialectTranslation(DialectRegistry ®istry)
Register the OpenMP dialect and the translation from it to the LLVM IR in the given registry;.
auto get(MLIRContext *context, Ts &&...params)
Helper method that injects context only if needed, this helps unify some of the attribute constructio...
RAII object calling stackPush/stackPop on construction/destruction.