25 #include "llvm/ADT/SetVector.h"
26 #include "llvm/ADT/TypeSwitch.h"
27 #include "llvm/Frontend/OpenMP/OMPConstants.h"
28 #include "llvm/Frontend/OpenMP/OMPIRBuilder.h"
29 #include "llvm/IR/DebugInfoMetadata.h"
30 #include "llvm/IR/IRBuilder.h"
31 #include "llvm/Support/FileSystem.h"
32 #include "llvm/TargetParser/Triple.h"
33 #include "llvm/Transforms/Utils/ModuleUtils.h"
42 static llvm::omp::ScheduleKind
43 convertToScheduleKind(std::optional<omp::ClauseScheduleKind> schedKind) {
44 if (!schedKind.has_value())
45 return llvm::omp::OMP_SCHEDULE_Default;
46 switch (schedKind.value()) {
47 case omp::ClauseScheduleKind::Static:
48 return llvm::omp::OMP_SCHEDULE_Static;
49 case omp::ClauseScheduleKind::Dynamic:
50 return llvm::omp::OMP_SCHEDULE_Dynamic;
51 case omp::ClauseScheduleKind::Guided:
52 return llvm::omp::OMP_SCHEDULE_Guided;
53 case omp::ClauseScheduleKind::Auto:
54 return llvm::omp::OMP_SCHEDULE_Auto;
56 return llvm::omp::OMP_SCHEDULE_Runtime;
58 llvm_unreachable(
"unhandled schedule clause argument");
63 class OpenMPAllocaStackFrame
68 explicit OpenMPAllocaStackFrame(llvm::OpenMPIRBuilder::InsertPointTy allocaIP)
69 : allocaInsertPoint(allocaIP) {}
70 llvm::OpenMPIRBuilder::InsertPointTy allocaInsertPoint;
75 class OpenMPVarMappingStackFrame
77 OpenMPVarMappingStackFrame> {
81 explicit OpenMPVarMappingStackFrame(
91 static llvm::OpenMPIRBuilder::InsertPointTy
97 llvm::OpenMPIRBuilder::InsertPointTy allocaInsertPoint;
99 [&](
const OpenMPAllocaStackFrame &frame) {
100 allocaInsertPoint = frame.allocaInsertPoint;
104 return allocaInsertPoint;
113 if (builder.GetInsertBlock() ==
114 &builder.GetInsertBlock()->getParent()->getEntryBlock()) {
115 assert(builder.GetInsertPoint() == builder.GetInsertBlock()->end() &&
116 "Assuming end of basic block");
117 llvm::BasicBlock *entryBB = llvm::BasicBlock::Create(
118 builder.getContext(),
"entry", builder.GetInsertBlock()->getParent(),
119 builder.GetInsertBlock()->getNextNode());
120 builder.CreateBr(entryBB);
121 builder.SetInsertPoint(entryBB);
124 llvm::BasicBlock &funcEntryBlock =
125 builder.GetInsertBlock()->getParent()->getEntryBlock();
126 return llvm::OpenMPIRBuilder::InsertPointTy(
127 &funcEntryBlock, funcEntryBlock.getFirstInsertionPt());
136 Region ®ion, StringRef blockName, llvm::IRBuilderBase &builder,
139 llvm::BasicBlock *continuationBlock =
140 splitBB(builder,
true,
"omp.region.cont");
141 llvm::BasicBlock *sourceBlock = builder.GetInsertBlock();
143 llvm::LLVMContext &llvmContext = builder.getContext();
144 for (
Block &bb : region) {
145 llvm::BasicBlock *llvmBB = llvm::BasicBlock::Create(
146 llvmContext, blockName, builder.GetInsertBlock()->getParent(),
147 builder.GetInsertBlock()->getNextNode());
148 moduleTranslation.
mapBlock(&bb, llvmBB);
151 llvm::Instruction *sourceTerminator = sourceBlock->getTerminator();
157 bool operandsProcessed =
false;
158 unsigned numYields = 0;
160 if (omp::YieldOp yield = dyn_cast<omp::YieldOp>(bb.getTerminator())) {
161 if (!operandsProcessed) {
162 for (
unsigned i = 0, e = yield->getNumOperands(); i < e; ++i) {
163 continuationBlockPHITypes.push_back(
164 moduleTranslation.
convertType(yield->getOperand(i).getType()));
166 operandsProcessed =
true;
168 assert(continuationBlockPHITypes.size() == yield->getNumOperands() &&
169 "mismatching number of values yielded from the region");
170 for (
unsigned i = 0, e = yield->getNumOperands(); i < e; ++i) {
171 llvm::Type *operandType =
172 moduleTranslation.
convertType(yield->getOperand(i).getType());
174 assert(continuationBlockPHITypes[i] == operandType &&
175 "values of mismatching types yielded from the region");
184 if (!continuationBlockPHITypes.empty())
186 continuationBlockPHIs &&
187 "expected continuation block PHIs if converted regions yield values");
188 if (continuationBlockPHIs) {
189 llvm::IRBuilderBase::InsertPointGuard guard(builder);
190 continuationBlockPHIs->reserve(continuationBlockPHITypes.size());
191 builder.SetInsertPoint(continuationBlock, continuationBlock->begin());
192 for (llvm::Type *ty : continuationBlockPHITypes)
193 continuationBlockPHIs->push_back(builder.CreatePHI(ty, numYields));
199 for (
Block *bb : blocks) {
200 llvm::BasicBlock *llvmBB = moduleTranslation.
lookupBlock(bb);
203 if (bb->isEntryBlock()) {
204 assert(sourceTerminator->getNumSuccessors() == 1 &&
205 "provided entry block has multiple successors");
206 assert(sourceTerminator->getSuccessor(0) == continuationBlock &&
207 "ContinuationBlock is not the successor of the entry block");
208 sourceTerminator->setSuccessor(0, llvmBB);
211 llvm::IRBuilderBase::InsertPointGuard guard(builder);
213 moduleTranslation.
convertBlock(*bb, bb->isEntryBlock(), builder))) {
215 return continuationBlock;
225 Operation *terminator = bb->getTerminator();
226 if (isa<omp::TerminatorOp, omp::YieldOp>(terminator)) {
227 builder.CreateBr(continuationBlock);
229 for (
unsigned i = 0, e = terminator->
getNumOperands(); i < e; ++i)
230 (*continuationBlockPHIs)[i]->addIncoming(
244 return continuationBlock;
250 case omp::ClauseProcBindKind::Close:
251 return llvm::omp::ProcBindKind::OMP_PROC_BIND_close;
252 case omp::ClauseProcBindKind::Master:
253 return llvm::omp::ProcBindKind::OMP_PROC_BIND_master;
254 case omp::ClauseProcBindKind::Primary:
255 return llvm::omp::ProcBindKind::OMP_PROC_BIND_primary;
256 case omp::ClauseProcBindKind::Spread:
257 return llvm::omp::ProcBindKind::OMP_PROC_BIND_spread;
259 llvm_unreachable(
"Unknown ClauseProcBindKind kind");
266 using InsertPointTy = llvm::OpenMPIRBuilder::InsertPointTy;
271 auto bodyGenCB = [&](InsertPointTy allocaIP, InsertPointTy codeGenIP) {
273 auto ®ion = cast<omp::MasterOp>(opInst).getRegion();
274 builder.restoreIP(codeGenIP);
281 auto finiCB = [&](InsertPointTy codeGenIP) {};
283 llvm::OpenMPIRBuilder::LocationDescription ompLoc(builder);
285 ompLoc, bodyGenCB, finiCB));
293 using InsertPointTy = llvm::OpenMPIRBuilder::InsertPointTy;
294 auto criticalOp = cast<omp::CriticalOp>(opInst);
299 auto bodyGenCB = [&](InsertPointTy allocaIP, InsertPointTy codeGenIP) {
301 auto ®ion = cast<omp::CriticalOp>(opInst).getRegion();
302 builder.restoreIP(codeGenIP);
304 moduleTranslation, bodyGenStatus);
309 auto finiCB = [&](InsertPointTy codeGenIP) {};
311 llvm::OpenMPIRBuilder::LocationDescription ompLoc(builder);
312 llvm::LLVMContext &llvmContext = moduleTranslation.
getLLVMContext();
313 llvm::Constant *hint =
nullptr;
316 if (criticalOp.getNameAttr()) {
319 auto symbolRef = cast<SymbolRefAttr>(criticalOp.getNameAttr());
320 auto criticalDeclareOp =
321 SymbolTable::lookupNearestSymbolFrom<omp::CriticalDeclareOp>(criticalOp,
324 llvm::Type::getInt32Ty(llvmContext),
325 static_cast<int>(criticalDeclareOp.getHintVal()));
328 ompLoc, bodyGenCB, finiCB, criticalOp.getName().value_or(
""), hint));
336 template <
typename T>
337 static std::optional<omp::ReductionDeclareOp>
339 for (
unsigned i = 0, e = container.getNumReductionVars(); i < e; ++i) {
340 if (container.getReductionVars()[i] != reduction.getAccumulator())
343 SymbolRefAttr reductionSymbol =
344 cast<SymbolRefAttr>((*container.getReductions())[i]);
346 SymbolTable::lookupNearestSymbolFrom<omp::ReductionDeclareOp>(
347 container, reductionSymbol);
356 omp::ReductionOp reduction) {
357 std::optional<omp::ReductionDeclareOp> declareOp = std::nullopt;
360 while (!declareOp.has_value() && container) {
362 if (
auto par = dyn_cast<omp::ParallelOp>(*container)) {
364 }
else if (
auto loop = dyn_cast<omp::WsLoopOp>(*container)) {
374 assert(declareOp.has_value() &&
375 "reduction operation must be associated with a declaration");
381 template <
typename T>
385 std::optional<ArrayAttr> attr = loop.getReductions();
389 reductions.reserve(reductions.size() + loop.getNumReductionVars());
390 for (
auto symbolRef : attr->getAsRange<SymbolRefAttr>()) {
391 reductions.push_back(
392 SymbolTable::lookupNearestSymbolFrom<omp::ReductionDeclareOp>(
403 Region ®ion, StringRef blockName, llvm::IRBuilderBase &builder,
411 if (llvm::hasSingleElement(region)) {
412 moduleTranslation.
mapBlock(®ion.
front(), builder.GetInsertBlock());
414 region.
front(),
true, builder)))
418 if (continuationBlockArgs)
420 *continuationBlockArgs,
432 region, blockName, builder, moduleTranslation, bodyGenStatus, &phis);
433 if (
failed(bodyGenStatus))
435 if (continuationBlockArgs)
436 llvm::append_range(*continuationBlockArgs, phis);
437 builder.SetInsertPoint(continuationBlock,
438 continuationBlock->getFirstInsertionPt());
445 using OwningReductionGen = std::function<llvm::OpenMPIRBuilder::InsertPointTy(
446 llvm::OpenMPIRBuilder::InsertPointTy, llvm::Value *, llvm::Value *,
448 using OwningAtomicReductionGen =
449 std::function<llvm::OpenMPIRBuilder::InsertPointTy(
450 llvm::OpenMPIRBuilder::InsertPointTy, llvm::Type *, llvm::Value *,
457 static OwningReductionGen
463 OwningReductionGen gen =
464 [&, decl](llvm::OpenMPIRBuilder::InsertPointTy insertPoint,
465 llvm::Value *lhs, llvm::Value *rhs,
466 llvm::Value *&result)
mutable {
467 Region &reductionRegion = decl.getReductionRegion();
470 builder.restoreIP(insertPoint);
473 "omp.reduction.nonatomic.body",
474 builder, moduleTranslation, &phis)))
475 return llvm::OpenMPIRBuilder::InsertPointTy();
476 assert(phis.size() == 1);
478 return builder.saveIP();
487 static OwningAtomicReductionGen
489 llvm::IRBuilderBase &builder,
491 if (decl.getAtomicReductionRegion().empty())
492 return OwningAtomicReductionGen();
497 OwningAtomicReductionGen atomicGen =
498 [&, decl](llvm::OpenMPIRBuilder::InsertPointTy insertPoint, llvm::Type *,
499 llvm::Value *lhs, llvm::Value *rhs)
mutable {
500 Region &atomicRegion = decl.getAtomicReductionRegion();
503 builder.restoreIP(insertPoint);
506 "omp.reduction.atomic.body", builder,
507 moduleTranslation, &phis)))
508 return llvm::OpenMPIRBuilder::InsertPointTy();
509 assert(phis.empty());
510 return builder.saveIP();
519 auto orderedOp = cast<omp::OrderedOp>(opInst);
521 omp::ClauseDepend dependType = *orderedOp.getDependTypeVal();
522 bool isDependSource = dependType == omp::ClauseDepend::dependsource;
523 unsigned numLoops = *orderedOp.getNumLoopsVal();
525 moduleTranslation.
lookupValues(orderedOp.getDependVecVars());
527 size_t indexVecValues = 0;
528 while (indexVecValues < vecValues.size()) {
530 storeValues.reserve(numLoops);
531 for (
unsigned i = 0; i < numLoops; i++) {
532 storeValues.push_back(vecValues[indexVecValues]);
535 llvm::OpenMPIRBuilder::InsertPointTy allocaIP =
537 llvm::OpenMPIRBuilder::LocationDescription ompLoc(builder);
538 builder.restoreIP(moduleTranslation.
getOpenMPBuilder()->createOrderedDepend(
539 ompLoc, allocaIP, numLoops, storeValues,
".cnt.addr", isDependSource));
549 using InsertPointTy = llvm::OpenMPIRBuilder::InsertPointTy;
550 auto orderedRegionOp = cast<omp::OrderedRegionOp>(opInst);
553 if (orderedRegionOp.getSimd())
560 auto bodyGenCB = [&](InsertPointTy allocaIP, InsertPointTy codeGenIP) {
562 auto ®ion = cast<omp::OrderedRegionOp>(opInst).getRegion();
563 builder.restoreIP(codeGenIP);
565 moduleTranslation, bodyGenStatus);
570 auto finiCB = [&](InsertPointTy codeGenIP) {};
572 llvm::OpenMPIRBuilder::LocationDescription ompLoc(builder);
575 ompLoc, bodyGenCB, finiCB, !orderedRegionOp.getSimd()));
576 return bodyGenStatus;
582 using InsertPointTy = llvm::OpenMPIRBuilder::InsertPointTy;
583 using StorableBodyGenCallbackTy =
584 llvm::OpenMPIRBuilder::StorableBodyGenCallbackTy;
586 auto sectionsOp = cast<omp::SectionsOp>(opInst);
590 if (!sectionsOp.getReductionVars().empty() || sectionsOp.getReductions() ||
591 !sectionsOp.getAllocateVars().empty() ||
592 !sectionsOp.getAllocatorsVars().empty())
594 <<
"reduction and allocate clauses are not supported for sections "
601 auto sectionOp = dyn_cast<omp::SectionOp>(op);
605 Region ®ion = sectionOp.getRegion();
606 auto sectionCB = [®ion, &builder, &moduleTranslation, &bodyGenStatus](
607 InsertPointTy allocaIP, InsertPointTy codeGenIP) {
608 builder.restoreIP(codeGenIP);
610 moduleTranslation, bodyGenStatus);
612 sectionCBs.push_back(sectionCB);
618 if (sectionCBs.empty())
621 assert(isa<omp::SectionOp>(*sectionsOp.getRegion().op_begin()));
626 auto privCB = [&](InsertPointTy, InsertPointTy codeGenIP, llvm::Value &,
628 llvm::Value *&replacementValue) -> InsertPointTy {
629 replacementValue = &vPtr;
635 auto finiCB = [&](InsertPointTy codeGenIP) {};
637 llvm::OpenMPIRBuilder::InsertPointTy allocaIP =
639 llvm::OpenMPIRBuilder::LocationDescription ompLoc(builder);
641 ompLoc, allocaIP, sectionCBs, privCB, finiCB,
false,
642 sectionsOp.getNowait()));
643 return bodyGenStatus;
650 using InsertPointTy = llvm::OpenMPIRBuilder::InsertPointTy;
651 llvm::OpenMPIRBuilder::LocationDescription ompLoc(builder);
653 auto bodyCB = [&](InsertPointTy allocaIP, InsertPointTy codegenIP) {
654 builder.restoreIP(codegenIP);
656 moduleTranslation, bodyGenStatus);
658 auto finiCB = [&](InsertPointTy codeGenIP) {};
660 ompLoc, bodyCB, finiCB, singleOp.getNowait(),
nullptr));
661 return bodyGenStatus;
668 using InsertPointTy = llvm::OpenMPIRBuilder::InsertPointTy;
670 if (!op.getAllocatorsVars().empty() || op.getReductions())
671 return op.
emitError(
"unhandled clauses for translation to LLVM IR");
673 auto bodyCB = [&](InsertPointTy allocaIP, InsertPointTy codegenIP) {
675 moduleTranslation, allocaIP);
676 builder.restoreIP(codegenIP);
678 moduleTranslation, bodyGenStatus);
681 llvm::Value *numTeamsLower =
nullptr;
682 if (
Value numTeamsLowerVar = op.getNumTeamsLower())
683 numTeamsLower = moduleTranslation.
lookupValue(numTeamsLowerVar);
685 llvm::Value *numTeamsUpper =
nullptr;
686 if (
Value numTeamsUpperVar = op.getNumTeamsUpper())
687 numTeamsUpper = moduleTranslation.
lookupValue(numTeamsUpperVar);
689 llvm::Value *threadLimit =
nullptr;
690 if (
Value threadLimitVar = op.getThreadLimit())
691 threadLimit = moduleTranslation.
lookupValue(threadLimitVar);
693 llvm::Value *ifExpr =
nullptr;
694 if (
Value ifExprVar = op.getIfExpr())
697 llvm::OpenMPIRBuilder::LocationDescription ompLoc(builder);
699 ompLoc, bodyCB, numTeamsLower, numTeamsUpper, threadLimit, ifExpr));
700 return bodyGenStatus;
707 using InsertPointTy = llvm::OpenMPIRBuilder::InsertPointTy;
709 if (taskOp.getUntiedAttr() || taskOp.getMergeableAttr() ||
710 taskOp.getInReductions() || taskOp.getPriority() ||
711 !taskOp.getAllocateVars().empty()) {
712 return taskOp.emitError(
"unhandled clauses for translation to LLVM IR");
714 auto bodyCB = [&](InsertPointTy allocaIP, InsertPointTy codegenIP) {
718 moduleTranslation, allocaIP);
720 builder.restoreIP(codegenIP);
722 moduleTranslation, bodyGenStatus);
726 if (!taskOp.getDependVars().empty() && taskOp.getDepends()) {
728 llvm::zip(taskOp.getDependVars(), taskOp.getDepends()->getValue())) {
729 llvm::omp::RTLDependenceKindTy type;
731 cast<mlir::omp::ClauseTaskDependAttr>(std::get<1>(dep)).getValue()) {
732 case mlir::omp::ClauseTaskDepend::taskdependin:
733 type = llvm::omp::RTLDependenceKindTy::DepIn;
738 case mlir::omp::ClauseTaskDepend::taskdependout:
739 case mlir::omp::ClauseTaskDepend::taskdependinout:
740 type = llvm::omp::RTLDependenceKindTy::DepInOut;
743 llvm::Value *depVal = moduleTranslation.
lookupValue(std::get<0>(dep));
744 llvm::OpenMPIRBuilder::DependData dd(type, depVal->getType(), depVal);
745 dds.emplace_back(dd);
749 llvm::OpenMPIRBuilder::InsertPointTy allocaIP =
751 llvm::OpenMPIRBuilder::LocationDescription ompLoc(builder);
753 ompLoc, allocaIP, bodyCB, !taskOp.getUntied(),
754 moduleTranslation.
lookupValue(taskOp.getFinalExpr()),
755 moduleTranslation.
lookupValue(taskOp.getIfExpr()), dds));
756 return bodyGenStatus;
763 using InsertPointTy = llvm::OpenMPIRBuilder::InsertPointTy;
765 if (!tgOp.getTaskReductionVars().empty() || !tgOp.getAllocateVars().empty()) {
766 return tgOp.emitError(
"unhandled clauses for translation to LLVM IR");
768 auto bodyCB = [&](InsertPointTy allocaIP, InsertPointTy codegenIP) {
769 builder.restoreIP(codegenIP);
771 moduleTranslation, bodyGenStatus);
774 llvm::OpenMPIRBuilder::LocationDescription ompLoc(builder);
776 ompLoc, allocaIP, bodyCB));
777 return bodyGenStatus;
781 template <
typename T>
785 llvm::OpenMPIRBuilder::InsertPointTy &allocaIP,
789 unsigned numReductions = loop.getNumReductionVars();
790 privateReductionVariables.reserve(numReductions);
791 if (numReductions != 0) {
792 llvm::IRBuilderBase::InsertPointGuard guard(builder);
793 builder.restoreIP(allocaIP);
794 for (
unsigned i = 0; i < numReductions; ++i) {
795 llvm::Value *var = builder.CreateAlloca(
796 moduleTranslation.
convertType(reductionDecls[i].getType()));
797 privateReductionVariables.push_back(var);
798 reductionVariableMap.try_emplace(loop.getReductionVars()[i], var);
804 template <
typename T>
806 T loop, llvm::IRBuilderBase &builder,
813 unsigned numReductions = loop.getNumReductionVars();
815 for (
unsigned i = 0; i < numReductions; ++i) {
816 owningReductionGens.push_back(
818 owningAtomicReductionGens.push_back(
823 reductionInfos.reserve(numReductions);
824 for (
unsigned i = 0; i < numReductions; ++i) {
825 llvm::OpenMPIRBuilder::AtomicReductionGenTy atomicGen =
nullptr;
826 if (owningAtomicReductionGens[i])
827 atomicGen = owningAtomicReductionGens[i];
828 llvm::Value *variable =
829 moduleTranslation.
lookupValue(loop.getReductionVars()[i]);
830 reductionInfos.push_back(
831 {moduleTranslation.
convertType(reductionDecls[i].getType()), variable,
832 privateReductionVariables[i], owningReductionGens[i], atomicGen});
840 auto loop = cast<omp::WsLoopOp>(opInst);
842 if (loop.getLowerBound().empty())
847 loop.getScheduleVal().value_or(omp::ClauseScheduleKind::Static);
850 llvm::Value *step = moduleTranslation.
lookupValue(loop.getStep()[0]);
851 llvm::Type *ivType = step->getType();
852 llvm::Value *chunk =
nullptr;
853 if (loop.getScheduleChunkVar()) {
854 llvm::Value *chunkVar =
855 moduleTranslation.
lookupValue(loop.getScheduleChunkVar());
856 chunk = builder.CreateSExtOrTrunc(chunkVar, ivType);
861 llvm::OpenMPIRBuilder::InsertPointTy allocaIP =
867 privateReductionVariables, reductionVariableMap);
873 moduleTranslation, reductionVariableMap);
878 for (
unsigned i = 0; i < loop.getNumReductionVars(); ++i) {
881 "omp.reduction.neutral", builder,
882 moduleTranslation, &phis)))
884 assert(phis.size() == 1 &&
"expected one value to be yielded from the "
885 "reduction neutral element declaration region");
886 builder.CreateStore(phis[0], privateReductionVariables[i]);
890 llvm::OpenMPIRBuilder::LocationDescription ompLoc(builder);
898 auto bodyGen = [&](llvm::OpenMPIRBuilder::InsertPointTy ip, llvm::Value *iv) {
901 loop.getRegion().front().getArgument(loopInfos.size()), iv);
906 bodyInsertPoints.push_back(ip);
908 if (loopInfos.size() != loop.getNumLoops() - 1)
912 builder.restoreIP(ip);
914 moduleTranslation, bodyGenStatus);
922 for (
unsigned i = 0, e = loop.getNumLoops(); i < e; ++i) {
923 llvm::Value *lowerBound =
924 moduleTranslation.
lookupValue(loop.getLowerBound()[i]);
925 llvm::Value *upperBound =
926 moduleTranslation.
lookupValue(loop.getUpperBound()[i]);
927 llvm::Value *step = moduleTranslation.
lookupValue(loop.getStep()[i]);
932 llvm::OpenMPIRBuilder::LocationDescription loc = ompLoc;
933 llvm::OpenMPIRBuilder::InsertPointTy computeIP = ompLoc.IP;
935 loc = llvm::OpenMPIRBuilder::LocationDescription(bodyInsertPoints.back());
936 computeIP = loopInfos.front()->getPreheaderIP();
938 loopInfos.push_back(ompBuilder->createCanonicalLoop(
939 loc, bodyGen, lowerBound, upperBound, step,
940 true, loop.getInclusive(), computeIP));
942 if (
failed(bodyGenStatus))
948 llvm::IRBuilderBase::InsertPoint afterIP = loopInfos.front()->getAfterIP();
949 llvm::CanonicalLoopInfo *loopInfo =
950 ompBuilder->collapseLoops(ompLoc.DL, loopInfos, {});
955 bool isOrdered = loop.getOrderedVal().has_value();
956 std::optional<omp::ScheduleModifier> scheduleModifier =
957 loop.getScheduleModifier();
958 bool isSimd = loop.getSimdModifier();
960 ompBuilder->applyWorkshareLoop(
961 ompLoc.DL, loopInfo, allocaIP, !loop.getNowait(),
962 convertToScheduleKind(schedule), chunk, isSimd,
963 scheduleModifier == omp::ScheduleModifier::monotonic,
964 scheduleModifier == omp::ScheduleModifier::nonmonotonic, isOrdered);
970 builder.restoreIP(afterIP);
973 if (loop.getNumReductionVars() == 0)
982 owningReductionGens, owningAtomicReductionGens,
983 privateReductionVariables, reductionInfos);
988 llvm::UnreachableInst *tempTerminator = builder.CreateUnreachable();
989 builder.SetInsertPoint(tempTerminator);
990 llvm::OpenMPIRBuilder::InsertPointTy contInsertPoint =
991 ompBuilder->createReductions(builder.saveIP(), allocaIP, reductionInfos,
993 if (!contInsertPoint.getBlock())
994 return loop->emitOpError() <<
"failed to convert reductions";
995 auto nextInsertionPoint =
996 ompBuilder->createBarrier(contInsertPoint, llvm::omp::OMPD_for);
997 tempTerminator->eraseFromParent();
998 builder.restoreIP(nextInsertionPoint);
1007 using InsertPointTy = llvm::OpenMPIRBuilder::InsertPointTy;
1013 auto bodyGenCB = [&](InsertPointTy allocaIP, InsertPointTy codeGenIP) {
1022 reductionDecls, privateReductionVariables,
1023 reductionVariableMap);
1029 moduleTranslation, reductionVariableMap);
1032 builder.restoreIP(allocaIP);
1033 for (
unsigned i = 0; i < opInst.getNumReductionVars(); ++i) {
1036 reductionDecls[i].getInitializerRegion(),
"omp.reduction.neutral",
1037 builder, moduleTranslation, &phis)))
1039 assert(phis.size() == 1 &&
1040 "expected one value to be yielded from the "
1041 "reduction neutral element declaration region");
1042 builder.restoreIP(allocaIP);
1043 builder.CreateStore(phis[0], privateReductionVariables[i]);
1049 moduleTranslation, allocaIP);
1052 builder.restoreIP(codeGenIP);
1055 moduleTranslation, bodyGenStatus);
1058 if (opInst.getNumReductionVars() > 0) {
1064 owningReductionGens, owningAtomicReductionGens,
1065 privateReductionVariables, reductionInfos);
1068 builder.SetInsertPoint(regionBlock->getTerminator());
1071 llvm::UnreachableInst *tempTerminator = builder.CreateUnreachable();
1072 builder.SetInsertPoint(tempTerminator);
1074 llvm::OpenMPIRBuilder::InsertPointTy contInsertPoint =
1075 ompBuilder->createReductions(builder.saveIP(), allocaIP,
1076 reductionInfos,
false);
1077 if (!contInsertPoint.getBlock()) {
1078 bodyGenStatus = opInst->emitOpError() <<
"failed to convert reductions";
1082 tempTerminator->eraseFromParent();
1083 builder.restoreIP(contInsertPoint);
1090 auto privCB = [&](InsertPointTy allocaIP, InsertPointTy codeGenIP,
1091 llvm::Value &, llvm::Value &vPtr,
1092 llvm::Value *&replacementValue) -> InsertPointTy {
1093 replacementValue = &vPtr;
1100 auto finiCB = [&](InsertPointTy codeGenIP) {};
1102 llvm::Value *ifCond =
nullptr;
1103 if (
auto ifExprVar = opInst.getIfExprVar())
1104 ifCond = moduleTranslation.
lookupValue(ifExprVar);
1105 llvm::Value *numThreads =
nullptr;
1106 if (
auto numThreadsVar = opInst.getNumThreadsVar())
1107 numThreads = moduleTranslation.
lookupValue(numThreadsVar);
1108 auto pbKind = llvm::omp::OMP_PROC_BIND_default;
1109 if (
auto bind = opInst.getProcBindVal())
1112 bool isCancellable =
false;
1114 llvm::OpenMPIRBuilder::InsertPointTy allocaIP =
1116 llvm::OpenMPIRBuilder::LocationDescription ompLoc(builder);
1119 ompBuilder->createParallel(ompLoc, allocaIP, bodyGenCB, privCB, finiCB,
1120 ifCond, numThreads, pbKind, isCancellable));
1122 return bodyGenStatus;
1129 auto loop = cast<omp::SimdLoopOp>(opInst);
1131 llvm::OpenMPIRBuilder::LocationDescription ompLoc(builder);
1139 auto bodyGen = [&](llvm::OpenMPIRBuilder::InsertPointTy ip, llvm::Value *iv) {
1142 loop.getRegion().front().getArgument(loopInfos.size()), iv);
1147 bodyInsertPoints.push_back(ip);
1149 if (loopInfos.size() != loop.getNumLoops() - 1)
1153 builder.restoreIP(ip);
1155 moduleTranslation, bodyGenStatus);
1163 for (
unsigned i = 0, e = loop.getNumLoops(); i < e; ++i) {
1164 llvm::Value *lowerBound =
1165 moduleTranslation.
lookupValue(loop.getLowerBound()[i]);
1166 llvm::Value *upperBound =
1167 moduleTranslation.
lookupValue(loop.getUpperBound()[i]);
1168 llvm::Value *step = moduleTranslation.
lookupValue(loop.getStep()[i]);
1173 llvm::OpenMPIRBuilder::LocationDescription loc = ompLoc;
1174 llvm::OpenMPIRBuilder::InsertPointTy computeIP = ompLoc.IP;
1176 loc = llvm::OpenMPIRBuilder::LocationDescription(bodyInsertPoints.back(),
1178 computeIP = loopInfos.front()->getPreheaderIP();
1180 loopInfos.push_back(ompBuilder->createCanonicalLoop(
1181 loc, bodyGen, lowerBound, upperBound, step,
1182 true,
true, computeIP));
1184 if (
failed(bodyGenStatus))
1189 llvm::IRBuilderBase::InsertPoint afterIP = loopInfos.front()->getAfterIP();
1190 llvm::CanonicalLoopInfo *loopInfo =
1191 ompBuilder->collapseLoops(ompLoc.DL, loopInfos, {});
1193 llvm::ConstantInt *simdlen =
nullptr;
1194 if (std::optional<uint64_t> simdlenVar = loop.getSimdlen())
1195 simdlen = builder.getInt64(simdlenVar.value());
1197 llvm::ConstantInt *safelen =
nullptr;
1198 if (std::optional<uint64_t> safelenVar = loop.getSafelen())
1199 safelen = builder.getInt64(safelenVar.value());
1201 llvm::MapVector<llvm::Value *, llvm::Value *> alignedVars;
1202 ompBuilder->applySimd(
1203 loopInfo, alignedVars,
1204 loop.getIfExpr() ? moduleTranslation.
lookupValue(loop.getIfExpr())
1206 llvm::omp::OrderKind::OMP_ORDER_unknown, simdlen, safelen);
1208 builder.restoreIP(afterIP);
1213 static llvm::AtomicOrdering
1216 return llvm::AtomicOrdering::Monotonic;
1219 case omp::ClauseMemoryOrderKind::Seq_cst:
1220 return llvm::AtomicOrdering::SequentiallyConsistent;
1221 case omp::ClauseMemoryOrderKind::Acq_rel:
1222 return llvm::AtomicOrdering::AcquireRelease;
1223 case omp::ClauseMemoryOrderKind::Acquire:
1224 return llvm::AtomicOrdering::Acquire;
1225 case omp::ClauseMemoryOrderKind::Release:
1226 return llvm::AtomicOrdering::Release;
1227 case omp::ClauseMemoryOrderKind::Relaxed:
1228 return llvm::AtomicOrdering::Monotonic;
1230 llvm_unreachable(
"Unknown ClauseMemoryOrderKind kind");
1238 auto readOp = cast<omp::AtomicReadOp>(opInst);
1241 llvm::OpenMPIRBuilder::LocationDescription ompLoc(builder);
1244 llvm::Value *x = moduleTranslation.
lookupValue(readOp.getX());
1245 llvm::Value *v = moduleTranslation.
lookupValue(readOp.getV());
1247 llvm::Type *elementType =
1248 moduleTranslation.
convertType(readOp.getElementType());
1250 llvm::OpenMPIRBuilder::AtomicOpValue V = {v, elementType,
false,
false};
1251 llvm::OpenMPIRBuilder::AtomicOpValue X = {x, elementType,
false,
false};
1252 builder.restoreIP(ompBuilder->createAtomicRead(ompLoc, X, V, AO));
1260 auto writeOp = cast<omp::AtomicWriteOp>(opInst);
1263 llvm::OpenMPIRBuilder::LocationDescription ompLoc(builder);
1265 llvm::Value *expr = moduleTranslation.
lookupValue(writeOp.getExpr());
1266 llvm::Value *dest = moduleTranslation.
lookupValue(writeOp.getX());
1267 llvm::Type *ty = moduleTranslation.
convertType(writeOp.getExpr().getType());
1268 llvm::OpenMPIRBuilder::AtomicOpValue x = {dest, ty,
false,
1270 builder.restoreIP(ompBuilder->createAtomicWrite(ompLoc, x, expr, ao));
1278 .Case([&](LLVM::AddOp) {
return llvm::AtomicRMWInst::BinOp::Add; })
1279 .Case([&](LLVM::SubOp) {
return llvm::AtomicRMWInst::BinOp::Sub; })
1280 .Case([&](LLVM::AndOp) {
return llvm::AtomicRMWInst::BinOp::And; })
1281 .Case([&](LLVM::OrOp) {
return llvm::AtomicRMWInst::BinOp::Or; })
1282 .Case([&](LLVM::XOrOp) {
return llvm::AtomicRMWInst::BinOp::Xor; })
1283 .Case([&](LLVM::UMaxOp) {
return llvm::AtomicRMWInst::BinOp::UMax; })
1284 .Case([&](LLVM::UMinOp) {
return llvm::AtomicRMWInst::BinOp::UMin; })
1285 .Case([&](LLVM::FAddOp) {
return llvm::AtomicRMWInst::BinOp::FAdd; })
1286 .Case([&](LLVM::FSubOp) {
return llvm::AtomicRMWInst::BinOp::FSub; })
1287 .Default(llvm::AtomicRMWInst::BinOp::BAD_BINOP);
1293 llvm::IRBuilderBase &builder,
1298 auto &innerOpList = opInst.getRegion().front().getOperations();
1299 bool isRegionArgUsed{
false}, isXBinopExpr{
false};
1300 llvm::AtomicRMWInst::BinOp binop;
1304 for (
Operation &innerOp : innerOpList) {
1305 if (innerOp.getNumOperands() == 2) {
1307 if (!llvm::is_contained(innerOp.getOperands(),
1308 opInst.getRegion().getArgument(0)))
1310 isRegionArgUsed =
true;
1311 isXBinopExpr = innerOp.getNumOperands() > 0 &&
1312 innerOp.getOperand(0) == opInst.getRegion().getArgument(0);
1313 mlirExpr = (isXBinopExpr ? innerOp.getOperand(1) : innerOp.getOperand(0));
1317 if (!isRegionArgUsed)
1318 return opInst.emitError(
"no atomic update operation with region argument"
1319 " as operand found inside atomic.update region");
1321 llvm::Value *llvmExpr = moduleTranslation.
lookupValue(mlirExpr);
1322 llvm::Value *llvmX = moduleTranslation.
lookupValue(opInst.getX());
1323 llvm::Type *llvmXElementType = moduleTranslation.
convertType(
1324 opInst.getRegion().getArgument(0).getType());
1325 llvm::OpenMPIRBuilder::AtomicOpValue llvmAtomicX = {llvmX, llvmXElementType,
1329 llvm::AtomicOrdering atomicOrdering =
1334 auto updateFn = [&opInst, &moduleTranslation, &updateGenStatus](
1335 llvm::Value *atomicx,
1336 llvm::IRBuilder<> &builder) -> llvm::Value * {
1338 moduleTranslation.
mapValue(*opInst.getRegion().args_begin(), atomicx);
1339 moduleTranslation.
mapBlock(&bb, builder.GetInsertBlock());
1341 updateGenStatus = (opInst.emitError()
1342 <<
"unable to convert update operation to llvm IR");
1345 omp::YieldOp yieldop = dyn_cast<omp::YieldOp>(bb.
getTerminator());
1346 assert(yieldop && yieldop.getResults().size() == 1 &&
1347 "terminator must be omp.yield op and it must have exactly one "
1349 return moduleTranslation.
lookupValue(yieldop.getResults()[0]);
1354 llvm::OpenMPIRBuilder::LocationDescription ompLoc(builder);
1355 builder.restoreIP(ompBuilder->createAtomicUpdate(
1356 ompLoc, allocaIP, llvmAtomicX, llvmExpr, atomicOrdering, binop, updateFn,
1358 return updateGenStatus;
1363 llvm::IRBuilderBase &builder,
1367 bool isXBinopExpr =
false, isPostfixUpdate =
false;
1368 llvm::AtomicRMWInst::BinOp binop = llvm::AtomicRMWInst::BinOp::BAD_BINOP;
1370 omp::AtomicUpdateOp atomicUpdateOp = atomicCaptureOp.getAtomicUpdateOp();
1371 omp::AtomicWriteOp atomicWriteOp = atomicCaptureOp.getAtomicWriteOp();
1373 assert((atomicUpdateOp || atomicWriteOp) &&
1374 "internal op must be an atomic.update or atomic.write op");
1376 if (atomicWriteOp) {
1377 isPostfixUpdate =
true;
1378 mlirExpr = atomicWriteOp.getExpr();
1380 isPostfixUpdate = atomicCaptureOp.getSecondOp() ==
1381 atomicCaptureOp.getAtomicUpdateOp().getOperation();
1382 auto &innerOpList = atomicUpdateOp.getRegion().front().getOperations();
1383 bool isRegionArgUsed{
false};
1386 for (
Operation &innerOp : innerOpList) {
1387 if (innerOp.getNumOperands() == 2) {
1389 if (!llvm::is_contained(innerOp.getOperands(),
1390 atomicUpdateOp.getRegion().getArgument(0)))
1392 isRegionArgUsed =
true;
1394 innerOp.getNumOperands() > 0 &&
1395 innerOp.getOperand(0) == atomicUpdateOp.getRegion().getArgument(0);
1397 (isXBinopExpr ? innerOp.getOperand(1) : innerOp.getOperand(0));
1401 if (!isRegionArgUsed)
1402 return atomicUpdateOp.emitError(
1403 "no atomic update operation with region argument"
1404 " as operand found inside atomic.update region");
1407 llvm::Value *llvmExpr = moduleTranslation.
lookupValue(mlirExpr);
1408 llvm::Value *llvmX =
1409 moduleTranslation.
lookupValue(atomicCaptureOp.getAtomicReadOp().getX());
1410 llvm::Value *llvmV =
1411 moduleTranslation.
lookupValue(atomicCaptureOp.getAtomicReadOp().getV());
1412 llvm::Type *llvmXElementType = moduleTranslation.
convertType(
1413 atomicCaptureOp.getAtomicReadOp().getElementType());
1414 llvm::OpenMPIRBuilder::AtomicOpValue llvmAtomicX = {llvmX, llvmXElementType,
1417 llvm::OpenMPIRBuilder::AtomicOpValue llvmAtomicV = {llvmV, llvmXElementType,
1421 llvm::AtomicOrdering atomicOrdering =
1425 auto updateFn = [&](llvm::Value *atomicx,
1426 llvm::IRBuilder<> &builder) -> llvm::Value * {
1428 return moduleTranslation.
lookupValue(atomicWriteOp.getExpr());
1429 Block &bb = *atomicUpdateOp.getRegion().
begin();
1430 moduleTranslation.
mapValue(*atomicUpdateOp.getRegion().args_begin(),
1432 moduleTranslation.
mapBlock(&bb, builder.GetInsertBlock());
1434 updateGenStatus = (atomicUpdateOp.emitError()
1435 <<
"unable to convert update operation to llvm IR");
1438 omp::YieldOp yieldop = dyn_cast<omp::YieldOp>(bb.
getTerminator());
1439 assert(yieldop && yieldop.getResults().size() == 1 &&
1440 "terminator must be omp.yield op and it must have exactly one "
1442 return moduleTranslation.
lookupValue(yieldop.getResults()[0]);
1447 llvm::OpenMPIRBuilder::LocationDescription ompLoc(builder);
1448 builder.restoreIP(ompBuilder->createAtomicCapture(
1449 ompLoc, allocaIP, llvmAtomicX, llvmAtomicV, llvmExpr, atomicOrdering,
1450 binop, updateFn, atomicUpdateOp, isPostfixUpdate, isXBinopExpr));
1451 return updateGenStatus;
1460 llvm::IRBuilderBase &builder,
1463 omp::ReductionDeclareOp declaration;
1465 if (dyn_cast<omp::ParallelOp>(reductionParent) ||
1466 dyn_cast<omp::WsLoopOp>(reductionParent)) {
1469 llvm_unreachable(
"Unhandled reduction container");
1471 assert(declaration &&
"could not find reduction declaration");
1476 moduleTranslation.
stackWalk<OpenMPVarMappingStackFrame>(
1477 [&](
const OpenMPVarMappingStackFrame &frame) {
1478 if (frame.mapping.contains(reductionOp.getAccumulator())) {
1479 reductionVariableMap = &frame.mapping;
1484 assert(reductionVariableMap &&
"couldn't find private reduction variables");
1487 Region &reductionRegion = declaration.getReductionRegion();
1488 llvm::Value *privateReductionVar =
1489 reductionVariableMap->lookup(reductionOp.getAccumulator());
1490 llvm::Value *reductionVal = builder.CreateLoad(
1491 moduleTranslation.
convertType(reductionOp.getOperand().getType()),
1492 privateReductionVar);
1498 moduleTranslation.
lookupValue(reductionOp.getOperand()));
1502 builder, moduleTranslation, &phis)))
1504 assert(phis.size() == 1 &&
"expected one value to be yielded from "
1505 "the reduction body declaration region");
1506 builder.CreateStore(phis[0], privateReductionVar);
1515 llvm::OpenMPIRBuilder::LocationDescription ompLoc(builder);
1516 auto threadprivateOp = cast<omp::ThreadprivateOp>(opInst);
1518 Value symAddr = threadprivateOp.getSymAddr();
1520 if (!isa<LLVM::AddressOfOp>(symOp))
1521 return opInst.
emitError(
"Addressing symbol not found");
1522 LLVM::AddressOfOp addressOfOp = dyn_cast<LLVM::AddressOfOp>(symOp);
1524 LLVM::GlobalOp global =
1525 addressOfOp.getGlobal(moduleTranslation.
symbolTable());
1526 llvm::GlobalValue *globalValue = moduleTranslation.
lookupGlobal(global);
1527 llvm::Type *type = globalValue->getValueType();
1528 llvm::TypeSize typeSize =
1529 builder.GetInsertBlock()->getModule()->getDataLayout().getTypeStoreSize(
1531 llvm::ConstantInt *size = builder.getInt64(typeSize.getFixedValue());
1532 llvm::StringRef suffix = llvm::StringRef(
".cache", 6);
1533 std::string cacheName = (Twine(global.getSymName()).concat(suffix)).str();
1534 llvm::Value *callInst =
1536 ompLoc, globalValue, size, cacheName);
1541 static llvm::OffloadEntriesInfoManager::OMPTargetDeviceClauseKind
1543 switch (deviceClause) {
1544 case mlir::omp::DeclareTargetDeviceType::host:
1545 return llvm::OffloadEntriesInfoManager::OMPTargetDeviceClauseHost;
1547 case mlir::omp::DeclareTargetDeviceType::nohost:
1548 return llvm::OffloadEntriesInfoManager::OMPTargetDeviceClauseNoHost;
1550 case mlir::omp::DeclareTargetDeviceType::any:
1551 return llvm::OffloadEntriesInfoManager::OMPTargetDeviceClauseAny;
1554 llvm_unreachable(
"unhandled device clause");
1557 static llvm::OffloadEntriesInfoManager::OMPTargetGlobalVarEntryKind
1559 mlir::omp::DeclareTargetCaptureClause captureClasue) {
1560 switch (captureClasue) {
1561 case mlir::omp::DeclareTargetCaptureClause::to:
1562 return llvm::OffloadEntriesInfoManager::OMPTargetGlobalVarEntryTo;
1563 case mlir::omp::DeclareTargetCaptureClause::link:
1564 return llvm::OffloadEntriesInfoManager::OMPTargetGlobalVarEntryLink;
1565 case mlir::omp::DeclareTargetCaptureClause::enter:
1566 return llvm::OffloadEntriesInfoManager::OMPTargetGlobalVarEntryEnter;
1568 llvm_unreachable(
"unhandled capture clause");
1573 llvm::OpenMPIRBuilder &ompBuilder) {
1575 llvm::raw_svector_ostream os(suffix);
1577 auto loc = globalOp->getLoc()->findInstanceOf<FileLineColLoc>();
1578 auto fileInfoCallBack = [&loc]() {
1579 return std::pair<std::string, uint64_t>(
1580 llvm::StringRef(loc.getFilename()), loc.getLine());
1584 "_%x", ompBuilder.getTargetEntryUniqueInfo(fileInfoCallBack).FileID);
1586 os <<
"_decl_tgt_ref_ptr";
1594 static llvm::Value *
1601 if (
auto addressOfOp =
1602 llvm::dyn_cast_if_present<LLVM::AddressOfOp>(value.
getDefiningOp())) {
1603 if (
auto gOp = llvm::dyn_cast_or_null<LLVM::GlobalOp>(
1604 addressOfOp->getParentOfType<mlir::ModuleOp>().lookupSymbol(
1605 addressOfOp.getGlobalName()))) {
1607 if (
auto declareTargetGlobal =
1608 llvm::dyn_cast<mlir::omp::DeclareTargetInterface>(
1609 gOp.getOperation())) {
1613 if ((declareTargetGlobal.getDeclareTargetCaptureClause() ==
1614 mlir::omp::DeclareTargetCaptureClause::link) ||
1615 (declareTargetGlobal.getDeclareTargetCaptureClause() ==
1616 mlir::omp::DeclareTargetCaptureClause::to &&
1617 ompBuilder->Config.hasRequiresUnifiedSharedMemory())) {
1621 if (gOp.getSymName().contains(suffix))
1626 (gOp.getSymName().str() + suffix.str()).str());
1657 llvm::OpenMPIRBuilder::MapInfosTy::append(CurInfo);
1662 if (
auto nestedArrTy = llvm::dyn_cast_if_present<LLVM::LLVMArrayType>(
1663 arrTy.getElementType()))
1679 Operation *clauseOp, llvm::IRBuilderBase &builder,
1684 if (
auto arrTy = llvm::dyn_cast_if_present<LLVM::LLVMArrayType>(type)) {
1688 if (
auto memberClause =
1689 mlir::dyn_cast_if_present<mlir::omp::MapInfoOp>(clauseOp)) {
1694 if (!memberClause.getBounds().empty()) {
1695 llvm::Value *elementCount = builder.getInt64(1);
1696 for (
auto bounds : memberClause.getBounds()) {
1697 if (
auto boundOp = mlir::dyn_cast_if_present<mlir::omp::DataBoundsOp>(
1698 bounds.getDefiningOp())) {
1703 elementCount = builder.CreateMul(
1707 moduleTranslation.
lookupValue(boundOp.getUpperBound()),
1708 moduleTranslation.
lookupValue(boundOp.getLowerBound())),
1709 builder.getInt64(1)));
1718 return builder.CreateMul(elementCount,
1719 builder.getInt64(underlyingTypeSzInBits / 8));
1723 return builder.getInt64(underlyingTypeSzInBits / 8);
1730 llvm::IRBuilderBase &builder) {
1732 assert(mlir::isa<mlir::omp::MapInfoOp>(mapValue.getDefiningOp()) &&
1733 "missing map info operation or incorrect map info operation type");
1734 if (
auto mapOp = mlir::dyn_cast_if_present<mlir::omp::MapInfoOp>(
1735 mapValue.getDefiningOp())) {
1737 moduleTranslation.
lookupValue(mapOp.getVarPtr()));
1740 if (llvm::Value *refPtr =
1742 moduleTranslation)) {
1744 mapData.BasePointers.push_back(refPtr);
1747 mapData.BasePointers.push_back(mapData.
OriginalValue.back());
1750 mapData.Sizes.push_back(
getSizeInBytes(dl, mapOp.getVarType(), mapOp,
1751 builder, moduleTranslation));
1753 moduleTranslation.
convertType(mapOp.getVarType()));
1754 mapData.
MapClause.push_back(mapOp.getOperation());
1755 mapData.Types.push_back(
1756 llvm::omp::OpenMPOffloadMappingFlags(mapOp.getMapType().value()));
1759 mapData.DevicePointers.push_back(
1773 bool isTargetParams =
false) {
1776 auto fail = [&combinedInfo]() ->
void {
1777 combinedInfo.BasePointers.clear();
1778 combinedInfo.Pointers.clear();
1779 combinedInfo.DevicePointers.clear();
1780 combinedInfo.Sizes.clear();
1781 combinedInfo.Types.clear();
1782 combinedInfo.Names.clear();
1790 for (
size_t i = 0; i < mapData.
MapClause.size(); ++i) {
1794 auto mapFlag = mapData.Types[i];
1796 mapFlag |= llvm::omp::OpenMPOffloadMappingFlags::OMP_MAP_PTR_AND_OBJ;
1797 else if (isTargetParams)
1798 mapFlag |= llvm::omp::OpenMPOffloadMappingFlags::OMP_MAP_TARGET_PARAM;
1800 if (
auto mapInfoOp = dyn_cast<mlir::omp::MapInfoOp>(mapData.
MapClause[i]))
1801 if (mapInfoOp.getMapCaptureType().value() ==
1802 mlir::omp::VariableCaptureKind::ByCopy &&
1803 !mapInfoOp.getVarType().isa<LLVM::LLVMPointerType>())
1804 mapFlag |= llvm::omp::OpenMPOffloadMappingFlags::OMP_MAP_LITERAL;
1806 combinedInfo.BasePointers.emplace_back(mapData.BasePointers[i]);
1807 combinedInfo.Pointers.emplace_back(mapData.Pointers[i]);
1808 combinedInfo.DevicePointers.emplace_back(mapData.DevicePointers[i]);
1809 combinedInfo.Names.emplace_back(mapData.Names[i]);
1810 combinedInfo.Types.emplace_back(mapFlag);
1811 combinedInfo.Sizes.emplace_back(mapData.Sizes[i]);
1814 auto findMapInfo = [&combinedInfo](llvm::Value *val,
unsigned &index) {
1816 for (llvm::Value *basePtr : combinedInfo.BasePointers) {
1824 auto addDevInfos = [&, fail](
auto devOperands,
auto devOpType) ->
void {
1825 for (
const auto &devOp : devOperands) {
1827 if (!devOp.getType().template isa<LLVM::LLVMPointerType>())
1830 llvm::Value *mapOpValue = moduleTranslation.
lookupValue(devOp);
1834 if (findMapInfo(mapOpValue, infoIndex)) {
1835 combinedInfo.Types[infoIndex] |=
1836 llvm::omp::OpenMPOffloadMappingFlags::OMP_MAP_RETURN_PARAM;
1837 combinedInfo.DevicePointers[infoIndex] = devOpType;
1839 combinedInfo.BasePointers.emplace_back(mapOpValue);
1840 combinedInfo.Pointers.emplace_back(mapOpValue);
1841 combinedInfo.DevicePointers.emplace_back(devOpType);
1842 combinedInfo.Names.emplace_back(
1844 combinedInfo.Types.emplace_back(
1845 llvm::omp::OpenMPOffloadMappingFlags::OMP_MAP_RETURN_PARAM);
1846 combinedInfo.Sizes.emplace_back(builder.getInt64(0));
1851 addDevInfos(devPtrOperands, llvm::OpenMPIRBuilder::DeviceInfoTy::Pointer);
1852 addDevInfos(devAddrOperands, llvm::OpenMPIRBuilder::DeviceInfoTy::Address);
1858 llvm::Value *ifCond =
nullptr;
1859 int64_t deviceID = llvm::omp::OMP_DEVICEID_UNDEF;
1863 llvm::omp::RuntimeFunction RTLFn;
1870 .Case([&](omp::DataOp dataOp) {
1871 if (
auto ifExprVar = dataOp.getIfExpr())
1872 ifCond = moduleTranslation.
lookupValue(ifExprVar);
1874 if (
auto devId = dataOp.getDevice())
1876 dyn_cast<LLVM::ConstantOp>(devId.getDefiningOp()))
1877 if (
auto intAttr = dyn_cast<IntegerAttr>(constOp.getValue()))
1878 deviceID = intAttr.getInt();
1880 mapOperands = dataOp.getMapOperands();
1881 useDevPtrOperands = dataOp.getUseDevicePtr();
1882 useDevAddrOperands = dataOp.getUseDeviceAddr();
1885 .Case([&](omp::EnterDataOp enterDataOp) {
1886 if (enterDataOp.getNowait())
1889 if (
auto ifExprVar = enterDataOp.getIfExpr())
1890 ifCond = moduleTranslation.
lookupValue(ifExprVar);
1892 if (
auto devId = enterDataOp.getDevice())
1894 dyn_cast<LLVM::ConstantOp>(devId.getDefiningOp()))
1895 if (
auto intAttr = dyn_cast<IntegerAttr>(constOp.getValue()))
1896 deviceID = intAttr.getInt();
1897 RTLFn = llvm::omp::OMPRTL___tgt_target_data_begin_mapper;
1898 mapOperands = enterDataOp.getMapOperands();
1901 .Case([&](omp::ExitDataOp exitDataOp) {
1902 if (exitDataOp.getNowait())
1905 if (
auto ifExprVar = exitDataOp.getIfExpr())
1906 ifCond = moduleTranslation.
lookupValue(ifExprVar);
1908 if (
auto devId = exitDataOp.getDevice())
1910 dyn_cast<LLVM::ConstantOp>(devId.getDefiningOp()))
1911 if (
auto intAttr = dyn_cast<IntegerAttr>(constOp.getValue()))
1912 deviceID = intAttr.getInt();
1914 RTLFn = llvm::omp::OMPRTL___tgt_target_data_end_mapper;
1915 mapOperands = exitDataOp.getMapOperands();
1919 return op->
emitError(
"unsupported OpenMP operation: ")
1926 using InsertPointTy = llvm::OpenMPIRBuilder::InsertPointTy;
1936 builder.restoreIP(codeGenIP);
1937 if (
auto dataOp = dyn_cast<omp::DataOp>(op)) {
1938 genMapInfos(builder, moduleTranslation, DL, combinedInfo, mapData,
1939 useDevPtrOperands, useDevAddrOperands);
1941 genMapInfos(builder, moduleTranslation, DL, combinedInfo, mapData);
1943 return combinedInfo;
1946 llvm::OpenMPIRBuilder::TargetDataInfo info(
true,
1949 using BodyGenTy = llvm::OpenMPIRBuilder::BodyGenTy;
1951 auto bodyGenCB = [&](InsertPointTy codeGenIP, BodyGenTy bodyGenType) {
1952 assert(isa<omp::DataOp>(op) &&
"BodyGen requested for non DataOp");
1953 Region ®ion = cast<omp::DataOp>(op).getRegion();
1954 switch (bodyGenType) {
1955 case BodyGenTy::Priv:
1957 if (!info.DevicePtrInfoMap.empty()) {
1958 builder.restoreIP(codeGenIP);
1959 unsigned argIndex = 0;
1960 for (
auto &devPtrOp : useDevPtrOperands) {
1961 llvm::Value *mapOpValue = moduleTranslation.
lookupValue(devPtrOp);
1964 info.DevicePtrInfoMap[mapOpValue].second);
1968 for (
auto &devAddrOp : useDevAddrOperands) {
1969 llvm::Value *mapOpValue = moduleTranslation.
lookupValue(devAddrOp);
1971 auto *LI = builder.CreateLoad(
1972 builder.getPtrTy(), info.DevicePtrInfoMap[mapOpValue].second);
1973 moduleTranslation.
mapValue(arg, LI);
1978 builder, moduleTranslation);
1981 case BodyGenTy::DupNoPriv:
1983 case BodyGenTy::NoPriv:
1985 if (info.DevicePtrInfoMap.empty()) {
1986 builder.restoreIP(codeGenIP);
1988 builder, moduleTranslation);
1992 return builder.saveIP();
1995 llvm::OpenMPIRBuilder::LocationDescription ompLoc(builder);
1996 llvm::OpenMPIRBuilder::InsertPointTy allocaIP =
1998 if (isa<omp::DataOp>(op)) {
1999 builder.restoreIP(ompBuilder->createTargetData(
2000 ompLoc, allocaIP, builder.saveIP(), builder.getInt64(deviceID), ifCond,
2001 info, genMapInfoCB,
nullptr, bodyGenCB));
2003 builder.restoreIP(ompBuilder->createTargetData(
2004 ompLoc, allocaIP, builder.saveIP(), builder.getInt64(deviceID), ifCond,
2005 info, genMapInfoCB, &RTLFn));
2008 return bodyGenStatus;
2016 if (!cast<mlir::ModuleOp>(op))
2021 ompBuilder->createGlobalFlag(
2022 attribute.getDebugKind() ,
2023 "__omp_rtl_debug_kind");
2024 ompBuilder->createGlobalFlag(
2026 .getAssumeTeamsOversubscription()
2028 "__omp_rtl_assume_teams_oversubscription");
2029 ompBuilder->createGlobalFlag(
2031 .getAssumeThreadsOversubscription()
2033 "__omp_rtl_assume_threads_oversubscription");
2034 ompBuilder->createGlobalFlag(
2035 attribute.getAssumeNoThreadState() ,
2036 "__omp_rtl_assume_no_thread_state");
2037 ompBuilder->createGlobalFlag(
2039 .getAssumeNoNestedParallelism()
2041 "__omp_rtl_assume_no_nested_parallelism");
2042 ompBuilder->M.addModuleFlag(llvm::Module::Max,
"openmp-device",
2043 attribute.getOpenmpDeviceVersion());
2048 omp::TargetOp targetOp,
2049 llvm::StringRef parentName =
"") {
2050 auto fileLoc = targetOp.getLoc()->findInstanceOf<FileLineColLoc>();
2052 assert(fileLoc &&
"No file found from location");
2053 StringRef fileName = fileLoc.getFilename().getValue();
2055 llvm::sys::fs::UniqueID id;
2056 if (
auto ec = llvm::sys::fs::getUniqueID(fileName,
id)) {
2057 targetOp.emitError(
"Unable to get unique ID for file");
2061 uint64_t line = fileLoc.getLine();
2062 targetInfo = llvm::TargetRegionEntryInfo(parentName,
id.getDevice(),
2063 id.getFile(), line);
2068 auto targetOp = cast<omp::TargetOp>(opInst);
2069 if (targetOp.getIfExpr()) {
2070 opInst.
emitError(
"If clause not yet supported");
2074 if (targetOp.getDevice()) {
2075 opInst.
emitError(
"Device clause not yet supported");
2079 if (targetOp.getThreadLimit()) {
2080 opInst.
emitError(
"Thread limit clause not yet supported");
2084 if (targetOp.getNowait()) {
2085 opInst.
emitError(
"Nowait clause not yet supported");
2095 llvm::IRBuilderBase &builder) {
2096 for (
size_t i = 0; i < mapData.
MapClause.size(); ++i) {
2115 userVec.push_back(user);
2117 for (llvm::User *user : userVec) {
2118 if (
auto *insn = dyn_cast<llvm::Instruction>(user)) {
2119 auto *load = builder.CreateLoad(mapData.BasePointers[i]->getType(),
2120 mapData.BasePointers[i]);
2121 load->moveBefore(insn);
2168 static llvm::IRBuilderBase::InsertPoint
2170 llvm::Value *input, llvm::Value *&retVal,
2171 llvm::IRBuilderBase &builder,
2172 llvm::OpenMPIRBuilder &ompBuilder,
2174 llvm::IRBuilderBase::InsertPoint allocaIP,
2175 llvm::IRBuilderBase::InsertPoint codeGenIP) {
2176 builder.restoreIP(allocaIP);
2178 mlir::omp::VariableCaptureKind capture =
2179 mlir::omp::VariableCaptureKind::ByRef;
2182 for (
size_t i = 0; i < mapData.
MapClause.size(); ++i)
2184 if (
auto mapOp = mlir::dyn_cast_if_present<mlir::omp::MapInfoOp>(
2186 capture = mapOp.getMapCaptureType().value_or(
2187 mlir::omp::VariableCaptureKind::ByRef);
2193 unsigned int allocaAS = ompBuilder.M.getDataLayout().getAllocaAddrSpace();
2194 unsigned int defaultAS =
2195 ompBuilder.M.getDataLayout().getProgramAddressSpace();
2198 llvm::Value *v = builder.CreateAlloca(arg.getType(), allocaAS);
2200 if (allocaAS != defaultAS && arg.getType()->isPointerTy())
2201 v = builder.CreatePointerBitCastOrAddrSpaceCast(
2202 v, arg.getType()->getPointerTo(defaultAS));
2204 builder.CreateStore(&arg, v);
2206 builder.restoreIP(codeGenIP);
2209 case mlir::omp::VariableCaptureKind::ByCopy: {
2213 case mlir::omp::VariableCaptureKind::ByRef: {
2214 retVal = builder.CreateAlignedLoad(
2216 ompBuilder.M.getDataLayout().getPrefTypeAlign(v->getType()));
2219 case mlir::omp::VariableCaptureKind::This:
2220 case mlir::omp::VariableCaptureKind::VLAType:
2221 assert(
false &&
"Currently unsupported capture kind");
2225 return builder.saveIP();
2235 llvm::IRBuilderBase &builder) {
2236 for (
size_t i = 0; i < mapData.
MapClause.size(); ++i) {
2239 mlir::omp::VariableCaptureKind captureKind =
2240 mlir::omp::VariableCaptureKind::ByRef;
2242 if (
auto mapOp = mlir::dyn_cast_if_present<mlir::omp::MapInfoOp>(
2244 captureKind = mapOp.getMapCaptureType().value_or(
2245 mlir::omp::VariableCaptureKind::ByRef);
2248 switch (captureKind) {
2249 case mlir::omp::VariableCaptureKind::ByRef: {
2257 if (
auto mapOp = mlir::dyn_cast_if_present<mlir::omp::MapInfoOp>(
2259 if (!mapOp.getBounds().empty() && mapData.
BaseType[i]->isArrayTy()) {
2261 std::vector<llvm::Value *> idx =
2262 std::vector<llvm::Value *>{builder.getInt64(0)};
2263 for (
int i = mapOp.getBounds().size() - 1; i >= 0; --i) {
2265 mlir::dyn_cast_if_present<mlir::omp::DataBoundsOp>(
2266 mapOp.getBounds()[i].getDefiningOp())) {
2268 moduleTranslation.
lookupValue(boundOp.getLowerBound()));
2272 mapData.Pointers[i] = builder.CreateInBoundsGEP(
2273 mapData.
BaseType[i], mapData.Pointers[i], idx);
2277 case mlir::omp::VariableCaptureKind::ByCopy: {
2278 llvm::Type *type = mapData.
BaseType[i];
2280 if (mapData.Pointers[i]->getType()->isPointerTy())
2281 newV = builder.CreateLoad(type, mapData.Pointers[i]);
2283 newV = mapData.Pointers[i];
2285 if (!type->isPointerTy()) {
2286 auto curInsert = builder.saveIP();
2288 auto *memTempAlloc =
2289 builder.CreateAlloca(builder.getPtrTy(),
nullptr,
".casted");
2290 builder.restoreIP(curInsert);
2292 builder.CreateStore(newV, memTempAlloc);
2293 newV = builder.CreateLoad(builder.getPtrTy(), memTempAlloc);
2296 mapData.Pointers[i] = newV;
2297 mapData.BasePointers[i] = newV;
2299 case mlir::omp::VariableCaptureKind::This:
2300 case mlir::omp::VariableCaptureKind::VLAType:
2301 mapData.
MapClause[i]->emitOpError(
"Unhandled capture kind");
2315 auto targetOp = cast<omp::TargetOp>(opInst);
2316 auto &targetRegion = targetOp.getRegion();
2321 using InsertPointTy = llvm::OpenMPIRBuilder::InsertPointTy;
2322 auto bodyCB = [&](InsertPointTy allocaIP,
2323 InsertPointTy codeGenIP) -> InsertPointTy {
2324 builder.restoreIP(codeGenIP);
2325 unsigned argIndex = 0;
2326 for (
auto &mapOp : mapOperands) {
2328 mlir::dyn_cast<mlir::omp::MapInfoOp>(mapOp.getDefiningOp());
2329 llvm::Value *mapOpValue =
2330 moduleTranslation.
lookupValue(mapInfoOp.getVarPtr());
2331 const auto &arg = targetRegion.front().getArgument(argIndex);
2332 moduleTranslation.
mapValue(arg, mapOpValue);
2336 targetRegion,
"omp.target", builder, moduleTranslation, bodyGenStatus);
2337 builder.SetInsertPoint(exitBlock);
2338 return builder.saveIP();
2341 llvm::OpenMPIRBuilder::LocationDescription ompLoc(builder);
2342 StringRef parentName = opInst.
getParentOfType<LLVM::LLVMFuncOp>().getName();
2345 if (
auto earlyOutlineOp = llvm::dyn_cast<mlir::omp::EarlyOutliningInterface>(
2347 llvm::StringRef outlineParentName = earlyOutlineOp.getParentName();
2348 parentName = outlineParentName.empty() ? parentName : outlineParentName;
2351 llvm::TargetRegionEntryInfo entryInfo;
2356 int32_t defaultValTeams = -1;
2357 int32_t defaultValThreads = 0;
2359 llvm::OpenMPIRBuilder::InsertPointTy allocaIP =
2381 auto genMapInfoCB = [&](llvm::OpenMPIRBuilder::InsertPointTy codeGenIP)
2383 builder.restoreIP(codeGenIP);
2384 genMapInfos(builder, moduleTranslation, dl, combinedInfos, mapData, {}, {},
2386 return combinedInfos;
2389 auto argAccessorCB = [&](llvm::Argument &arg, llvm::Value *input,
2390 llvm::Value *&retVal, InsertPointTy allocaIP,
2391 InsertPointTy codeGenIP) {
2399 if (!ompBuilder->Config.isTargetDevice()) {
2400 retVal = cast<llvm::Value>(&arg);
2405 *ompBuilder, moduleTranslation,
2406 allocaIP, codeGenIP);
2410 for (
size_t i = 0; i < mapData.
MapClause.size(); ++i) {
2417 ompLoc, allocaIP, builder.saveIP(), entryInfo, defaultValTeams,
2418 defaultValThreads, kernelInput, genMapInfoCB, bodyCB, argAccessorCB));
2425 return bodyGenStatus;
2438 if (FunctionOpInterface funcOp = dyn_cast<FunctionOpInterface>(op)) {
2439 if (
auto offloadMod = dyn_cast<omp::OffloadModuleInterface>(
2441 if (!offloadMod.getIsTargetDevice())
2444 omp::DeclareTargetDeviceType declareType =
2445 attribute.getDeviceType().getValue();
2447 if (declareType == omp::DeclareTargetDeviceType::host) {
2448 llvm::Function *llvmFunc =
2450 llvmFunc->dropAllReferences();
2451 llvmFunc->eraseFromParent();
2457 if (LLVM::GlobalOp gOp = dyn_cast<LLVM::GlobalOp>(op)) {
2458 llvm::Module *llvmModule = moduleTranslation.
getLLVMModule();
2459 if (
auto *gVal = llvmModule->getNamedValue(gOp.getSymName())) {
2461 bool isDeclaration = gOp.isDeclaration();
2462 bool isExternallyVisible =
2465 llvm::StringRef mangledName = gOp.getSymName();
2466 auto captureClause =
2472 std::vector<llvm::GlobalVariable *> generatedRefs;
2474 std::vector<llvm::Triple> targetTriple;
2475 auto targetTripleAttr =
2477 ->getAttr(LLVM::LLVMDialect::getTargetTripleAttrName())
2478 .dyn_cast_or_null<mlir::StringAttr>();
2479 if (targetTripleAttr)
2480 targetTriple.emplace_back(targetTripleAttr.data());
2482 auto fileInfoCallBack = [&loc]() {
2483 std::string filename =
"";
2484 std::uint64_t lineNo = 0;
2487 filename = loc.getFilename().str();
2488 lineNo = loc.getLine();
2491 return std::pair<std::string, std::uint64_t>(llvm::StringRef(filename),
2495 ompBuilder->registerTargetGlobalVariable(
2496 captureClause, deviceClause, isDeclaration, isExternallyVisible,
2497 ompBuilder->getTargetEntryUniqueInfo(fileInfoCallBack), mangledName,
2498 generatedRefs,
false, targetTriple,
2500 gVal->getType(), gVal);
2502 if (ompBuilder->Config.isTargetDevice() &&
2503 (attribute.getCaptureClause().getValue() !=
2504 mlir::omp::DeclareTargetCaptureClause::to ||
2505 ompBuilder->Config.hasRequiresUnifiedSharedMemory())) {
2506 ompBuilder->getAddrOfDeclareTargetVar(
2507 captureClause, deviceClause, isDeclaration, isExternallyVisible,
2508 ompBuilder->getTargetEntryUniqueInfo(fileInfoCallBack), mangledName,
2509 generatedRefs,
false, targetTriple, gVal->getType(),
2530 ompBuilder->createRegisterRequires(ompBuilder->createPlatformSpecificName(
2531 {
"omp_offloading",
"requires_reg"}));
2535 llvm::appendToGlobalCtors(ompBuilder->M, regFn, 0);
2544 class OpenMPDialectLLVMIRTranslationInterface
2552 convertOperation(
Operation *op, llvm::IRBuilderBase &builder,
2564 LogicalResult OpenMPDialectLLVMIRTranslationInterface::amendOperation(
2569 .Case(
"omp.is_target_device",
2572 llvm::OpenMPIRBuilderConfig &config =
2574 config.setIsTargetDevice(deviceAttr.getValue());
2582 llvm::OpenMPIRBuilderConfig &config =
2584 config.setIsGPU(gpuAttr.getValue());
2589 .Case(
"omp.host_ir_filepath",
2591 if (
auto filepathAttr = attr.
dyn_cast<StringAttr>()) {
2592 llvm::OpenMPIRBuilder *ompBuilder =
2594 ompBuilder->loadOffloadInfoMetadata(filepathAttr.getValue());
2601 if (
auto rtlAttr = attr.
dyn_cast<omp::FlagsAttr>())
2605 .Case(
"omp.version",
2607 if (
auto versionAttr = attr.
dyn_cast<omp::VersionAttr>()) {
2608 llvm::OpenMPIRBuilder *ompBuilder =
2610 ompBuilder->M.addModuleFlag(llvm::Module::Max,
"openmp",
2611 versionAttr.getVersion());
2616 .Case(
"omp.declare_target",
2618 if (
auto declareTargetAttr =
2619 attr.
dyn_cast<omp::DeclareTargetAttr>())
2627 if (
auto requiresAttr = attr.
dyn_cast<omp::ClauseRequiresAttr>()) {
2628 using Requires = omp::ClauseRequires;
2629 Requires flags = requiresAttr.getValue();
2630 llvm::OpenMPIRBuilderConfig &config =
2632 config.setHasRequiresReverseOffload(
2633 bitEnumContainsAll(flags, Requires::reverse_offload));
2634 config.setHasRequiresUnifiedAddress(
2635 bitEnumContainsAll(flags, Requires::unified_address));
2636 config.setHasRequiresUnifiedSharedMemory(
2637 bitEnumContainsAll(flags, Requires::unified_shared_memory));
2638 config.setHasRequiresDynamicAllocators(
2639 bitEnumContainsAll(flags, Requires::dynamic_allocators));
2654 LogicalResult OpenMPDialectLLVMIRTranslationInterface::convertOperation(
2655 Operation *op, llvm::IRBuilderBase &builder,
2661 .Case([&](omp::BarrierOp) {
2662 ompBuilder->createBarrier(builder.saveIP(), llvm::omp::OMPD_barrier);
2665 .Case([&](omp::TaskwaitOp) {
2666 ompBuilder->createTaskwait(builder.saveIP());
2669 .Case([&](omp::TaskyieldOp) {
2670 ompBuilder->createTaskyield(builder.saveIP());
2673 .Case([&](omp::FlushOp) {
2682 ompBuilder->createFlush(builder.saveIP());
2685 .Case([&](omp::ParallelOp op) {
2688 .Case([&](omp::ReductionOp reductionOp) {
2691 .Case([&](omp::MasterOp) {
2694 .Case([&](omp::CriticalOp) {
2697 .Case([&](omp::OrderedRegionOp) {
2700 .Case([&](omp::OrderedOp) {
2703 .Case([&](omp::WsLoopOp) {
2706 .Case([&](omp::SimdLoopOp) {
2709 .Case([&](omp::AtomicReadOp) {
2712 .Case([&](omp::AtomicWriteOp) {
2715 .Case([&](omp::AtomicUpdateOp op) {
2718 .Case([&](omp::AtomicCaptureOp op) {
2721 .Case([&](omp::SectionsOp) {
2724 .Case([&](omp::SingleOp op) {
2727 .Case([&](omp::TeamsOp op) {
2730 .Case([&](omp::TaskOp op) {
2733 .Case([&](omp::TaskGroupOp op) {
2736 .Case<omp::YieldOp, omp::TerminatorOp, omp::ReductionDeclareOp,
2737 omp::CriticalDeclareOp>([](
auto op) {
2748 .Case([&](omp::ThreadprivateOp) {
2751 .Case<omp::DataOp, omp::EnterDataOp, omp::ExitDataOp>([&](
auto op) {
2754 .Case([&](omp::TargetOp) {
2757 .Case<omp::MapInfoOp, omp::DataBoundsOp>([&](
auto op) {
2764 return inst->
emitError(
"unsupported OpenMP operation: ")
2770 registry.
insert<omp::OpenMPDialect>();
2772 dialect->addInterfaces<OpenMPDialectLLVMIRTranslationInterface>();
static llvm::Value * getRefPtrIfDeclareTarget(mlir::Value value, LLVM::ModuleTranslation &moduleTranslation)
static LogicalResult convertOmpSimdLoop(Operation &opInst, llvm::IRBuilderBase &builder, LLVM::ModuleTranslation &moduleTranslation)
Converts an OpenMP simd loop into LLVM IR using OpenMPIRBuilder.
static LogicalResult convertOmpAtomicUpdate(omp::AtomicUpdateOp &opInst, llvm::IRBuilderBase &builder, LLVM::ModuleTranslation &moduleTranslation)
Converts an OpenMP atomic update operation using OpenMPIRBuilder.
static void allocReductionVars(T loop, llvm::IRBuilderBase &builder, LLVM::ModuleTranslation &moduleTranslation, llvm::OpenMPIRBuilder::InsertPointTy &allocaIP, SmallVector< omp::ReductionDeclareOp > &reductionDecls, SmallVector< llvm::Value * > &privateReductionVariables, DenseMap< Value, llvm::Value * > &reductionVariableMap)
Allocate space for privatized reduction variables.
static LogicalResult convertOmpAtomicCapture(omp::AtomicCaptureOp atomicCaptureOp, llvm::IRBuilderBase &builder, LLVM::ModuleTranslation &moduleTranslation)
static LogicalResult convertOmpOrderedRegion(Operation &opInst, llvm::IRBuilderBase &builder, LLVM::ModuleTranslation &moduleTranslation)
Converts an OpenMP 'ordered_region' operation into LLVM IR using OpenMPIRBuilder.
static llvm::OpenMPIRBuilder::InsertPointTy findAllocaInsertPoint(llvm::IRBuilderBase &builder, const LLVM::ModuleTranslation &moduleTranslation)
Find the insertion point for allocas given the current insertion point for normal operations in the b...
static LogicalResult convertOmpAtomicWrite(Operation &opInst, llvm::IRBuilderBase &builder, LLVM::ModuleTranslation &moduleTranslation)
Converts an omp.atomic.write operation to LLVM IR.
static LogicalResult convertOmpMaster(Operation &opInst, llvm::IRBuilderBase &builder, LLVM::ModuleTranslation &moduleTranslation)
Converts an OpenMP 'master' operation into LLVM IR using OpenMPIRBuilder.
static LogicalResult convertOmpTaskgroupOp(omp::TaskGroupOp tgOp, llvm::IRBuilderBase &builder, LLVM::ModuleTranslation &moduleTranslation)
Converts an OpenMP taskgroup construct into LLVM IR using OpenMPIRBuilder.
static OwningReductionGen makeReductionGen(omp::ReductionDeclareOp decl, llvm::IRBuilderBase &builder, LLVM::ModuleTranslation &moduleTranslation)
Create an OpenMPIRBuilder-compatible reduction generator for the given reduction declaration.
static LogicalResult convertOmpReductionOp(omp::ReductionOp reductionOp, llvm::IRBuilderBase &builder, LLVM::ModuleTranslation &moduleTranslation)
Converts an OpenMP reduction operation using OpenMPIRBuilder.
static void collectReductionInfo(T loop, llvm::IRBuilderBase &builder, LLVM::ModuleTranslation &moduleTranslation, SmallVector< omp::ReductionDeclareOp > &reductionDecls, SmallVector< OwningReductionGen > &owningReductionGens, SmallVector< OwningAtomicReductionGen > &owningAtomicReductionGens, const SmallVector< llvm::Value * > &privateReductionVariables, SmallVector< llvm::OpenMPIRBuilder::ReductionInfo > &reductionInfos)
Collect reduction info.
static void createAlteredByCaptureMap(MapInfoData &mapData, LLVM::ModuleTranslation &moduleTranslation, llvm::IRBuilderBase &builder)
static LogicalResult convertOmpTaskOp(omp::TaskOp taskOp, llvm::IRBuilderBase &builder, LLVM::ModuleTranslation &moduleTranslation)
Converts an OpenMP task construct into LLVM IR using OpenMPIRBuilder.
static std::optional< omp::ReductionDeclareOp > findReductionDeclInContainer(T container, omp::ReductionOp reduction)
Returns a reduction declaration that corresponds to the given reduction operation in the given contai...
llvm::AtomicRMWInst::BinOp convertBinOpToAtomic(Operation &op)
Converts an LLVM dialect binary operation to the corresponding enum value for atomicrmw supported bin...
static llvm::AtomicOrdering convertAtomicOrdering(std::optional< omp::ClauseMemoryOrderKind > ao)
Convert an Atomic Ordering attribute to llvm::AtomicOrdering.
static LogicalResult convertOmpSections(Operation &opInst, llvm::IRBuilderBase &builder, LLVM::ModuleTranslation &moduleTranslation)
static LogicalResult convertOmpCritical(Operation &opInst, llvm::IRBuilderBase &builder, LLVM::ModuleTranslation &moduleTranslation)
Converts an OpenMP 'critical' operation into LLVM IR using OpenMPIRBuilder.
static void genMapInfos(llvm::IRBuilderBase &builder, LLVM::ModuleTranslation &moduleTranslation, DataLayout &dl, llvm::OpenMPIRBuilder::MapInfosTy &combinedInfo, MapInfoData &mapData, const SmallVector< Value > &devPtrOperands={}, const SmallVector< Value > &devAddrOperands={}, bool isTargetParams=false)
static LogicalResult convertOmpTargetData(Operation *op, llvm::IRBuilderBase &builder, LLVM::ModuleTranslation &moduleTranslation)
static LogicalResult convertOmpOrdered(Operation &opInst, llvm::IRBuilderBase &builder, LLVM::ModuleTranslation &moduleTranslation)
Converts an OpenMP 'ordered' operation into LLVM IR using OpenMPIRBuilder.
void collectMapDataFromMapOperands(MapInfoData &mapData, llvm::SmallVectorImpl< Value > &mapOperands, LLVM::ModuleTranslation &moduleTranslation, DataLayout &dl, llvm::IRBuilderBase &builder)
static llvm::SmallString< 64 > getDeclareTargetRefPtrSuffix(LLVM::GlobalOp globalOp, llvm::OpenMPIRBuilder &ompBuilder)
static LogicalResult convertOmpParallel(omp::ParallelOp opInst, llvm::IRBuilderBase &builder, LLVM::ModuleTranslation &moduleTranslation)
Converts the OpenMP parallel operation to LLVM IR.
static LogicalResult inlineConvertOmpRegions(Region ®ion, StringRef blockName, llvm::IRBuilderBase &builder, LLVM::ModuleTranslation &moduleTranslation, SmallVectorImpl< llvm::Value * > *continuationBlockArgs=nullptr)
Translates the blocks contained in the given region and appends them to at the current insertion poin...
static LogicalResult convertOmpThreadprivate(Operation &opInst, llvm::IRBuilderBase &builder, LLVM::ModuleTranslation &moduleTranslation)
Converts an OpenMP Threadprivate operation into LLVM IR using OpenMPIRBuilder.
llvm::Value * getSizeInBytes(DataLayout &dl, const mlir::Type &type, Operation *clauseOp, llvm::IRBuilderBase &builder, LLVM::ModuleTranslation &moduleTranslation)
static LogicalResult convertDeclareTargetAttr(Operation *op, mlir::omp::DeclareTargetAttr attribute, LLVM::ModuleTranslation &moduleTranslation)
static OwningAtomicReductionGen makeAtomicReductionGen(omp::ReductionDeclareOp decl, llvm::IRBuilderBase &builder, LLVM::ModuleTranslation &moduleTranslation)
Create an OpenMPIRBuilder-compatible atomic reduction generator for the given reduction declaration.
LogicalResult convertFlagsAttr(Operation *op, mlir::omp::FlagsAttr attribute, LLVM::ModuleTranslation &moduleTranslation)
Lowers the FlagsAttr which is applied to the module on the device pass when offloading,...
static llvm::OffloadEntriesInfoManager::OMPTargetGlobalVarEntryKind convertToCaptureClauseKind(mlir::omp::DeclareTargetCaptureClause captureClasue)
uint64_t getArrayElementSizeInBits(LLVM::LLVMArrayType arrTy, DataLayout &dl)
static void handleDeclareTargetMapVar(MapInfoData &mapData, LLVM::ModuleTranslation &moduleTranslation, llvm::IRBuilderBase &builder)
static bool targetOpSupported(Operation &opInst)
static llvm::IRBuilderBase::InsertPoint createDeviceArgumentAccessor(MapInfoData &mapData, llvm::Argument &arg, llvm::Value *input, llvm::Value *&retVal, llvm::IRBuilderBase &builder, llvm::OpenMPIRBuilder &ompBuilder, LLVM::ModuleTranslation &moduleTranslation, llvm::IRBuilderBase::InsertPoint allocaIP, llvm::IRBuilderBase::InsertPoint codeGenIP)
static llvm::omp::ProcBindKind getProcBindKind(omp::ClauseProcBindKind kind)
Convert ProcBindKind from MLIR-generated enum to LLVM enum.
static LogicalResult convertOmpSingle(omp::SingleOp &singleOp, llvm::IRBuilderBase &builder, LLVM::ModuleTranslation &moduleTranslation)
Converts an OpenMP single construct into LLVM IR using OpenMPIRBuilder.
static LogicalResult convertOmpTarget(Operation &opInst, llvm::IRBuilderBase &builder, LLVM::ModuleTranslation &moduleTranslation)
static void collectReductionDecls(T loop, SmallVectorImpl< omp::ReductionDeclareOp > &reductions)
Populates reductions with reduction declarations used in the given loop.
static llvm::OffloadEntriesInfoManager::OMPTargetDeviceClauseKind convertToDeviceClauseKind(mlir::omp::DeclareTargetDeviceType deviceClause)
static LogicalResult convertRequiresAttr(Operation &op, omp::ClauseRequiresAttr requiresAttr, LLVM::ModuleTranslation &moduleTranslation)
Converts the module-level set of OpenMP requires clauses into LLVM IR using OpenMPIRBuilder.
static LogicalResult convertOmpWsLoop(Operation &opInst, llvm::IRBuilderBase &builder, LLVM::ModuleTranslation &moduleTranslation)
Converts an OpenMP workshare loop into LLVM IR using OpenMPIRBuilder.
static LogicalResult convertOmpAtomicRead(Operation &opInst, llvm::IRBuilderBase &builder, LLVM::ModuleTranslation &moduleTranslation)
Convert omp.atomic.read operation to LLVM IR.
static bool getTargetEntryUniqueInfo(llvm::TargetRegionEntryInfo &targetInfo, omp::TargetOp targetOp, llvm::StringRef parentName="")
static LogicalResult convertOmpTeams(omp::TeamsOp op, llvm::IRBuilderBase &builder, LLVM::ModuleTranslation &moduleTranslation)
static llvm::BasicBlock * convertOmpOpRegions(Region ®ion, StringRef blockName, llvm::IRBuilderBase &builder, LLVM::ModuleTranslation &moduleTranslation, LogicalResult &bodyGenStatus, SmallVectorImpl< llvm::PHINode * > *continuationBlockPHIs=nullptr)
Converts the given region that appears within an OpenMP dialect operation to LLVM IR,...
static omp::ReductionDeclareOp findReductionDecl(Operation &containerOp, omp::ReductionOp reduction)
Searches for a reduction in a provided region and the regions it is nested in.
#define MLIR_DEFINE_EXPLICIT_INTERNAL_INLINE_TYPE_ID(CLASS_NAME)
Attributes are known-constant values of operations.
Block represents an ordered list of Operations.
BlockArgument getArgument(unsigned i)
Operation * getTerminator()
Get the terminator operation of this block.
Special case of IntegerAttr to represent boolean integers, i.e., signless i1 integers.
The main mechanism for performing data layout queries.
llvm::TypeSize getTypeSizeInBits(Type t) const
Returns the size in bits of the given type in the current scope.
The DialectRegistry maps a dialect namespace to a constructor for the matching dialect.
void addExtension(std::unique_ptr< DialectExtensionBase > extension)
Add the given extension to the registry.
Base class for dialect interfaces providing translation to LLVM IR.
LLVMTranslationDialectInterface(Dialect *dialect)
Concrete CRTP base class for ModuleTranslation stack frames.
Implementation class for module translation.
LogicalResult convertBlock(Block &bb, bool ignoreArguments, llvm::IRBuilderBase &builder)
Translates the contents of the given block to LLVM IR using this translator.
llvm::Value * lookupValue(Value value) const
Finds an LLVM IR value corresponding to the given MLIR value.
SmallVector< llvm::Value * > lookupValues(ValueRange values)
Looks up remapped a list of remapped values.
llvm::BasicBlock * lookupBlock(Block *block) const
Finds an LLVM IR basic block that corresponds to the given MLIR block.
SymbolTableCollection & symbolTable()
llvm::Type * convertType(Type type)
Converts the type from MLIR LLVM dialect to LLVM.
llvm::OpenMPIRBuilder * getOpenMPBuilder()
Returns the OpenMP IR builder associated with the LLVM IR module being constructed.
llvm::LLVMContext & getLLVMContext() const
Returns the LLVM context in which the IR is being constructed.
llvm::GlobalValue * lookupGlobal(Operation *op)
Finds an LLVM IR global value that corresponds to the given MLIR operation defining a global value.
llvm::Module * getLLVMModule()
Returns the LLVM module in which the IR is being constructed.
llvm::Function * lookupFunction(StringRef name) const
Finds an LLVM IR function by its name.
void mapBlock(Block *mlir, llvm::BasicBlock *llvm)
Stores the mapping between an MLIR block and LLVM IR basic block.
WalkResult stackWalk(llvm::function_ref< WalkResult(const T &)> callback) const
Calls callback for every ModuleTranslation stack frame of type T starting from the top of the stack.
void forgetMapping(Region ®ion)
Removes the mapping for blocks contained in the region and values defined in these blocks.
void mapValue(Value mlir, llvm::Value *llvm)
Stores the mapping between an MLIR value and its LLVM IR counterpart.
T findInstanceOf()
Return an instance of the given location type if one is nested under the current location.
MLIRContext is the top-level object for a collection of MLIR operations.
void appendDialectRegistry(const DialectRegistry ®istry)
Append the contents of the given dialect registry to the registry associated with this context.
NamedAttribute represents a combination of a name and an Attribute value.
StringAttr getName() const
Return the name of the attribute.
Attribute getValue() const
Return the value of the attribute.
Operation is the basic unit of execution within MLIR.
Value getOperand(unsigned idx)
OpResult getResult(unsigned idx)
Get the 'idx'th result of this operation.
Location getLoc()
The source location the operation was defined or derived from.
unsigned getNumOperands()
Operation * getParentOp()
Returns the closest surrounding operation that contains this operation or nullptr if this is a top-le...
InFlightDiagnostic emitError(const Twine &message={})
Emit an error about fatal conditions with this operation, reporting up to any diagnostic handlers tha...
OpTy getParentOfType()
Return the closest surrounding parent operation that is of type 'OpTy'.
Region & getRegion(unsigned index)
Returns the region held by this operation at position 'index'.
OperationName getName()
The name of an operation is the key identifier for it.
operand_range getOperands()
Returns an iterator on the underlying Value's.
This class contains a list of basic blocks and a link to the parent operation it is attached to.
BlockListType & getBlocks()
@ Private
The symbol is private and may only be referenced by SymbolRefAttrs local to the operations within the...
Instances of the Type class are uniqued, have an immutable identifier and an optional mutable compone...
This class represents an instance of an SSA value in the MLIR system, representing a computable value...
Operation * getDefiningOp() const
If this value is the result of an operation, return the operation that defines it.
A utility result that is used to signal how to proceed with an ongoing walk:
static WalkResult advance()
bool wasInterrupted() const
Returns true if the walk was interrupted.
static WalkResult interrupt()
void connectPHINodes(Region ®ion, const ModuleTranslation &state)
For all blocks in the region that were converted to LLVM IR using the given ModuleTranslation,...
llvm::Constant * createMappingInformation(Location loc, llvm::OpenMPIRBuilder &builder)
Create a constant string representing the mapping information extracted from the MLIR location inform...
Runtime
Potential runtimes for AMD GPU kernels.
Include the generated interface declarations.
LogicalResult failure(bool isFailure=true)
Utility function to generate a LogicalResult.
SetVector< Block * > getTopologicallySortedBlocks(Region ®ion)
Get a topologically sorted list of blocks of the given region.
InFlightDiagnostic emitError(Location loc)
Utility method to emit an error message using this location.
LogicalResult success(bool isSuccess=true)
Utility function to generate a LogicalResult.
void registerOpenMPDialectTranslation(DialectRegistry ®istry)
Register the OpenMP dialect and the translation from it to the LLVM IR in the given registry;.
auto get(MLIRContext *context, Ts &&...params)
Helper method that injects context only if needed, this helps unify some of the attribute constructio...
bool failed(LogicalResult result)
Utility function that returns true if the provided LogicalResult corresponds to a failure value.
llvm::SmallVector< llvm::Value *, 4 > OriginalValue
llvm::SmallVector< bool, 4 > IsDeclareTarget
llvm::SmallVector< llvm::Type *, 4 > BaseType
void append(MapInfoData &CurInfo)
Append arrays in CurInfo.
llvm::SmallVector< mlir::Operation *, 4 > MapClause
RAII object calling stackPush/stackPop on construction/destruction.
This class represents an efficient way to signal success or failure.