25 #include "llvm/ADT/SetVector.h"
26 #include "llvm/ADT/TypeSwitch.h"
27 #include "llvm/Frontend/OpenMP/OMPConstants.h"
28 #include "llvm/Frontend/OpenMP/OMPIRBuilder.h"
29 #include "llvm/IR/DebugInfoMetadata.h"
30 #include "llvm/IR/IRBuilder.h"
31 #include "llvm/Support/FileSystem.h"
32 #include "llvm/TargetParser/Triple.h"
33 #include "llvm/Transforms/Utils/ModuleUtils.h"
42 static llvm::omp::ScheduleKind
43 convertToScheduleKind(std::optional<omp::ClauseScheduleKind> schedKind) {
44 if (!schedKind.has_value())
45 return llvm::omp::OMP_SCHEDULE_Default;
46 switch (schedKind.value()) {
47 case omp::ClauseScheduleKind::Static:
48 return llvm::omp::OMP_SCHEDULE_Static;
49 case omp::ClauseScheduleKind::Dynamic:
50 return llvm::omp::OMP_SCHEDULE_Dynamic;
51 case omp::ClauseScheduleKind::Guided:
52 return llvm::omp::OMP_SCHEDULE_Guided;
53 case omp::ClauseScheduleKind::Auto:
54 return llvm::omp::OMP_SCHEDULE_Auto;
56 return llvm::omp::OMP_SCHEDULE_Runtime;
58 llvm_unreachable(
"unhandled schedule clause argument");
63 class OpenMPAllocaStackFrame
68 explicit OpenMPAllocaStackFrame(llvm::OpenMPIRBuilder::InsertPointTy allocaIP)
69 : allocaInsertPoint(allocaIP) {}
70 llvm::OpenMPIRBuilder::InsertPointTy allocaInsertPoint;
75 class OpenMPVarMappingStackFrame
77 OpenMPVarMappingStackFrame> {
81 explicit OpenMPVarMappingStackFrame(
91 static llvm::OpenMPIRBuilder::InsertPointTy
97 llvm::OpenMPIRBuilder::InsertPointTy allocaInsertPoint;
99 [&](
const OpenMPAllocaStackFrame &frame) {
100 allocaInsertPoint = frame.allocaInsertPoint;
104 return allocaInsertPoint;
113 if (builder.GetInsertBlock() ==
114 &builder.GetInsertBlock()->getParent()->getEntryBlock()) {
115 assert(builder.GetInsertPoint() == builder.GetInsertBlock()->end() &&
116 "Assuming end of basic block");
117 llvm::BasicBlock *entryBB = llvm::BasicBlock::Create(
118 builder.getContext(),
"entry", builder.GetInsertBlock()->getParent(),
119 builder.GetInsertBlock()->getNextNode());
120 builder.CreateBr(entryBB);
121 builder.SetInsertPoint(entryBB);
124 llvm::BasicBlock &funcEntryBlock =
125 builder.GetInsertBlock()->getParent()->getEntryBlock();
126 return llvm::OpenMPIRBuilder::InsertPointTy(
127 &funcEntryBlock, funcEntryBlock.getFirstInsertionPt());
136 Region ®ion, StringRef blockName, llvm::IRBuilderBase &builder,
139 llvm::BasicBlock *continuationBlock =
140 splitBB(builder,
true,
"omp.region.cont");
141 llvm::BasicBlock *sourceBlock = builder.GetInsertBlock();
143 llvm::LLVMContext &llvmContext = builder.getContext();
144 for (
Block &bb : region) {
145 llvm::BasicBlock *llvmBB = llvm::BasicBlock::Create(
146 llvmContext, blockName, builder.GetInsertBlock()->getParent(),
147 builder.GetInsertBlock()->getNextNode());
148 moduleTranslation.
mapBlock(&bb, llvmBB);
151 llvm::Instruction *sourceTerminator = sourceBlock->getTerminator();
157 bool operandsProcessed =
false;
158 unsigned numYields = 0;
160 if (omp::YieldOp yield = dyn_cast<omp::YieldOp>(bb.getTerminator())) {
161 if (!operandsProcessed) {
162 for (
unsigned i = 0, e = yield->getNumOperands(); i < e; ++i) {
163 continuationBlockPHITypes.push_back(
164 moduleTranslation.
convertType(yield->getOperand(i).getType()));
166 operandsProcessed =
true;
168 assert(continuationBlockPHITypes.size() == yield->getNumOperands() &&
169 "mismatching number of values yielded from the region");
170 for (
unsigned i = 0, e = yield->getNumOperands(); i < e; ++i) {
171 llvm::Type *operandType =
172 moduleTranslation.
convertType(yield->getOperand(i).getType());
174 assert(continuationBlockPHITypes[i] == operandType &&
175 "values of mismatching types yielded from the region");
184 if (!continuationBlockPHITypes.empty())
186 continuationBlockPHIs &&
187 "expected continuation block PHIs if converted regions yield values");
188 if (continuationBlockPHIs) {
189 llvm::IRBuilderBase::InsertPointGuard guard(builder);
190 continuationBlockPHIs->reserve(continuationBlockPHITypes.size());
191 builder.SetInsertPoint(continuationBlock, continuationBlock->begin());
192 for (llvm::Type *ty : continuationBlockPHITypes)
193 continuationBlockPHIs->push_back(builder.CreatePHI(ty, numYields));
199 for (
Block *bb : blocks) {
200 llvm::BasicBlock *llvmBB = moduleTranslation.
lookupBlock(bb);
203 if (bb->isEntryBlock()) {
204 assert(sourceTerminator->getNumSuccessors() == 1 &&
205 "provided entry block has multiple successors");
206 assert(sourceTerminator->getSuccessor(0) == continuationBlock &&
207 "ContinuationBlock is not the successor of the entry block");
208 sourceTerminator->setSuccessor(0, llvmBB);
211 llvm::IRBuilderBase::InsertPointGuard guard(builder);
213 moduleTranslation.
convertBlock(*bb, bb->isEntryBlock(), builder))) {
215 return continuationBlock;
225 Operation *terminator = bb->getTerminator();
226 if (isa<omp::TerminatorOp, omp::YieldOp>(terminator)) {
227 builder.CreateBr(continuationBlock);
229 for (
unsigned i = 0, e = terminator->
getNumOperands(); i < e; ++i)
230 (*continuationBlockPHIs)[i]->addIncoming(
244 return continuationBlock;
250 case omp::ClauseProcBindKind::Close:
251 return llvm::omp::ProcBindKind::OMP_PROC_BIND_close;
252 case omp::ClauseProcBindKind::Master:
253 return llvm::omp::ProcBindKind::OMP_PROC_BIND_master;
254 case omp::ClauseProcBindKind::Primary:
255 return llvm::omp::ProcBindKind::OMP_PROC_BIND_primary;
256 case omp::ClauseProcBindKind::Spread:
257 return llvm::omp::ProcBindKind::OMP_PROC_BIND_spread;
259 llvm_unreachable(
"Unknown ClauseProcBindKind kind");
266 using InsertPointTy = llvm::OpenMPIRBuilder::InsertPointTy;
271 auto bodyGenCB = [&](InsertPointTy allocaIP, InsertPointTy codeGenIP) {
273 auto ®ion = cast<omp::MasterOp>(opInst).getRegion();
274 builder.restoreIP(codeGenIP);
281 auto finiCB = [&](InsertPointTy codeGenIP) {};
283 llvm::OpenMPIRBuilder::LocationDescription ompLoc(builder);
285 ompLoc, bodyGenCB, finiCB));
293 using InsertPointTy = llvm::OpenMPIRBuilder::InsertPointTy;
294 auto criticalOp = cast<omp::CriticalOp>(opInst);
299 auto bodyGenCB = [&](InsertPointTy allocaIP, InsertPointTy codeGenIP) {
301 auto ®ion = cast<omp::CriticalOp>(opInst).getRegion();
302 builder.restoreIP(codeGenIP);
304 moduleTranslation, bodyGenStatus);
309 auto finiCB = [&](InsertPointTy codeGenIP) {};
311 llvm::OpenMPIRBuilder::LocationDescription ompLoc(builder);
312 llvm::LLVMContext &llvmContext = moduleTranslation.
getLLVMContext();
313 llvm::Constant *hint =
nullptr;
316 if (criticalOp.getNameAttr()) {
319 auto symbolRef = cast<SymbolRefAttr>(criticalOp.getNameAttr());
320 auto criticalDeclareOp =
321 SymbolTable::lookupNearestSymbolFrom<omp::CriticalDeclareOp>(criticalOp,
324 llvm::Type::getInt32Ty(llvmContext),
325 static_cast<int>(criticalDeclareOp.getHintVal()));
328 ompLoc, bodyGenCB, finiCB, criticalOp.getName().value_or(
""), hint));
336 template <
typename T>
337 static std::optional<omp::DeclareReductionOp>
339 for (
unsigned i = 0, e = container.getNumReductionVars(); i < e; ++i) {
340 if (container.getReductionVars()[i] != reduction.getAccumulator())
343 SymbolRefAttr reductionSymbol =
344 cast<SymbolRefAttr>((*container.getReductions())[i]);
346 SymbolTable::lookupNearestSymbolFrom<omp::DeclareReductionOp>(
347 container, reductionSymbol);
356 omp::ReductionOp reduction) {
357 std::optional<omp::DeclareReductionOp> declareOp = std::nullopt;
360 while (!declareOp.has_value() && container) {
362 if (
auto par = dyn_cast<omp::ParallelOp>(*container)) {
364 }
else if (
auto loop = dyn_cast<omp::WsloopOp>(*container)) {
374 assert(declareOp.has_value() &&
375 "reduction operation must be associated with a declaration");
381 template <
typename T>
385 std::optional<ArrayAttr> attr = loop.getReductions();
389 reductions.reserve(reductions.size() + loop.getNumReductionVars());
390 for (
auto symbolRef : attr->getAsRange<SymbolRefAttr>()) {
391 reductions.push_back(
392 SymbolTable::lookupNearestSymbolFrom<omp::DeclareReductionOp>(
403 Region ®ion, StringRef blockName, llvm::IRBuilderBase &builder,
411 if (llvm::hasSingleElement(region)) {
412 llvm::Instruction *potentialTerminator =
413 builder.GetInsertBlock()->empty() ? nullptr
414 : &builder.GetInsertBlock()->back();
416 if (potentialTerminator && potentialTerminator->isTerminator())
417 potentialTerminator->removeFromParent();
418 moduleTranslation.
mapBlock(®ion.
front(), builder.GetInsertBlock());
421 region.
front(),
true, builder)))
425 if (continuationBlockArgs)
427 *continuationBlockArgs,
434 if (potentialTerminator && potentialTerminator->isTerminator())
435 potentialTerminator->insertAfter(&builder.GetInsertBlock()->back());
443 region, blockName, builder, moduleTranslation, bodyGenStatus, &phis);
444 if (
failed(bodyGenStatus))
446 if (continuationBlockArgs)
447 llvm::append_range(*continuationBlockArgs, phis);
448 builder.SetInsertPoint(continuationBlock,
449 continuationBlock->getFirstInsertionPt());
456 using OwningReductionGen = std::function<llvm::OpenMPIRBuilder::InsertPointTy(
457 llvm::OpenMPIRBuilder::InsertPointTy, llvm::Value *, llvm::Value *,
459 using OwningAtomicReductionGen =
460 std::function<llvm::OpenMPIRBuilder::InsertPointTy(
461 llvm::OpenMPIRBuilder::InsertPointTy, llvm::Type *, llvm::Value *,
468 static OwningReductionGen
474 OwningReductionGen gen =
475 [&, decl](llvm::OpenMPIRBuilder::InsertPointTy insertPoint,
476 llvm::Value *lhs, llvm::Value *rhs,
477 llvm::Value *&result)
mutable {
478 Region &reductionRegion = decl.getReductionRegion();
481 builder.restoreIP(insertPoint);
484 "omp.reduction.nonatomic.body",
485 builder, moduleTranslation, &phis)))
486 return llvm::OpenMPIRBuilder::InsertPointTy();
487 assert(phis.size() == 1);
489 return builder.saveIP();
498 static OwningAtomicReductionGen
500 llvm::IRBuilderBase &builder,
502 if (decl.getAtomicReductionRegion().empty())
503 return OwningAtomicReductionGen();
508 OwningAtomicReductionGen atomicGen =
509 [&, decl](llvm::OpenMPIRBuilder::InsertPointTy insertPoint, llvm::Type *,
510 llvm::Value *lhs, llvm::Value *rhs)
mutable {
511 Region &atomicRegion = decl.getAtomicReductionRegion();
514 builder.restoreIP(insertPoint);
517 "omp.reduction.atomic.body", builder,
518 moduleTranslation, &phis)))
519 return llvm::OpenMPIRBuilder::InsertPointTy();
520 assert(phis.empty());
521 return builder.saveIP();
530 auto orderedOp = cast<omp::OrderedOp>(opInst);
532 omp::ClauseDepend dependType = *orderedOp.getDependTypeVal();
533 bool isDependSource = dependType == omp::ClauseDepend::dependsource;
534 unsigned numLoops = *orderedOp.getNumLoopsVal();
536 moduleTranslation.
lookupValues(orderedOp.getDependVecVars());
538 size_t indexVecValues = 0;
539 while (indexVecValues < vecValues.size()) {
541 storeValues.reserve(numLoops);
542 for (
unsigned i = 0; i < numLoops; i++) {
543 storeValues.push_back(vecValues[indexVecValues]);
546 llvm::OpenMPIRBuilder::InsertPointTy allocaIP =
548 llvm::OpenMPIRBuilder::LocationDescription ompLoc(builder);
549 builder.restoreIP(moduleTranslation.
getOpenMPBuilder()->createOrderedDepend(
550 ompLoc, allocaIP, numLoops, storeValues,
".cnt.addr", isDependSource));
560 using InsertPointTy = llvm::OpenMPIRBuilder::InsertPointTy;
561 auto orderedRegionOp = cast<omp::OrderedRegionOp>(opInst);
564 if (orderedRegionOp.getSimd())
571 auto bodyGenCB = [&](InsertPointTy allocaIP, InsertPointTy codeGenIP) {
573 auto ®ion = cast<omp::OrderedRegionOp>(opInst).getRegion();
574 builder.restoreIP(codeGenIP);
576 moduleTranslation, bodyGenStatus);
581 auto finiCB = [&](InsertPointTy codeGenIP) {};
583 llvm::OpenMPIRBuilder::LocationDescription ompLoc(builder);
586 ompLoc, bodyGenCB, finiCB, !orderedRegionOp.getSimd()));
587 return bodyGenStatus;
593 using InsertPointTy = llvm::OpenMPIRBuilder::InsertPointTy;
594 using StorableBodyGenCallbackTy =
595 llvm::OpenMPIRBuilder::StorableBodyGenCallbackTy;
597 auto sectionsOp = cast<omp::SectionsOp>(opInst);
601 if (!sectionsOp.getReductionVars().empty() || sectionsOp.getReductions() ||
602 !sectionsOp.getAllocateVars().empty() ||
603 !sectionsOp.getAllocatorsVars().empty())
605 <<
"reduction and allocate clauses are not supported for sections "
612 auto sectionOp = dyn_cast<omp::SectionOp>(op);
616 Region ®ion = sectionOp.getRegion();
617 auto sectionCB = [®ion, &builder, &moduleTranslation, &bodyGenStatus](
618 InsertPointTy allocaIP, InsertPointTy codeGenIP) {
619 builder.restoreIP(codeGenIP);
621 moduleTranslation, bodyGenStatus);
623 sectionCBs.push_back(sectionCB);
629 if (sectionCBs.empty())
632 assert(isa<omp::SectionOp>(*sectionsOp.getRegion().op_begin()));
637 auto privCB = [&](InsertPointTy, InsertPointTy codeGenIP, llvm::Value &,
639 llvm::Value *&replacementValue) -> InsertPointTy {
640 replacementValue = &vPtr;
646 auto finiCB = [&](InsertPointTy codeGenIP) {};
648 llvm::OpenMPIRBuilder::InsertPointTy allocaIP =
650 llvm::OpenMPIRBuilder::LocationDescription ompLoc(builder);
652 ompLoc, allocaIP, sectionCBs, privCB, finiCB,
false,
653 sectionsOp.getNowait()));
654 return bodyGenStatus;
661 using InsertPointTy = llvm::OpenMPIRBuilder::InsertPointTy;
662 llvm::OpenMPIRBuilder::LocationDescription ompLoc(builder);
664 auto bodyCB = [&](InsertPointTy allocaIP, InsertPointTy codegenIP) {
665 builder.restoreIP(codegenIP);
667 moduleTranslation, bodyGenStatus);
669 auto finiCB = [&](InsertPointTy codeGenIP) {};
673 std::optional<ArrayAttr> cpFuncs = singleOp.getCopyprivateFuncs();
676 for (
size_t i = 0, e = cpVars.size(); i < e; ++i) {
677 llvmCPVars.push_back(moduleTranslation.
lookupValue(cpVars[i]));
678 auto llvmFuncOp = SymbolTable::lookupNearestSymbolFrom<LLVM::LLVMFuncOp>(
679 singleOp, cast<SymbolRefAttr>((*cpFuncs)[i]));
680 llvmCPFuncs.push_back(
685 ompLoc, bodyCB, finiCB, singleOp.getNowait(), llvmCPVars, llvmCPFuncs));
686 return bodyGenStatus;
693 using InsertPointTy = llvm::OpenMPIRBuilder::InsertPointTy;
695 if (!op.getAllocatorsVars().empty() || op.getReductions())
696 return op.
emitError(
"unhandled clauses for translation to LLVM IR");
698 auto bodyCB = [&](InsertPointTy allocaIP, InsertPointTy codegenIP) {
700 moduleTranslation, allocaIP);
701 builder.restoreIP(codegenIP);
703 moduleTranslation, bodyGenStatus);
706 llvm::Value *numTeamsLower =
nullptr;
707 if (
Value numTeamsLowerVar = op.getNumTeamsLower())
708 numTeamsLower = moduleTranslation.
lookupValue(numTeamsLowerVar);
710 llvm::Value *numTeamsUpper =
nullptr;
711 if (
Value numTeamsUpperVar = op.getNumTeamsUpper())
712 numTeamsUpper = moduleTranslation.
lookupValue(numTeamsUpperVar);
714 llvm::Value *threadLimit =
nullptr;
715 if (
Value threadLimitVar = op.getThreadLimit())
716 threadLimit = moduleTranslation.
lookupValue(threadLimitVar);
718 llvm::Value *ifExpr =
nullptr;
719 if (
Value ifExprVar = op.getIfExpr())
722 llvm::OpenMPIRBuilder::LocationDescription ompLoc(builder);
724 ompLoc, bodyCB, numTeamsLower, numTeamsUpper, threadLimit, ifExpr));
725 return bodyGenStatus;
732 using InsertPointTy = llvm::OpenMPIRBuilder::InsertPointTy;
734 if (taskOp.getUntiedAttr() || taskOp.getMergeableAttr() ||
735 taskOp.getInReductions() || taskOp.getPriority() ||
736 !taskOp.getAllocateVars().empty()) {
737 return taskOp.emitError(
"unhandled clauses for translation to LLVM IR");
739 auto bodyCB = [&](InsertPointTy allocaIP, InsertPointTy codegenIP) {
743 moduleTranslation, allocaIP);
745 builder.restoreIP(codegenIP);
747 moduleTranslation, bodyGenStatus);
751 if (!taskOp.getDependVars().empty() && taskOp.getDepends()) {
753 llvm::zip(taskOp.getDependVars(), taskOp.getDepends()->getValue())) {
754 llvm::omp::RTLDependenceKindTy type;
756 cast<mlir::omp::ClauseTaskDependAttr>(std::get<1>(dep)).getValue()) {
757 case mlir::omp::ClauseTaskDepend::taskdependin:
758 type = llvm::omp::RTLDependenceKindTy::DepIn;
763 case mlir::omp::ClauseTaskDepend::taskdependout:
764 case mlir::omp::ClauseTaskDepend::taskdependinout:
765 type = llvm::omp::RTLDependenceKindTy::DepInOut;
768 llvm::Value *depVal = moduleTranslation.
lookupValue(std::get<0>(dep));
769 llvm::OpenMPIRBuilder::DependData dd(type, depVal->getType(), depVal);
770 dds.emplace_back(dd);
774 llvm::OpenMPIRBuilder::InsertPointTy allocaIP =
776 llvm::OpenMPIRBuilder::LocationDescription ompLoc(builder);
778 ompLoc, allocaIP, bodyCB, !taskOp.getUntied(),
779 moduleTranslation.
lookupValue(taskOp.getFinalExpr()),
780 moduleTranslation.
lookupValue(taskOp.getIfExpr()), dds));
781 return bodyGenStatus;
788 using InsertPointTy = llvm::OpenMPIRBuilder::InsertPointTy;
790 if (!tgOp.getTaskReductionVars().empty() || !tgOp.getAllocateVars().empty()) {
791 return tgOp.emitError(
"unhandled clauses for translation to LLVM IR");
793 auto bodyCB = [&](InsertPointTy allocaIP, InsertPointTy codegenIP) {
794 builder.restoreIP(codegenIP);
796 moduleTranslation, bodyGenStatus);
799 llvm::OpenMPIRBuilder::LocationDescription ompLoc(builder);
801 ompLoc, allocaIP, bodyCB));
802 return bodyGenStatus;
806 template <
typename T>
808 T loop, llvm::IRBuilderBase &builder,
810 llvm::OpenMPIRBuilder::InsertPointTy &allocaIP,
814 llvm::IRBuilderBase::InsertPointGuard guard(builder);
815 builder.restoreIP(allocaIP);
817 loop.getRegion().getArguments().take_back(loop.getNumReductionVars());
819 for (std::size_t i = 0; i < loop.getNumReductionVars(); ++i) {
820 llvm::Value *var = builder.CreateAlloca(
821 moduleTranslation.
convertType(reductionDecls[i].getType()));
822 moduleTranslation.
mapValue(args[i], var);
823 privateReductionVariables.push_back(var);
824 reductionVariableMap.try_emplace(loop.getReductionVars()[i], var);
829 template <
typename T>
835 mlir::omp::DeclareReductionOp &reduction = reductionDecls[i];
836 Region &initializerRegion = reduction.getInitializerRegion();
839 "the initialization region has one argument");
841 mlir::Value mlirSource = loop.getReductionVars()[i];
842 llvm::Value *llvmSource = moduleTranslation.
lookupValue(mlirSource);
843 assert(llvmSource &&
"lookup reduction var");
848 template <
typename T>
850 T loop, llvm::IRBuilderBase &builder,
857 unsigned numReductions = loop.getNumReductionVars();
859 for (
unsigned i = 0; i < numReductions; ++i) {
860 owningReductionGens.push_back(
862 owningAtomicReductionGens.push_back(
867 reductionInfos.reserve(numReductions);
868 for (
unsigned i = 0; i < numReductions; ++i) {
869 llvm::OpenMPIRBuilder::AtomicReductionGenTy atomicGen =
nullptr;
870 if (owningAtomicReductionGens[i])
871 atomicGen = owningAtomicReductionGens[i];
872 llvm::Value *variable =
873 moduleTranslation.
lookupValue(loop.getReductionVars()[i]);
874 reductionInfos.push_back(
875 {moduleTranslation.
convertType(reductionDecls[i].getType()), variable,
876 privateReductionVariables[i], owningReductionGens[i], atomicGen});
886 Region &cleanupRegion = reductionDecl.getCleanupRegion();
887 if (cleanupRegion.
empty())
893 llvm::Instruction *potentialTerminator =
894 builder.GetInsertBlock()->
empty() ? nullptr
895 : &builder.GetInsertBlock()->back();
896 if (potentialTerminator && potentialTerminator->isTerminator())
897 builder.SetInsertPoint(potentialTerminator);
898 llvm::Value *reductionVar = builder.CreateLoad(
900 privateReductionVariables[i]);
905 builder, moduleTranslation)))
919 auto wsloopOp = cast<omp::WsloopOp>(opInst);
920 auto loopOp = cast<omp::LoopNestOp>(wsloopOp.getWrappedLoop());
921 const bool isByRef = wsloopOp.getByref();
924 if (loopOp.getLowerBound().empty())
929 wsloopOp.getScheduleVal().value_or(omp::ClauseScheduleKind::Static);
932 llvm::Value *step = moduleTranslation.
lookupValue(loopOp.getStep()[0]);
933 llvm::Type *ivType = step->getType();
934 llvm::Value *chunk =
nullptr;
935 if (wsloopOp.getScheduleChunkVar()) {
936 llvm::Value *chunkVar =
937 moduleTranslation.
lookupValue(wsloopOp.getScheduleChunkVar());
938 chunk = builder.CreateSExtOrTrunc(chunkVar, ivType);
943 llvm::OpenMPIRBuilder::InsertPointTy allocaIP =
950 reductionDecls, privateReductionVariables,
951 reductionVariableMap);
958 for (
unsigned i = 0; i < wsloopOp.getNumReductionVars(); ++i) {
965 "omp.reduction.neutral", builder,
966 moduleTranslation, &phis)))
968 assert(phis.size() == 1 &&
"expected one value to be yielded from the "
969 "reduction neutral element declaration region");
973 llvm::Value *var = builder.CreateAlloca(
974 moduleTranslation.
convertType(reductionDecls[i].getType()));
977 builder.CreateStore(phis[0], var);
979 privateReductionVariables.push_back(var);
980 moduleTranslation.
mapValue(reductionArgs[i], phis[0]);
981 reductionVariableMap.try_emplace(wsloopOp.getReductionVars()[i], phis[0]);
984 builder.CreateStore(phis[0], privateReductionVariables[i]);
991 moduleTranslation.
forgetMapping(reductionDecls[i].getInitializerRegion());
998 moduleTranslation, reductionVariableMap);
1001 llvm::OpenMPIRBuilder::LocationDescription ompLoc(builder);
1009 auto bodyGen = [&](llvm::OpenMPIRBuilder::InsertPointTy ip, llvm::Value *iv) {
1012 loopOp.getRegion().front().getArgument(loopInfos.size()), iv);
1017 bodyInsertPoints.push_back(ip);
1019 if (loopInfos.size() != loopOp.getNumLoops() - 1)
1023 builder.restoreIP(ip);
1025 moduleTranslation, bodyGenStatus);
1034 for (
unsigned i = 0, e = loopOp.getNumLoops(); i < e; ++i) {
1035 llvm::Value *lowerBound =
1036 moduleTranslation.
lookupValue(loopOp.getLowerBound()[i]);
1037 llvm::Value *upperBound =
1038 moduleTranslation.
lookupValue(loopOp.getUpperBound()[i]);
1039 llvm::Value *step = moduleTranslation.
lookupValue(loopOp.getStep()[i]);
1044 llvm::OpenMPIRBuilder::LocationDescription loc = ompLoc;
1045 llvm::OpenMPIRBuilder::InsertPointTy computeIP = ompLoc.IP;
1047 loc = llvm::OpenMPIRBuilder::LocationDescription(bodyInsertPoints.back());
1048 computeIP = loopInfos.front()->getPreheaderIP();
1050 loopInfos.push_back(ompBuilder->createCanonicalLoop(
1051 loc, bodyGen, lowerBound, upperBound, step,
1052 true, loopOp.getInclusive(), computeIP));
1054 if (
failed(bodyGenStatus))
1060 llvm::IRBuilderBase::InsertPoint afterIP = loopInfos.front()->getAfterIP();
1061 llvm::CanonicalLoopInfo *loopInfo =
1062 ompBuilder->collapseLoops(ompLoc.DL, loopInfos, {});
1067 bool isOrdered = wsloopOp.getOrderedVal().has_value();
1068 std::optional<omp::ScheduleModifier> scheduleModifier =
1069 wsloopOp.getScheduleModifier();
1070 bool isSimd = wsloopOp.getSimdModifier();
1072 ompBuilder->applyWorkshareLoop(
1073 ompLoc.DL, loopInfo, allocaIP, !wsloopOp.getNowait(),
1074 convertToScheduleKind(schedule), chunk, isSimd,
1075 scheduleModifier == omp::ScheduleModifier::monotonic,
1076 scheduleModifier == omp::ScheduleModifier::nonmonotonic, isOrdered);
1082 builder.restoreIP(afterIP);
1085 if (wsloopOp.getNumReductionVars() == 0)
1094 owningReductionGens, owningAtomicReductionGens,
1095 privateReductionVariables, reductionInfos);
1100 llvm::UnreachableInst *tempTerminator = builder.CreateUnreachable();
1101 builder.SetInsertPoint(tempTerminator);
1102 llvm::OpenMPIRBuilder::InsertPointTy contInsertPoint =
1103 ompBuilder->createReductions(builder.saveIP(), allocaIP, reductionInfos,
1104 wsloopOp.getNowait(), isByRef);
1105 if (!contInsertPoint.getBlock())
1106 return wsloopOp->emitOpError() <<
"failed to convert reductions";
1107 auto nextInsertionPoint =
1108 ompBuilder->createBarrier(contInsertPoint, llvm::omp::OMPD_for);
1109 tempTerminator->eraseFromParent();
1110 builder.restoreIP(nextInsertionPoint);
1114 moduleTranslation, builder);
1127 : region(opInst.getRegion()), privateVars(opInst.getPrivateVars()),
1128 privateArgBeginIdx(opInst.getNumReductionVars()),
1129 privateArgEndIdx(privateArgBeginIdx + privateVars.size()) {
1130 auto privateVarsIt = privateVars.begin();
1132 for (
size_t argIdx = privateArgBeginIdx; argIdx < privateArgEndIdx;
1133 ++argIdx, ++privateVarsIt)
1135 *privateVarsIt, region);
1139 auto privateVarsIt = privateVars.begin();
1141 for (
size_t argIdx = privateArgBeginIdx; argIdx < privateArgEndIdx;
1142 ++argIdx, ++privateVarsIt)
1144 region.getArgument(argIdx), region);
1150 unsigned privateArgBeginIdx;
1151 unsigned privateArgEndIdx;
1158 using InsertPointTy = llvm::OpenMPIRBuilder::InsertPointTy;
1160 const bool isByRef = opInst.getByref();
1172 auto bodyGenCB = [&](InsertPointTy allocaIP, InsertPointTy codeGenIP) {
1177 reductionDecls, privateReductionVariables,
1178 reductionVariableMap);
1182 builder.restoreIP(allocaIP);
1184 opInst.getRegion().getArguments().take_back(
1185 opInst.getNumReductionVars());
1186 for (
unsigned i = 0; i < opInst.getNumReductionVars(); ++i) {
1192 reductionDecls[i].getInitializerRegion(),
"omp.reduction.neutral",
1193 builder, moduleTranslation, &phis)))
1195 assert(phis.size() == 1 &&
1196 "expected one value to be yielded from the "
1197 "reduction neutral element declaration region");
1198 builder.restoreIP(allocaIP);
1203 llvm::Value *var = builder.CreateAlloca(
1204 moduleTranslation.
convertType(reductionDecls[i].getType()));
1207 builder.CreateStore(phis[0], var);
1209 privateReductionVariables.push_back(var);
1210 moduleTranslation.
mapValue(reductionArgs[i], phis[0]);
1211 reductionVariableMap.try_emplace(opInst.getReductionVars()[i], phis[0]);
1214 builder.CreateStore(phis[0], privateReductionVariables[i]);
1220 moduleTranslation.
forgetMapping(reductionDecls[i].getInitializerRegion());
1227 moduleTranslation, reductionVariableMap);
1232 moduleTranslation, allocaIP);
1235 builder.restoreIP(codeGenIP);
1238 moduleTranslation, bodyGenStatus);
1241 if (opInst.getNumReductionVars() > 0) {
1247 owningReductionGens, owningAtomicReductionGens,
1248 privateReductionVariables, reductionInfos);
1251 builder.SetInsertPoint(regionBlock->getTerminator());
1254 llvm::UnreachableInst *tempTerminator = builder.CreateUnreachable();
1255 builder.SetInsertPoint(tempTerminator);
1257 llvm::OpenMPIRBuilder::InsertPointTy contInsertPoint =
1258 ompBuilder->createReductions(builder.saveIP(), allocaIP,
1259 reductionInfos,
false, isByRef);
1260 if (!contInsertPoint.getBlock()) {
1261 bodyGenStatus = opInst->emitOpError() <<
"failed to convert reductions";
1265 tempTerminator->eraseFromParent();
1266 builder.restoreIP(contInsertPoint);
1273 auto privCB = [&](InsertPointTy allocaIP, InsertPointTy codeGenIP,
1274 llvm::Value &, llvm::Value &vPtr,
1275 llvm::Value *&replacementValue) -> InsertPointTy {
1276 replacementValue = &vPtr;
1281 auto [privVar, privatizerClone] =
1282 [&]() -> std::pair<mlir::Value, omp::PrivateClauseOp> {
1283 if (!opInst.getPrivateVars().empty()) {
1284 auto privVars = opInst.getPrivateVars();
1285 auto privatizers = opInst.getPrivatizers();
1287 for (
auto [privVar, privatizerAttr] :
1288 llvm::zip_equal(privVars, *privatizers)) {
1291 llvm::Value *llvmPrivVar = moduleTranslation.
lookupValue(privVar);
1292 if (llvmPrivVar != &vPtr)
1295 SymbolRefAttr privSym = llvm::cast<SymbolRefAttr>(privatizerAttr);
1296 omp::PrivateClauseOp privatizer =
1297 SymbolTable::lookupNearestSymbolFrom<omp::PrivateClauseOp>(
1304 return {privVar, privatizer.clone()};
1312 Region &allocRegion = privatizerClone.getAllocRegion();
1315 if (privatizerClone.getDataSharingType() ==
1316 omp::DataSharingClauseType::FirstPrivate) {
1317 auto oldAllocBackBlock = std::prev(allocRegion.
end());
1318 omp::YieldOp oldAllocYieldOp =
1319 llvm::cast<omp::YieldOp>(oldAllocBackBlock->getTerminator());
1321 Region ©Region = privatizerClone.getCopyRegion();
1328 auto newCopyRegionFrontBlock = std::next(oldAllocBackBlock);
1336 &*newCopyRegionFrontBlock, &*oldAllocBackBlock,
1337 {allocRegion.
getArgument(0), oldAllocYieldOp.getOperand(0)});
1341 oldAllocYieldOp.erase();
1350 auto oldIP = builder.saveIP();
1351 builder.restoreIP(allocaIP);
1355 moduleTranslation, &yieldedValues))) {
1356 opInst.emitError(
"failed to inline `alloc` region of an `omp.private` "
1357 "op in the parallel region");
1360 assert(yieldedValues.size() == 1);
1361 replacementValue = yieldedValues.front();
1364 privatizerClone.erase();
1365 builder.restoreIP(oldIP);
1373 auto finiCB = [&](InsertPointTy codeGenIP) {
1374 InsertPointTy oldIP = builder.saveIP();
1375 builder.restoreIP(codeGenIP);
1380 moduleTranslation, builder)))
1383 builder.restoreIP(oldIP);
1386 llvm::Value *ifCond =
nullptr;
1387 if (
auto ifExprVar = opInst.getIfExprVar())
1388 ifCond = moduleTranslation.
lookupValue(ifExprVar);
1389 llvm::Value *numThreads =
nullptr;
1390 if (
auto numThreadsVar = opInst.getNumThreadsVar())
1391 numThreads = moduleTranslation.
lookupValue(numThreadsVar);
1392 auto pbKind = llvm::omp::OMP_PROC_BIND_default;
1393 if (
auto bind = opInst.getProcBindVal())
1396 bool isCancellable =
false;
1398 llvm::OpenMPIRBuilder::InsertPointTy allocaIP =
1400 llvm::OpenMPIRBuilder::LocationDescription ompLoc(builder);
1403 ompBuilder->createParallel(ompLoc, allocaIP, bodyGenCB, privCB, finiCB,
1404 ifCond, numThreads, pbKind, isCancellable));
1406 return bodyGenStatus;
1413 auto simdOp = cast<omp::SimdOp>(opInst);
1414 auto loopOp = cast<omp::LoopNestOp>(simdOp.getWrappedLoop());
1416 llvm::OpenMPIRBuilder::LocationDescription ompLoc(builder);
1424 auto bodyGen = [&](llvm::OpenMPIRBuilder::InsertPointTy ip, llvm::Value *iv) {
1427 loopOp.getRegion().front().getArgument(loopInfos.size()), iv);
1432 bodyInsertPoints.push_back(ip);
1434 if (loopInfos.size() != loopOp.getNumLoops() - 1)
1438 builder.restoreIP(ip);
1440 moduleTranslation, bodyGenStatus);
1449 for (
unsigned i = 0, e = loopOp.getNumLoops(); i < e; ++i) {
1450 llvm::Value *lowerBound =
1451 moduleTranslation.
lookupValue(loopOp.getLowerBound()[i]);
1452 llvm::Value *upperBound =
1453 moduleTranslation.
lookupValue(loopOp.getUpperBound()[i]);
1454 llvm::Value *step = moduleTranslation.
lookupValue(loopOp.getStep()[i]);
1459 llvm::OpenMPIRBuilder::LocationDescription loc = ompLoc;
1460 llvm::OpenMPIRBuilder::InsertPointTy computeIP = ompLoc.IP;
1462 loc = llvm::OpenMPIRBuilder::LocationDescription(bodyInsertPoints.back(),
1464 computeIP = loopInfos.front()->getPreheaderIP();
1466 loopInfos.push_back(ompBuilder->createCanonicalLoop(
1467 loc, bodyGen, lowerBound, upperBound, step,
1468 true,
true, computeIP));
1470 if (
failed(bodyGenStatus))
1475 llvm::IRBuilderBase::InsertPoint afterIP = loopInfos.front()->getAfterIP();
1476 llvm::CanonicalLoopInfo *loopInfo =
1477 ompBuilder->collapseLoops(ompLoc.DL, loopInfos, {});
1479 llvm::ConstantInt *simdlen =
nullptr;
1480 if (std::optional<uint64_t> simdlenVar = simdOp.getSimdlen())
1481 simdlen = builder.getInt64(simdlenVar.value());
1483 llvm::ConstantInt *safelen =
nullptr;
1484 if (std::optional<uint64_t> safelenVar = simdOp.getSafelen())
1485 safelen = builder.getInt64(safelenVar.value());
1487 llvm::MapVector<llvm::Value *, llvm::Value *> alignedVars;
1488 ompBuilder->applySimd(
1489 loopInfo, alignedVars,
1490 simdOp.getIfExpr() ? moduleTranslation.
lookupValue(simdOp.getIfExpr())
1492 llvm::omp::OrderKind::OMP_ORDER_unknown, simdlen, safelen);
1494 builder.restoreIP(afterIP);
1499 static llvm::AtomicOrdering
1502 return llvm::AtomicOrdering::Monotonic;
1505 case omp::ClauseMemoryOrderKind::Seq_cst:
1506 return llvm::AtomicOrdering::SequentiallyConsistent;
1507 case omp::ClauseMemoryOrderKind::Acq_rel:
1508 return llvm::AtomicOrdering::AcquireRelease;
1509 case omp::ClauseMemoryOrderKind::Acquire:
1510 return llvm::AtomicOrdering::Acquire;
1511 case omp::ClauseMemoryOrderKind::Release:
1512 return llvm::AtomicOrdering::Release;
1513 case omp::ClauseMemoryOrderKind::Relaxed:
1514 return llvm::AtomicOrdering::Monotonic;
1516 llvm_unreachable(
"Unknown ClauseMemoryOrderKind kind");
1524 auto readOp = cast<omp::AtomicReadOp>(opInst);
1527 llvm::OpenMPIRBuilder::LocationDescription ompLoc(builder);
1530 llvm::Value *x = moduleTranslation.
lookupValue(readOp.getX());
1531 llvm::Value *v = moduleTranslation.
lookupValue(readOp.getV());
1533 llvm::Type *elementType =
1534 moduleTranslation.
convertType(readOp.getElementType());
1536 llvm::OpenMPIRBuilder::AtomicOpValue V = {v, elementType,
false,
false};
1537 llvm::OpenMPIRBuilder::AtomicOpValue X = {x, elementType,
false,
false};
1538 builder.restoreIP(ompBuilder->createAtomicRead(ompLoc, X, V, AO));
1546 auto writeOp = cast<omp::AtomicWriteOp>(opInst);
1549 llvm::OpenMPIRBuilder::LocationDescription ompLoc(builder);
1551 llvm::Value *expr = moduleTranslation.
lookupValue(writeOp.getExpr());
1552 llvm::Value *dest = moduleTranslation.
lookupValue(writeOp.getX());
1553 llvm::Type *ty = moduleTranslation.
convertType(writeOp.getExpr().getType());
1554 llvm::OpenMPIRBuilder::AtomicOpValue x = {dest, ty,
false,
1556 builder.restoreIP(ompBuilder->createAtomicWrite(ompLoc, x, expr, ao));
1564 .Case([&](LLVM::AddOp) {
return llvm::AtomicRMWInst::BinOp::Add; })
1565 .Case([&](LLVM::SubOp) {
return llvm::AtomicRMWInst::BinOp::Sub; })
1566 .Case([&](LLVM::AndOp) {
return llvm::AtomicRMWInst::BinOp::And; })
1567 .Case([&](LLVM::OrOp) {
return llvm::AtomicRMWInst::BinOp::Or; })
1568 .Case([&](LLVM::XOrOp) {
return llvm::AtomicRMWInst::BinOp::Xor; })
1569 .Case([&](LLVM::UMaxOp) {
return llvm::AtomicRMWInst::BinOp::UMax; })
1570 .Case([&](LLVM::UMinOp) {
return llvm::AtomicRMWInst::BinOp::UMin; })
1571 .Case([&](LLVM::FAddOp) {
return llvm::AtomicRMWInst::BinOp::FAdd; })
1572 .Case([&](LLVM::FSubOp) {
return llvm::AtomicRMWInst::BinOp::FSub; })
1573 .Default(llvm::AtomicRMWInst::BinOp::BAD_BINOP);
1579 llvm::IRBuilderBase &builder,
1584 auto &innerOpList = opInst.getRegion().front().getOperations();
1585 bool isRegionArgUsed{
false}, isXBinopExpr{
false};
1586 llvm::AtomicRMWInst::BinOp binop;
1590 for (
Operation &innerOp : innerOpList) {
1591 if (innerOp.getNumOperands() == 2) {
1593 if (!llvm::is_contained(innerOp.getOperands(),
1594 opInst.getRegion().getArgument(0)))
1596 isRegionArgUsed =
true;
1597 isXBinopExpr = innerOp.getNumOperands() > 0 &&
1598 innerOp.getOperand(0) == opInst.getRegion().getArgument(0);
1599 mlirExpr = (isXBinopExpr ? innerOp.getOperand(1) : innerOp.getOperand(0));
1603 if (!isRegionArgUsed)
1604 return opInst.emitError(
"no atomic update operation with region argument"
1605 " as operand found inside atomic.update region");
1607 llvm::Value *llvmExpr = moduleTranslation.
lookupValue(mlirExpr);
1608 llvm::Value *llvmX = moduleTranslation.
lookupValue(opInst.getX());
1609 llvm::Type *llvmXElementType = moduleTranslation.
convertType(
1610 opInst.getRegion().getArgument(0).getType());
1611 llvm::OpenMPIRBuilder::AtomicOpValue llvmAtomicX = {llvmX, llvmXElementType,
1615 llvm::AtomicOrdering atomicOrdering =
1620 auto updateFn = [&opInst, &moduleTranslation, &updateGenStatus](
1621 llvm::Value *atomicx,
1622 llvm::IRBuilder<> &builder) -> llvm::Value * {
1624 moduleTranslation.
mapValue(*opInst.getRegion().args_begin(), atomicx);
1625 moduleTranslation.
mapBlock(&bb, builder.GetInsertBlock());
1627 updateGenStatus = (opInst.emitError()
1628 <<
"unable to convert update operation to llvm IR");
1631 omp::YieldOp yieldop = dyn_cast<omp::YieldOp>(bb.
getTerminator());
1632 assert(yieldop && yieldop.getResults().size() == 1 &&
1633 "terminator must be omp.yield op and it must have exactly one "
1635 return moduleTranslation.
lookupValue(yieldop.getResults()[0]);
1640 llvm::OpenMPIRBuilder::LocationDescription ompLoc(builder);
1641 builder.restoreIP(ompBuilder->createAtomicUpdate(
1642 ompLoc, allocaIP, llvmAtomicX, llvmExpr, atomicOrdering, binop, updateFn,
1644 return updateGenStatus;
1649 llvm::IRBuilderBase &builder,
1653 bool isXBinopExpr =
false, isPostfixUpdate =
false;
1654 llvm::AtomicRMWInst::BinOp binop = llvm::AtomicRMWInst::BinOp::BAD_BINOP;
1656 omp::AtomicUpdateOp atomicUpdateOp = atomicCaptureOp.getAtomicUpdateOp();
1657 omp::AtomicWriteOp atomicWriteOp = atomicCaptureOp.getAtomicWriteOp();
1659 assert((atomicUpdateOp || atomicWriteOp) &&
1660 "internal op must be an atomic.update or atomic.write op");
1662 if (atomicWriteOp) {
1663 isPostfixUpdate =
true;
1664 mlirExpr = atomicWriteOp.getExpr();
1666 isPostfixUpdate = atomicCaptureOp.getSecondOp() ==
1667 atomicCaptureOp.getAtomicUpdateOp().getOperation();
1668 auto &innerOpList = atomicUpdateOp.getRegion().front().getOperations();
1669 bool isRegionArgUsed{
false};
1672 for (
Operation &innerOp : innerOpList) {
1673 if (innerOp.getNumOperands() == 2) {
1675 if (!llvm::is_contained(innerOp.getOperands(),
1676 atomicUpdateOp.getRegion().getArgument(0)))
1678 isRegionArgUsed =
true;
1680 innerOp.getNumOperands() > 0 &&
1681 innerOp.getOperand(0) == atomicUpdateOp.getRegion().getArgument(0);
1683 (isXBinopExpr ? innerOp.getOperand(1) : innerOp.getOperand(0));
1687 if (!isRegionArgUsed)
1688 return atomicUpdateOp.emitError(
1689 "no atomic update operation with region argument"
1690 " as operand found inside atomic.update region");
1693 llvm::Value *llvmExpr = moduleTranslation.
lookupValue(mlirExpr);
1694 llvm::Value *llvmX =
1695 moduleTranslation.
lookupValue(atomicCaptureOp.getAtomicReadOp().getX());
1696 llvm::Value *llvmV =
1697 moduleTranslation.
lookupValue(atomicCaptureOp.getAtomicReadOp().getV());
1698 llvm::Type *llvmXElementType = moduleTranslation.
convertType(
1699 atomicCaptureOp.getAtomicReadOp().getElementType());
1700 llvm::OpenMPIRBuilder::AtomicOpValue llvmAtomicX = {llvmX, llvmXElementType,
1703 llvm::OpenMPIRBuilder::AtomicOpValue llvmAtomicV = {llvmV, llvmXElementType,
1707 llvm::AtomicOrdering atomicOrdering =
1711 auto updateFn = [&](llvm::Value *atomicx,
1712 llvm::IRBuilder<> &builder) -> llvm::Value * {
1714 return moduleTranslation.
lookupValue(atomicWriteOp.getExpr());
1715 Block &bb = *atomicUpdateOp.getRegion().
begin();
1716 moduleTranslation.
mapValue(*atomicUpdateOp.getRegion().args_begin(),
1718 moduleTranslation.
mapBlock(&bb, builder.GetInsertBlock());
1720 updateGenStatus = (atomicUpdateOp.emitError()
1721 <<
"unable to convert update operation to llvm IR");
1724 omp::YieldOp yieldop = dyn_cast<omp::YieldOp>(bb.
getTerminator());
1725 assert(yieldop && yieldop.getResults().size() == 1 &&
1726 "terminator must be omp.yield op and it must have exactly one "
1728 return moduleTranslation.
lookupValue(yieldop.getResults()[0]);
1733 llvm::OpenMPIRBuilder::LocationDescription ompLoc(builder);
1734 builder.restoreIP(ompBuilder->createAtomicCapture(
1735 ompLoc, allocaIP, llvmAtomicX, llvmAtomicV, llvmExpr, atomicOrdering,
1736 binop, updateFn, atomicUpdateOp, isPostfixUpdate, isXBinopExpr));
1737 return updateGenStatus;
1746 llvm::IRBuilderBase &builder,
1749 omp::DeclareReductionOp declaration;
1751 if (dyn_cast<omp::ParallelOp>(reductionParent) ||
1752 dyn_cast<omp::WsloopOp>(reductionParent)) {
1755 llvm_unreachable(
"Unhandled reduction container");
1757 assert(declaration &&
"could not find reduction declaration");
1762 moduleTranslation.
stackWalk<OpenMPVarMappingStackFrame>(
1763 [&](
const OpenMPVarMappingStackFrame &frame) {
1764 if (frame.mapping.contains(reductionOp.getAccumulator())) {
1765 reductionVariableMap = &frame.mapping;
1770 assert(reductionVariableMap &&
"couldn't find private reduction variables");
1773 Region &reductionRegion = declaration.getReductionRegion();
1774 llvm::Value *privateReductionVar =
1775 reductionVariableMap->lookup(reductionOp.getAccumulator());
1776 llvm::Value *reductionVal = builder.CreateLoad(
1777 moduleTranslation.
convertType(reductionOp.getOperand().getType()),
1778 privateReductionVar);
1784 moduleTranslation.
lookupValue(reductionOp.getOperand()));
1788 builder, moduleTranslation, &phis)))
1790 assert(phis.size() == 1 &&
"expected one value to be yielded from "
1791 "the reduction body declaration region");
1792 builder.CreateStore(phis[0], privateReductionVar);
1801 llvm::OpenMPIRBuilder::LocationDescription ompLoc(builder);
1802 auto threadprivateOp = cast<omp::ThreadprivateOp>(opInst);
1804 Value symAddr = threadprivateOp.getSymAddr();
1806 if (!isa<LLVM::AddressOfOp>(symOp))
1807 return opInst.
emitError(
"Addressing symbol not found");
1808 LLVM::AddressOfOp addressOfOp = dyn_cast<LLVM::AddressOfOp>(symOp);
1810 LLVM::GlobalOp global =
1811 addressOfOp.getGlobal(moduleTranslation.
symbolTable());
1812 llvm::GlobalValue *globalValue = moduleTranslation.
lookupGlobal(global);
1813 llvm::Type *type = globalValue->getValueType();
1814 llvm::TypeSize typeSize =
1815 builder.GetInsertBlock()->getModule()->getDataLayout().getTypeStoreSize(
1817 llvm::ConstantInt *size = builder.getInt64(typeSize.getFixedValue());
1818 llvm::StringRef suffix = llvm::StringRef(
".cache", 6);
1819 std::string cacheName = (Twine(global.getSymName()).concat(suffix)).str();
1820 llvm::Value *callInst =
1822 ompLoc, globalValue, size, cacheName);
1827 static llvm::OffloadEntriesInfoManager::OMPTargetDeviceClauseKind
1829 switch (deviceClause) {
1830 case mlir::omp::DeclareTargetDeviceType::host:
1831 return llvm::OffloadEntriesInfoManager::OMPTargetDeviceClauseHost;
1833 case mlir::omp::DeclareTargetDeviceType::nohost:
1834 return llvm::OffloadEntriesInfoManager::OMPTargetDeviceClauseNoHost;
1836 case mlir::omp::DeclareTargetDeviceType::any:
1837 return llvm::OffloadEntriesInfoManager::OMPTargetDeviceClauseAny;
1840 llvm_unreachable(
"unhandled device clause");
1843 static llvm::OffloadEntriesInfoManager::OMPTargetGlobalVarEntryKind
1845 mlir::omp::DeclareTargetCaptureClause captureClasue) {
1846 switch (captureClasue) {
1847 case mlir::omp::DeclareTargetCaptureClause::to:
1848 return llvm::OffloadEntriesInfoManager::OMPTargetGlobalVarEntryTo;
1849 case mlir::omp::DeclareTargetCaptureClause::link:
1850 return llvm::OffloadEntriesInfoManager::OMPTargetGlobalVarEntryLink;
1851 case mlir::omp::DeclareTargetCaptureClause::enter:
1852 return llvm::OffloadEntriesInfoManager::OMPTargetGlobalVarEntryEnter;
1854 llvm_unreachable(
"unhandled capture clause");
1859 llvm::OpenMPIRBuilder &ompBuilder) {
1861 llvm::raw_svector_ostream os(suffix);
1863 auto loc = globalOp->getLoc()->findInstanceOf<FileLineColLoc>();
1864 auto fileInfoCallBack = [&loc]() {
1865 return std::pair<std::string, uint64_t>(
1866 llvm::StringRef(loc.getFilename()), loc.getLine());
1870 "_%x", ompBuilder.getTargetEntryUniqueInfo(fileInfoCallBack).FileID);
1872 os <<
"_decl_tgt_ref_ptr";
1878 if (
auto addressOfOp =
1879 llvm::dyn_cast_if_present<LLVM::AddressOfOp>(value.
getDefiningOp())) {
1880 auto modOp = addressOfOp->getParentOfType<mlir::ModuleOp>();
1881 Operation *gOp = modOp.lookupSymbol(addressOfOp.getGlobalName());
1882 if (
auto declareTargetGlobal =
1883 llvm::dyn_cast<mlir::omp::DeclareTargetInterface>(gOp))
1884 if (declareTargetGlobal.getDeclareTargetCaptureClause() ==
1885 mlir::omp::DeclareTargetCaptureClause::link)
1894 static llvm::Value *
1901 if (
auto addressOfOp =
1902 llvm::dyn_cast_if_present<LLVM::AddressOfOp>(value.
getDefiningOp())) {
1903 if (
auto gOp = llvm::dyn_cast_or_null<LLVM::GlobalOp>(
1904 addressOfOp->getParentOfType<mlir::ModuleOp>().lookupSymbol(
1905 addressOfOp.getGlobalName()))) {
1907 if (
auto declareTargetGlobal =
1908 llvm::dyn_cast<mlir::omp::DeclareTargetInterface>(
1909 gOp.getOperation())) {
1913 if ((declareTargetGlobal.getDeclareTargetCaptureClause() ==
1914 mlir::omp::DeclareTargetCaptureClause::link) ||
1915 (declareTargetGlobal.getDeclareTargetCaptureClause() ==
1916 mlir::omp::DeclareTargetCaptureClause::to &&
1917 ompBuilder->Config.hasRequiresUnifiedSharedMemory())) {
1921 if (gOp.getSymName().contains(suffix))
1926 (gOp.getSymName().str() + suffix.str()).str());
1958 llvm::OpenMPIRBuilder::MapInfosTy::append(CurInfo);
1963 if (
auto nestedArrTy = llvm::dyn_cast_if_present<LLVM::LLVMArrayType>(
1964 arrTy.getElementType()))
1980 Operation *clauseOp, llvm::Value *basePointer,
1981 llvm::Type *baseType, llvm::IRBuilderBase &builder,
1986 if (
auto arrTy = llvm::dyn_cast_if_present<LLVM::LLVMArrayType>(type))
1989 if (
auto memberClause =
1990 mlir::dyn_cast_if_present<mlir::omp::MapInfoOp>(clauseOp)) {
1995 if (!memberClause.getBounds().empty()) {
1996 llvm::Value *elementCount = builder.getInt64(1);
1997 for (
auto bounds : memberClause.getBounds()) {
1998 if (
auto boundOp = mlir::dyn_cast_if_present<mlir::omp::MapBoundsOp>(
1999 bounds.getDefiningOp())) {
2004 elementCount = builder.CreateMul(
2008 moduleTranslation.
lookupValue(boundOp.getUpperBound()),
2009 moduleTranslation.
lookupValue(boundOp.getLowerBound())),
2010 builder.getInt64(1)));
2019 return builder.CreateMul(elementCount,
2020 builder.getInt64(underlyingTypeSzInBits / 8));
2024 return builder.getInt64(underlyingTypeSzInBits / 8);
2031 llvm::IRBuilderBase &builder) {
2033 if (
auto mapOp = mlir::dyn_cast_if_present<mlir::omp::MapInfoOp>(
2034 mapValue.getDefiningOp())) {
2036 mapOp.getVarPtrPtr() ? mapOp.getVarPtrPtr() : mapOp.getVarPtr();
2041 if (llvm::Value *refPtr =
2043 moduleTranslation)) {
2045 mapData.BasePointers.push_back(refPtr);
2048 mapData.BasePointers.push_back(mapData.
OriginalValue.back());
2052 moduleTranslation.
convertType(mapOp.getVarType()));
2054 dl, mapOp.getVarType(), mapOp, mapData.BasePointers.back(),
2055 mapData.
BaseType.back(), builder, moduleTranslation));
2056 mapData.
MapClause.push_back(mapOp.getOperation());
2057 mapData.Types.push_back(
2058 llvm::omp::OpenMPOffloadMappingFlags(mapOp.getMapType().value()));
2061 mapData.DevicePointers.push_back(
2073 if (
auto map = mlir::dyn_cast_if_present<mlir::omp::MapInfoOp>(
2074 mapValue.getDefiningOp())) {
2075 for (
auto member : map.getMembers()) {
2076 if (member == mapOp) {
2105 std::vector<llvm::Value *>
2107 llvm::IRBuilderBase &builder,
bool isArrayTy,
2109 std::vector<llvm::Value *> idx;
2120 idx.push_back(builder.getInt64(0));
2121 for (
int i = bounds.size() - 1; i >= 0; --i) {
2122 if (
auto boundOp = mlir::dyn_cast_if_present<mlir::omp::MapBoundsOp>(
2123 bounds[i].getDefiningOp())) {
2124 idx.push_back(moduleTranslation.
lookupValue(boundOp.getLowerBound()));
2146 std::vector<llvm::Value *> dimensionIndexSizeOffset{builder.getInt64(1)};
2147 for (
size_t i = 1; i < bounds.size(); ++i) {
2148 if (
auto boundOp = mlir::dyn_cast_if_present<mlir::omp::MapBoundsOp>(
2149 bounds[i].getDefiningOp())) {
2150 dimensionIndexSizeOffset.push_back(builder.CreateMul(
2151 moduleTranslation.
lookupValue(boundOp.getExtent()),
2152 dimensionIndexSizeOffset[i - 1]));
2160 for (
int i = bounds.size() - 1; i >= 0; --i) {
2161 if (
auto boundOp = mlir::dyn_cast_if_present<mlir::omp::MapBoundsOp>(
2162 bounds[i].getDefiningOp())) {
2164 idx.emplace_back(builder.CreateMul(
2165 moduleTranslation.
lookupValue(boundOp.getLowerBound()),
2166 dimensionIndexSizeOffset[i]));
2168 idx.back() = builder.CreateAdd(
2169 idx.back(), builder.CreateMul(moduleTranslation.
lookupValue(
2170 boundOp.getLowerBound()),
2171 dimensionIndexSizeOffset[i]));
2193 llvm::OpenMPIRBuilder &ompBuilder,
DataLayout &dl,
2195 uint64_t mapDataIndex,
bool isTargetParams) {
2197 combinedInfo.Types.emplace_back(
2199 ? llvm::omp::OpenMPOffloadMappingFlags::OMP_MAP_TARGET_PARAM
2200 : llvm::omp::OpenMPOffloadMappingFlags::OMP_MAP_NONE);
2201 combinedInfo.DevicePointers.emplace_back(
2204 mapData.
MapClause[mapDataIndex]->getLoc(), ompBuilder));
2205 combinedInfo.BasePointers.emplace_back(mapData.BasePointers[mapDataIndex]);
2206 combinedInfo.Pointers.emplace_back(mapData.Pointers[mapDataIndex]);
2215 llvm::Value *lowAddr = builder.CreatePointerCast(
2216 mapData.Pointers[mapDataIndex], builder.getPtrTy());
2217 llvm::Value *highAddr = builder.CreatePointerCast(
2218 builder.CreateConstGEP1_32(mapData.
BaseType[mapDataIndex],
2219 mapData.Pointers[mapDataIndex], 1),
2220 builder.getPtrTy());
2221 llvm::Value *size = builder.CreateIntCast(
2222 builder.CreatePtrDiff(builder.getInt8Ty(), highAddr, lowAddr),
2223 builder.getInt64Ty(),
2225 combinedInfo.Sizes.push_back(size);
2233 llvm::omp::OpenMPOffloadMappingFlags mapFlag =
2234 llvm::omp::OpenMPOffloadMappingFlags::OMP_MAP_TO;
2236 mapFlag &= ~
llvm::omp::OpenMPOffloadMappingFlags::OMP_MAP_TARGET_PARAM;
2238 llvm::omp::OpenMPOffloadMappingFlags memberOfFlag =
2239 ompBuilder.getMemberOfFlag(combinedInfo.BasePointers.size() - 1);
2240 ompBuilder.setCorrectMemberOfFlag(mapFlag, memberOfFlag);
2242 combinedInfo.Types.emplace_back(mapFlag);
2243 combinedInfo.DevicePointers.emplace_back(
2246 mapData.
MapClause[mapDataIndex]->getLoc(), ompBuilder));
2247 combinedInfo.BasePointers.emplace_back(mapData.BasePointers[mapDataIndex]);
2248 combinedInfo.Pointers.emplace_back(mapData.Pointers[mapDataIndex]);
2249 combinedInfo.Sizes.emplace_back(mapData.Sizes[mapDataIndex]);
2251 return memberOfFlag;
2263 if (mapOp.getVarPtrPtr())
2278 llvm::OpenMPIRBuilder &ompBuilder,
DataLayout &dl,
2280 uint64_t mapDataIndex, llvm::omp::OpenMPOffloadMappingFlags memberOfFlag) {
2283 mlir::dyn_cast<mlir::omp::MapInfoOp>(mapData.
MapClause[mapDataIndex]);
2285 for (
auto mappedMembers : parentClause.getMembers()) {
2287 mlir::dyn_cast<mlir::omp::MapInfoOp>(mappedMembers.getDefiningOp());
2288 int memberDataIdx = -1;
2289 for (
size_t i = 0; i < mapData.
MapClause.size(); ++i) {
2290 if (mapData.
MapClause[i] == memberClause)
2294 assert(memberDataIdx >= 0 &&
"could not find mapped member of structure");
2299 llvm::omp::OpenMPOffloadMappingFlags(memberClause.getMapType().value());
2300 mapFlag &= ~
llvm::omp::OpenMPOffloadMappingFlags::OMP_MAP_TARGET_PARAM;
2301 mapFlag |= llvm::omp::OpenMPOffloadMappingFlags::OMP_MAP_MEMBER_OF;
2302 ompBuilder.setCorrectMemberOfFlag(mapFlag, memberOfFlag);
2304 mapFlag |= llvm::omp::OpenMPOffloadMappingFlags::OMP_MAP_PTR_AND_OBJ;
2306 combinedInfo.Types.emplace_back(mapFlag);
2307 combinedInfo.DevicePointers.emplace_back(
2309 combinedInfo.Names.emplace_back(
2312 combinedInfo.BasePointers.emplace_back(mapData.BasePointers[memberDataIdx]);
2313 combinedInfo.Pointers.emplace_back(mapData.Pointers[memberDataIdx]);
2314 combinedInfo.Sizes.emplace_back(mapData.Sizes[memberDataIdx]);
2320 llvm::OpenMPIRBuilder &ompBuilder,
DataLayout &dl,
2322 uint64_t mapDataIndex,
bool isTargetParams) {
2323 llvm::omp::OpenMPOffloadMappingFlags memberOfParentFlag =
2325 combinedInfo, mapData, mapDataIndex, isTargetParams);
2327 combinedInfo, mapData, mapDataIndex,
2328 memberOfParentFlag);
2338 llvm::IRBuilderBase &builder) {
2339 for (
size_t i = 0; i < mapData.
MapClause.size(); ++i) {
2343 mlir::dyn_cast_if_present<mlir::omp::MapInfoOp>(mapData.
MapClause[i]);
2344 mlir::omp::VariableCaptureKind captureKind =
2345 mapOp.getMapCaptureType().value_or(
2346 mlir::omp::VariableCaptureKind::ByRef);
2356 switch (captureKind) {
2357 case mlir::omp::VariableCaptureKind::ByRef: {
2358 llvm::Value *newV = mapData.Pointers[i];
2360 moduleTranslation, builder, mapData.
BaseType[i]->isArrayTy(),
2363 newV = builder.CreateLoad(builder.getPtrTy(), newV);
2365 if (!offsetIdx.empty())
2366 newV = builder.CreateInBoundsGEP(mapData.
BaseType[i], newV, offsetIdx,
2368 mapData.Pointers[i] = newV;
2370 case mlir::omp::VariableCaptureKind::ByCopy: {
2371 llvm::Type *type = mapData.
BaseType[i];
2373 if (mapData.Pointers[i]->getType()->isPointerTy())
2374 newV = builder.CreateLoad(type, mapData.Pointers[i]);
2376 newV = mapData.Pointers[i];
2379 auto curInsert = builder.saveIP();
2381 auto *memTempAlloc =
2382 builder.CreateAlloca(builder.getPtrTy(),
nullptr,
".casted");
2383 builder.restoreIP(curInsert);
2385 builder.CreateStore(newV, memTempAlloc);
2386 newV = builder.CreateLoad(builder.getPtrTy(), memTempAlloc);
2389 mapData.Pointers[i] = newV;
2390 mapData.BasePointers[i] = newV;
2392 case mlir::omp::VariableCaptureKind::This:
2393 case mlir::omp::VariableCaptureKind::VLAType:
2394 mapData.
MapClause[i]->emitOpError(
"Unhandled capture kind");
2409 bool isTargetParams =
false) {
2426 auto fail = [&combinedInfo]() ->
void {
2427 combinedInfo.BasePointers.clear();
2428 combinedInfo.Pointers.clear();
2429 combinedInfo.DevicePointers.clear();
2430 combinedInfo.Sizes.clear();
2431 combinedInfo.Types.clear();
2432 combinedInfo.Names.clear();
2440 for (
size_t i = 0; i < mapData.
MapClause.size(); ++i) {
2450 auto mapInfoOp = mlir::dyn_cast<mlir::omp::MapInfoOp>(mapData.
MapClause[i]);
2451 if (!mapInfoOp.getMembers().empty()) {
2453 combinedInfo, mapData, i, isTargetParams);
2457 auto mapFlag = mapData.Types[i];
2460 mapFlag |= llvm::omp::OpenMPOffloadMappingFlags::OMP_MAP_PTR_AND_OBJ;
2465 mapFlag |= llvm::omp::OpenMPOffloadMappingFlags::OMP_MAP_TARGET_PARAM;
2467 if (
auto mapInfoOp = dyn_cast<mlir::omp::MapInfoOp>(mapData.
MapClause[i]))
2468 if (mapInfoOp.getMapCaptureType().value() ==
2469 mlir::omp::VariableCaptureKind::ByCopy &&
2471 mapFlag |= llvm::omp::OpenMPOffloadMappingFlags::OMP_MAP_LITERAL;
2473 combinedInfo.BasePointers.emplace_back(mapData.BasePointers[i]);
2474 combinedInfo.Pointers.emplace_back(mapData.Pointers[i]);
2475 combinedInfo.DevicePointers.emplace_back(mapData.DevicePointers[i]);
2476 combinedInfo.Names.emplace_back(mapData.Names[i]);
2477 combinedInfo.Types.emplace_back(mapFlag);
2478 combinedInfo.Sizes.emplace_back(mapData.Sizes[i]);
2481 auto findMapInfo = [&combinedInfo](llvm::Value *val,
unsigned &index) {
2483 for (llvm::Value *basePtr : combinedInfo.BasePointers) {
2491 auto addDevInfos = [&, fail](
auto devOperands,
auto devOpType) ->
void {
2492 for (
const auto &devOp : devOperands) {
2494 if (!isa<LLVM::LLVMPointerType>(devOp.getType()))
2497 llvm::Value *mapOpValue = moduleTranslation.
lookupValue(devOp);
2501 if (findMapInfo(mapOpValue, infoIndex)) {
2502 combinedInfo.Types[infoIndex] |=
2503 llvm::omp::OpenMPOffloadMappingFlags::OMP_MAP_RETURN_PARAM;
2504 combinedInfo.DevicePointers[infoIndex] = devOpType;
2506 combinedInfo.BasePointers.emplace_back(mapOpValue);
2507 combinedInfo.Pointers.emplace_back(mapOpValue);
2508 combinedInfo.DevicePointers.emplace_back(devOpType);
2509 combinedInfo.Names.emplace_back(
2511 combinedInfo.Types.emplace_back(
2512 llvm::omp::OpenMPOffloadMappingFlags::OMP_MAP_RETURN_PARAM);
2513 combinedInfo.Sizes.emplace_back(builder.getInt64(0));
2518 addDevInfos(devPtrOperands, llvm::OpenMPIRBuilder::DeviceInfoTy::Pointer);
2519 addDevInfos(devAddrOperands, llvm::OpenMPIRBuilder::DeviceInfoTy::Address);
2525 llvm::Value *ifCond =
nullptr;
2526 int64_t deviceID = llvm::omp::OMP_DEVICEID_UNDEF;
2530 llvm::omp::RuntimeFunction RTLFn;
2537 .Case([&](omp::TargetDataOp dataOp) {
2538 if (
auto ifExprVar = dataOp.getIfExpr())
2539 ifCond = moduleTranslation.
lookupValue(ifExprVar);
2541 if (
auto devId = dataOp.getDevice())
2543 dyn_cast<LLVM::ConstantOp>(devId.getDefiningOp()))
2544 if (
auto intAttr = dyn_cast<IntegerAttr>(constOp.getValue()))
2545 deviceID = intAttr.getInt();
2547 mapOperands = dataOp.getMapOperands();
2548 useDevPtrOperands = dataOp.getUseDevicePtr();
2549 useDevAddrOperands = dataOp.getUseDeviceAddr();
2552 .Case([&](omp::TargetEnterDataOp enterDataOp) {
2553 if (enterDataOp.getNowait())
2555 "`nowait` is not supported yet"));
2557 if (
auto ifExprVar = enterDataOp.getIfExpr())
2558 ifCond = moduleTranslation.
lookupValue(ifExprVar);
2560 if (
auto devId = enterDataOp.getDevice())
2562 dyn_cast<LLVM::ConstantOp>(devId.getDefiningOp()))
2563 if (
auto intAttr = dyn_cast<IntegerAttr>(constOp.getValue()))
2564 deviceID = intAttr.getInt();
2565 RTLFn = llvm::omp::OMPRTL___tgt_target_data_begin_mapper;
2566 mapOperands = enterDataOp.getMapOperands();
2569 .Case([&](omp::TargetExitDataOp exitDataOp) {
2570 if (exitDataOp.getNowait())
2572 "`nowait` is not supported yet"));
2574 if (
auto ifExprVar = exitDataOp.getIfExpr())
2575 ifCond = moduleTranslation.
lookupValue(ifExprVar);
2577 if (
auto devId = exitDataOp.getDevice())
2579 dyn_cast<LLVM::ConstantOp>(devId.getDefiningOp()))
2580 if (
auto intAttr = dyn_cast<IntegerAttr>(constOp.getValue()))
2581 deviceID = intAttr.getInt();
2583 RTLFn = llvm::omp::OMPRTL___tgt_target_data_end_mapper;
2584 mapOperands = exitDataOp.getMapOperands();
2587 .Case([&](omp::TargetUpdateOp updateDataOp) {
2588 if (updateDataOp.getNowait())
2590 "`nowait` is not supported yet"));
2592 if (
auto ifExprVar = updateDataOp.getIfExpr())
2593 ifCond = moduleTranslation.
lookupValue(ifExprVar);
2595 if (
auto devId = updateDataOp.getDevice())
2597 dyn_cast<LLVM::ConstantOp>(devId.getDefiningOp()))
2598 if (
auto intAttr = dyn_cast<IntegerAttr>(constOp.getValue()))
2599 deviceID = intAttr.getInt();
2601 RTLFn = llvm::omp::OMPRTL___tgt_target_data_update_mapper;
2602 mapOperands = updateDataOp.getMapOperands();
2606 return op->
emitError(
"unsupported OpenMP operation: ")
2613 using InsertPointTy = llvm::OpenMPIRBuilder::InsertPointTy;
2623 builder.restoreIP(codeGenIP);
2624 if (
auto dataOp = dyn_cast<omp::TargetDataOp>(op)) {
2625 genMapInfos(builder, moduleTranslation, DL, combinedInfo, mapData,
2626 useDevPtrOperands, useDevAddrOperands);
2628 genMapInfos(builder, moduleTranslation, DL, combinedInfo, mapData);
2630 return combinedInfo;
2633 llvm::OpenMPIRBuilder::TargetDataInfo info(
true,
2636 using BodyGenTy = llvm::OpenMPIRBuilder::BodyGenTy;
2638 auto bodyGenCB = [&](InsertPointTy codeGenIP, BodyGenTy bodyGenType) {
2639 assert(isa<omp::TargetDataOp>(op) &&
2640 "BodyGen requested for non TargetDataOp");
2641 Region ®ion = cast<omp::TargetDataOp>(op).getRegion();
2642 switch (bodyGenType) {
2643 case BodyGenTy::Priv:
2645 if (!info.DevicePtrInfoMap.empty()) {
2646 builder.restoreIP(codeGenIP);
2647 unsigned argIndex = 0;
2648 for (
auto &devPtrOp : useDevPtrOperands) {
2649 llvm::Value *mapOpValue = moduleTranslation.
lookupValue(devPtrOp);
2652 info.DevicePtrInfoMap[mapOpValue].second);
2656 for (
auto &devAddrOp : useDevAddrOperands) {
2657 llvm::Value *mapOpValue = moduleTranslation.
lookupValue(devAddrOp);
2659 auto *LI = builder.CreateLoad(
2660 builder.getPtrTy(), info.DevicePtrInfoMap[mapOpValue].second);
2661 moduleTranslation.
mapValue(arg, LI);
2666 builder, moduleTranslation);
2669 case BodyGenTy::DupNoPriv:
2671 case BodyGenTy::NoPriv:
2673 if (info.DevicePtrInfoMap.empty()) {
2674 builder.restoreIP(codeGenIP);
2676 builder, moduleTranslation);
2680 return builder.saveIP();
2683 llvm::OpenMPIRBuilder::LocationDescription ompLoc(builder);
2684 llvm::OpenMPIRBuilder::InsertPointTy allocaIP =
2686 if (isa<omp::TargetDataOp>(op)) {
2687 builder.restoreIP(ompBuilder->createTargetData(
2688 ompLoc, allocaIP, builder.saveIP(), builder.getInt64(deviceID), ifCond,
2689 info, genMapInfoCB,
nullptr, bodyGenCB));
2691 builder.restoreIP(ompBuilder->createTargetData(
2692 ompLoc, allocaIP, builder.saveIP(), builder.getInt64(deviceID), ifCond,
2693 info, genMapInfoCB, &RTLFn));
2696 return bodyGenStatus;
2704 if (!cast<mlir::ModuleOp>(op))
2709 ompBuilder->M.addModuleFlag(llvm::Module::Max,
"openmp-device",
2710 attribute.getOpenmpDeviceVersion());
2712 if (attribute.getNoGpuLib())
2715 ompBuilder->createGlobalFlag(
2716 attribute.getDebugKind() ,
2717 "__omp_rtl_debug_kind");
2718 ompBuilder->createGlobalFlag(
2720 .getAssumeTeamsOversubscription()
2722 "__omp_rtl_assume_teams_oversubscription");
2723 ompBuilder->createGlobalFlag(
2725 .getAssumeThreadsOversubscription()
2727 "__omp_rtl_assume_threads_oversubscription");
2728 ompBuilder->createGlobalFlag(
2729 attribute.getAssumeNoThreadState() ,
2730 "__omp_rtl_assume_no_thread_state");
2731 ompBuilder->createGlobalFlag(
2733 .getAssumeNoNestedParallelism()
2735 "__omp_rtl_assume_no_nested_parallelism");
2740 omp::TargetOp targetOp,
2741 llvm::StringRef parentName =
"") {
2742 auto fileLoc = targetOp.getLoc()->findInstanceOf<FileLineColLoc>();
2744 assert(fileLoc &&
"No file found from location");
2745 StringRef fileName = fileLoc.getFilename().getValue();
2747 llvm::sys::fs::UniqueID id;
2748 if (
auto ec = llvm::sys::fs::getUniqueID(fileName,
id)) {
2749 targetOp.emitError(
"Unable to get unique ID for file");
2753 uint64_t line = fileLoc.getLine();
2754 targetInfo = llvm::TargetRegionEntryInfo(parentName,
id.getDevice(),
2755 id.getFile(), line);
2760 auto targetOp = cast<omp::TargetOp>(opInst);
2761 if (targetOp.getIfExpr()) {
2762 opInst.
emitError(
"If clause not yet supported");
2766 if (targetOp.getDevice()) {
2767 opInst.
emitError(
"Device clause not yet supported");
2771 if (targetOp.getThreadLimit()) {
2772 opInst.
emitError(
"Thread limit clause not yet supported");
2776 if (targetOp.getNowait()) {
2777 opInst.
emitError(
"Nowait clause not yet supported");
2787 llvm::IRBuilderBase &builder) {
2788 for (
size_t i = 0; i < mapData.
MapClause.size(); ++i) {
2807 userVec.push_back(user);
2809 for (llvm::User *user : userVec) {
2810 if (
auto *insn = dyn_cast<llvm::Instruction>(user)) {
2811 auto *load = builder.CreateLoad(mapData.BasePointers[i]->getType(),
2812 mapData.BasePointers[i]);
2813 load->moveBefore(insn);
2860 static llvm::IRBuilderBase::InsertPoint
2862 llvm::Value *input, llvm::Value *&retVal,
2863 llvm::IRBuilderBase &builder,
2864 llvm::OpenMPIRBuilder &ompBuilder,
2866 llvm::IRBuilderBase::InsertPoint allocaIP,
2867 llvm::IRBuilderBase::InsertPoint codeGenIP) {
2868 builder.restoreIP(allocaIP);
2870 mlir::omp::VariableCaptureKind capture =
2871 mlir::omp::VariableCaptureKind::ByRef;
2874 for (
size_t i = 0; i < mapData.
MapClause.size(); ++i)
2876 if (
auto mapOp = mlir::dyn_cast_if_present<mlir::omp::MapInfoOp>(
2878 capture = mapOp.getMapCaptureType().value_or(
2879 mlir::omp::VariableCaptureKind::ByRef);
2885 unsigned int allocaAS = ompBuilder.M.getDataLayout().getAllocaAddrSpace();
2886 unsigned int defaultAS =
2887 ompBuilder.M.getDataLayout().getProgramAddressSpace();
2890 llvm::Value *v = builder.CreateAlloca(arg.getType(), allocaAS);
2892 if (allocaAS != defaultAS && arg.getType()->isPointerTy())
2893 v = builder.CreatePointerBitCastOrAddrSpaceCast(
2894 v, arg.getType()->getPointerTo(defaultAS));
2896 builder.CreateStore(&arg, v);
2898 builder.restoreIP(codeGenIP);
2901 case mlir::omp::VariableCaptureKind::ByCopy: {
2905 case mlir::omp::VariableCaptureKind::ByRef: {
2906 retVal = builder.CreateAlignedLoad(
2908 ompBuilder.M.getDataLayout().getPrefTypeAlign(v->getType()));
2911 case mlir::omp::VariableCaptureKind::This:
2912 case mlir::omp::VariableCaptureKind::VLAType:
2913 assert(
false &&
"Currently unsupported capture kind");
2917 return builder.saveIP();
2928 auto targetOp = cast<omp::TargetOp>(opInst);
2929 auto &targetRegion = targetOp.getRegion();
2934 using InsertPointTy = llvm::OpenMPIRBuilder::InsertPointTy;
2935 auto bodyCB = [&](InsertPointTy allocaIP,
2936 InsertPointTy codeGenIP) -> InsertPointTy {
2939 llvm::Function *llvmParentFn =
2941 llvm::Function *llvmOutlinedFn = codeGenIP.getBlock()->getParent();
2942 assert(llvmParentFn && llvmOutlinedFn &&
2943 "Both parent and outlined functions must exist at this point");
2945 if (
auto attr = llvmParentFn->getFnAttribute(
"target-cpu");
2946 attr.isStringAttribute())
2947 llvmOutlinedFn->addFnAttr(attr);
2949 if (
auto attr = llvmParentFn->getFnAttribute(
"target-features");
2950 attr.isStringAttribute())
2951 llvmOutlinedFn->addFnAttr(attr);
2953 builder.restoreIP(codeGenIP);
2954 unsigned argIndex = 0;
2955 for (
auto &mapOp : mapOperands) {
2957 mlir::dyn_cast<mlir::omp::MapInfoOp>(mapOp.getDefiningOp());
2958 llvm::Value *mapOpValue =
2959 moduleTranslation.
lookupValue(mapInfoOp.getVarPtr());
2960 const auto &arg = targetRegion.front().getArgument(argIndex);
2961 moduleTranslation.
mapValue(arg, mapOpValue);
2965 targetRegion,
"omp.target", builder, moduleTranslation, bodyGenStatus);
2966 builder.SetInsertPoint(exitBlock);
2967 return builder.saveIP();
2970 llvm::OpenMPIRBuilder::LocationDescription ompLoc(builder);
2971 StringRef parentName = parentFn.getName();
2973 llvm::TargetRegionEntryInfo entryInfo;
2978 int32_t defaultValTeams = -1;
2979 int32_t defaultValThreads = 0;
2981 llvm::OpenMPIRBuilder::InsertPointTy allocaIP =
2989 auto genMapInfoCB = [&](llvm::OpenMPIRBuilder::InsertPointTy codeGenIP)
2991 builder.restoreIP(codeGenIP);
2992 genMapInfos(builder, moduleTranslation, dl, combinedInfos, mapData, {}, {},
2994 return combinedInfos;
2997 auto argAccessorCB = [&](llvm::Argument &arg, llvm::Value *input,
2998 llvm::Value *&retVal, InsertPointTy allocaIP,
2999 InsertPointTy codeGenIP) {
3007 if (!ompBuilder->Config.isTargetDevice()) {
3008 retVal = cast<llvm::Value>(&arg);
3013 *ompBuilder, moduleTranslation,
3014 allocaIP, codeGenIP);
3018 for (
size_t i = 0; i < mapOperands.size(); ++i) {
3030 ompLoc, allocaIP, builder.saveIP(), entryInfo, defaultValTeams,
3031 defaultValThreads, kernelInput, genMapInfoCB, bodyCB, argAccessorCB));
3038 return bodyGenStatus;
3051 if (FunctionOpInterface funcOp = dyn_cast<FunctionOpInterface>(op)) {
3052 if (
auto offloadMod = dyn_cast<omp::OffloadModuleInterface>(
3054 if (!offloadMod.getIsTargetDevice())
3057 omp::DeclareTargetDeviceType declareType =
3058 attribute.getDeviceType().getValue();
3060 if (declareType == omp::DeclareTargetDeviceType::host) {
3061 llvm::Function *llvmFunc =
3063 llvmFunc->dropAllReferences();
3064 llvmFunc->eraseFromParent();
3070 if (LLVM::GlobalOp gOp = dyn_cast<LLVM::GlobalOp>(op)) {
3071 llvm::Module *llvmModule = moduleTranslation.
getLLVMModule();
3072 if (
auto *gVal = llvmModule->getNamedValue(gOp.getSymName())) {
3074 bool isDeclaration = gOp.isDeclaration();
3075 bool isExternallyVisible =
3078 llvm::StringRef mangledName = gOp.getSymName();
3079 auto captureClause =
3085 std::vector<llvm::GlobalVariable *> generatedRefs;
3087 std::vector<llvm::Triple> targetTriple;
3088 auto targetTripleAttr = dyn_cast_or_null<mlir::StringAttr>(
3090 LLVM::LLVMDialect::getTargetTripleAttrName()));
3091 if (targetTripleAttr)
3092 targetTriple.emplace_back(targetTripleAttr.data());
3094 auto fileInfoCallBack = [&loc]() {
3095 std::string filename =
"";
3096 std::uint64_t lineNo = 0;
3099 filename = loc.getFilename().str();
3100 lineNo = loc.getLine();
3103 return std::pair<std::string, std::uint64_t>(llvm::StringRef(filename),
3107 ompBuilder->registerTargetGlobalVariable(
3108 captureClause, deviceClause, isDeclaration, isExternallyVisible,
3109 ompBuilder->getTargetEntryUniqueInfo(fileInfoCallBack), mangledName,
3110 generatedRefs,
false, targetTriple,
3112 gVal->getType(), gVal);
3114 if (ompBuilder->Config.isTargetDevice() &&
3115 (attribute.getCaptureClause().getValue() !=
3116 mlir::omp::DeclareTargetCaptureClause::to ||
3117 ompBuilder->Config.hasRequiresUnifiedSharedMemory())) {
3118 ompBuilder->getAddrOfDeclareTargetVar(
3119 captureClause, deviceClause, isDeclaration, isExternallyVisible,
3120 ompBuilder->getTargetEntryUniqueInfo(fileInfoCallBack), mangledName,
3121 generatedRefs,
false, targetTriple, gVal->getType(),
3139 if (
auto declareTargetIface =
3140 llvm::dyn_cast<mlir::omp::DeclareTargetInterface>(
3141 parentFn.getOperation()))
3142 if (declareTargetIface.isDeclareTarget() &&
3143 declareTargetIface.getDeclareTargetDeviceType() !=
3144 mlir::omp::DeclareTargetDeviceType::host)
3159 .Case([&](omp::BarrierOp) {
3160 ompBuilder->createBarrier(builder.saveIP(), llvm::omp::OMPD_barrier);
3163 .Case([&](omp::TaskwaitOp) {
3164 ompBuilder->createTaskwait(builder.saveIP());
3167 .Case([&](omp::TaskyieldOp) {
3168 ompBuilder->createTaskyield(builder.saveIP());
3171 .Case([&](omp::FlushOp) {
3180 ompBuilder->createFlush(builder.saveIP());
3183 .Case([&](omp::ParallelOp op) {
3186 .Case([&](omp::ReductionOp reductionOp) {
3189 .Case([&](omp::MasterOp) {
3192 .Case([&](omp::CriticalOp) {
3195 .Case([&](omp::OrderedRegionOp) {
3198 .Case([&](omp::OrderedOp) {
3201 .Case([&](omp::WsloopOp) {
3204 .Case([&](omp::SimdOp) {
3207 .Case([&](omp::AtomicReadOp) {
3210 .Case([&](omp::AtomicWriteOp) {
3213 .Case([&](omp::AtomicUpdateOp op) {
3216 .Case([&](omp::AtomicCaptureOp op) {
3219 .Case([&](omp::SectionsOp) {
3222 .Case([&](omp::SingleOp op) {
3225 .Case([&](omp::TeamsOp op) {
3228 .Case([&](omp::TaskOp op) {
3231 .Case([&](omp::TaskgroupOp op) {
3234 .Case<omp::YieldOp, omp::TerminatorOp, omp::DeclareReductionOp,
3235 omp::CriticalDeclareOp>([](
auto op) {
3246 .Case([&](omp::ThreadprivateOp) {
3249 .Case<omp::TargetDataOp, omp::TargetEnterDataOp, omp::TargetExitDataOp,
3250 omp::TargetUpdateOp>([&](
auto op) {
3253 .Case([&](omp::TargetOp) {
3256 .Case<omp::MapInfoOp, omp::MapBoundsOp, omp::PrivateClauseOp>(
3264 return inst->
emitError(
"unsupported OpenMP operation: ")
3278 if (isa<omp::TargetOp>(op))
3280 if (isa<omp::TargetDataOp>(op))
3284 if (isa<omp::TargetOp>(oper)) {
3289 if (isa<omp::TargetDataOp>(oper)) {
3295 }).wasInterrupted();
3303 class OpenMPDialectLLVMIRTranslationInterface
3311 convertOperation(
Operation *op, llvm::IRBuilderBase &builder,
3324 LogicalResult OpenMPDialectLLVMIRTranslationInterface::amendOperation(
3330 .Case(
"omp.is_target_device",
3332 if (
auto deviceAttr = dyn_cast<BoolAttr>(attr)) {
3333 llvm::OpenMPIRBuilderConfig &config =
3335 config.setIsTargetDevice(deviceAttr.getValue());
3342 if (
auto gpuAttr = dyn_cast<BoolAttr>(attr)) {
3343 llvm::OpenMPIRBuilderConfig &config =
3345 config.setIsGPU(gpuAttr.getValue());
3350 .Case(
"omp.host_ir_filepath",
3352 if (
auto filepathAttr = dyn_cast<StringAttr>(attr)) {
3353 llvm::OpenMPIRBuilder *ompBuilder =
3355 ompBuilder->loadOffloadInfoMetadata(filepathAttr.getValue());
3362 if (
auto rtlAttr = dyn_cast<omp::FlagsAttr>(attr))
3366 .Case(
"omp.version",
3368 if (
auto versionAttr = dyn_cast<omp::VersionAttr>(attr)) {
3369 llvm::OpenMPIRBuilder *ompBuilder =
3371 ompBuilder->M.addModuleFlag(llvm::Module::Max,
"openmp",
3372 versionAttr.getVersion());
3377 .Case(
"omp.declare_target",
3379 if (
auto declareTargetAttr =
3380 dyn_cast<omp::DeclareTargetAttr>(attr))
3385 .Case(
"omp.requires",
3387 if (
auto requiresAttr = dyn_cast<omp::ClauseRequiresAttr>(attr)) {
3388 using Requires = omp::ClauseRequires;
3389 Requires flags = requiresAttr.getValue();
3390 llvm::OpenMPIRBuilderConfig &config =
3392 config.setHasRequiresReverseOffload(
3393 bitEnumContainsAll(flags, Requires::reverse_offload));
3394 config.setHasRequiresUnifiedAddress(
3395 bitEnumContainsAll(flags, Requires::unified_address));
3396 config.setHasRequiresUnifiedSharedMemory(
3397 bitEnumContainsAll(flags, Requires::unified_shared_memory));
3398 config.setHasRequiresDynamicAllocators(
3399 bitEnumContainsAll(flags, Requires::dynamic_allocators));
3414 LogicalResult OpenMPDialectLLVMIRTranslationInterface::convertOperation(
3415 Operation *op, llvm::IRBuilderBase &builder,
3419 if (ompBuilder->Config.isTargetDevice()) {
3430 registry.
insert<omp::OpenMPDialect>();
3432 dialect->addInterfaces<OpenMPDialectLLVMIRTranslationInterface>();
static llvm::Value * getRefPtrIfDeclareTarget(mlir::Value value, LLVM::ModuleTranslation &moduleTranslation)
static LogicalResult convertOmpAtomicUpdate(omp::AtomicUpdateOp &opInst, llvm::IRBuilderBase &builder, LLVM::ModuleTranslation &moduleTranslation)
Converts an OpenMP atomic update operation using OpenMPIRBuilder.
static LogicalResult convertOmpAtomicCapture(omp::AtomicCaptureOp atomicCaptureOp, llvm::IRBuilderBase &builder, LLVM::ModuleTranslation &moduleTranslation)
static LogicalResult convertOmpOrderedRegion(Operation &opInst, llvm::IRBuilderBase &builder, LLVM::ModuleTranslation &moduleTranslation)
Converts an OpenMP 'ordered_region' operation into LLVM IR using OpenMPIRBuilder.
static llvm::OpenMPIRBuilder::InsertPointTy findAllocaInsertPoint(llvm::IRBuilderBase &builder, const LLVM::ModuleTranslation &moduleTranslation)
Find the insertion point for allocas given the current insertion point for normal operations in the b...
static LogicalResult convertOmpAtomicWrite(Operation &opInst, llvm::IRBuilderBase &builder, LLVM::ModuleTranslation &moduleTranslation)
Converts an omp.atomic.write operation to LLVM IR.
static OwningAtomicReductionGen makeAtomicReductionGen(omp::DeclareReductionOp decl, llvm::IRBuilderBase &builder, LLVM::ModuleTranslation &moduleTranslation)
Create an OpenMPIRBuilder-compatible atomic reduction generator for the given reduction declaration.
static LogicalResult convertOmpWsloop(Operation &opInst, llvm::IRBuilderBase &builder, LLVM::ModuleTranslation &moduleTranslation)
Converts an OpenMP workshare loop into LLVM IR using OpenMPIRBuilder.
static LogicalResult convertOmpMaster(Operation &opInst, llvm::IRBuilderBase &builder, LLVM::ModuleTranslation &moduleTranslation)
Converts an OpenMP 'master' operation into LLVM IR using OpenMPIRBuilder.
static LogicalResult convertOmpReductionOp(omp::ReductionOp reductionOp, llvm::IRBuilderBase &builder, LLVM::ModuleTranslation &moduleTranslation)
Converts an OpenMP reduction operation using OpenMPIRBuilder.
static void createAlteredByCaptureMap(MapInfoData &mapData, LLVM::ModuleTranslation &moduleTranslation, llvm::IRBuilderBase &builder)
static LogicalResult convertOmpTaskOp(omp::TaskOp taskOp, llvm::IRBuilderBase &builder, LLVM::ModuleTranslation &moduleTranslation)
Converts an OpenMP task construct into LLVM IR using OpenMPIRBuilder.
llvm::AtomicRMWInst::BinOp convertBinOpToAtomic(Operation &op)
Converts an LLVM dialect binary operation to the corresponding enum value for atomicrmw supported bin...
static llvm::AtomicOrdering convertAtomicOrdering(std::optional< omp::ClauseMemoryOrderKind > ao)
Convert an Atomic Ordering attribute to llvm::AtomicOrdering.
static LogicalResult convertOmpSections(Operation &opInst, llvm::IRBuilderBase &builder, LLVM::ModuleTranslation &moduleTranslation)
static LogicalResult convertOmpCritical(Operation &opInst, llvm::IRBuilderBase &builder, LLVM::ModuleTranslation &moduleTranslation)
Converts an OpenMP 'critical' operation into LLVM IR using OpenMPIRBuilder.
static void genMapInfos(llvm::IRBuilderBase &builder, LLVM::ModuleTranslation &moduleTranslation, DataLayout &dl, llvm::OpenMPIRBuilder::MapInfosTy &combinedInfo, MapInfoData &mapData, const SmallVector< Value > &devPtrOperands={}, const SmallVector< Value > &devAddrOperands={}, bool isTargetParams=false)
static LogicalResult convertOmpTargetData(Operation *op, llvm::IRBuilderBase &builder, LLVM::ModuleTranslation &moduleTranslation)
static LogicalResult inlineReductionCleanup(llvm::SmallVectorImpl< omp::DeclareReductionOp > &reductionDecls, llvm::ArrayRef< llvm::Value * > privateReductionVariables, LLVM::ModuleTranslation &moduleTranslation, llvm::IRBuilderBase &builder)
handling of DeclareReductionOp's cleanup region
static LogicalResult convertOmpSimd(Operation &opInst, llvm::IRBuilderBase &builder, LLVM::ModuleTranslation &moduleTranslation)
Converts an OpenMP simd loop into LLVM IR using OpenMPIRBuilder.
static LogicalResult convertOmpOrdered(Operation &opInst, llvm::IRBuilderBase &builder, LLVM::ModuleTranslation &moduleTranslation)
Converts an OpenMP 'ordered' operation into LLVM IR using OpenMPIRBuilder.
static bool isTargetDeviceOp(Operation *op)
std::vector< llvm::Value * > calculateBoundsOffset(LLVM::ModuleTranslation &moduleTranslation, llvm::IRBuilderBase &builder, bool isArrayTy, mlir::OperandRange bounds)
This function calculates the array/pointer offset for map data provided with bounds operations,...
void collectMapDataFromMapOperands(MapInfoData &mapData, llvm::SmallVectorImpl< Value > &mapOperands, LLVM::ModuleTranslation &moduleTranslation, DataLayout &dl, llvm::IRBuilderBase &builder)
static void collectReductionInfo(T loop, llvm::IRBuilderBase &builder, LLVM::ModuleTranslation &moduleTranslation, SmallVector< omp::DeclareReductionOp > &reductionDecls, SmallVector< OwningReductionGen > &owningReductionGens, SmallVector< OwningAtomicReductionGen > &owningAtomicReductionGens, const SmallVector< llvm::Value * > &privateReductionVariables, SmallVector< llvm::OpenMPIRBuilder::ReductionInfo > &reductionInfos)
Collect reduction info.
static std::optional< omp::DeclareReductionOp > findReductionDeclInContainer(T container, omp::ReductionOp reduction)
Returns a reduction declaration that corresponds to the given reduction operation in the given contai...
static llvm::SmallString< 64 > getDeclareTargetRefPtrSuffix(LLVM::GlobalOp globalOp, llvm::OpenMPIRBuilder &ompBuilder)
static OwningReductionGen makeReductionGen(omp::DeclareReductionOp decl, llvm::IRBuilderBase &builder, LLVM::ModuleTranslation &moduleTranslation)
Create an OpenMPIRBuilder-compatible reduction generator for the given reduction declaration.
static LogicalResult convertOmpParallel(omp::ParallelOp opInst, llvm::IRBuilderBase &builder, LLVM::ModuleTranslation &moduleTranslation)
Converts the OpenMP parallel operation to LLVM IR.
static llvm::omp::OpenMPOffloadMappingFlags mapParentWithMembers(LLVM::ModuleTranslation &moduleTranslation, llvm::IRBuilderBase &builder, llvm::OpenMPIRBuilder &ompBuilder, DataLayout &dl, llvm::OpenMPIRBuilder::MapInfosTy &combinedInfo, MapInfoData &mapData, uint64_t mapDataIndex, bool isTargetParams)
static LogicalResult inlineConvertOmpRegions(Region ®ion, StringRef blockName, llvm::IRBuilderBase &builder, LLVM::ModuleTranslation &moduleTranslation, SmallVectorImpl< llvm::Value * > *continuationBlockArgs=nullptr)
Translates the blocks contained in the given region and appends them to at the current insertion poin...
static LogicalResult convertOmpThreadprivate(Operation &opInst, llvm::IRBuilderBase &builder, LLVM::ModuleTranslation &moduleTranslation)
Converts an OpenMP Threadprivate operation into LLVM IR using OpenMPIRBuilder.
static LogicalResult convertDeclareTargetAttr(Operation *op, mlir::omp::DeclareTargetAttr attribute, LLVM::ModuleTranslation &moduleTranslation)
LogicalResult convertFlagsAttr(Operation *op, mlir::omp::FlagsAttr attribute, LLVM::ModuleTranslation &moduleTranslation)
Lowers the FlagsAttr which is applied to the module on the device pass when offloading,...
static llvm::OffloadEntriesInfoManager::OMPTargetGlobalVarEntryKind convertToCaptureClauseKind(mlir::omp::DeclareTargetCaptureClause captureClasue)
uint64_t getArrayElementSizeInBits(LLVM::LLVMArrayType arrTy, DataLayout &dl)
static void mapInitializationArg(T loop, LLVM::ModuleTranslation &moduleTranslation, SmallVectorImpl< omp::DeclareReductionOp > &reductionDecls, unsigned i)
Map input argument to all reduction initialization regions.
static void handleDeclareTargetMapVar(MapInfoData &mapData, LLVM::ModuleTranslation &moduleTranslation, llvm::IRBuilderBase &builder)
static bool targetOpSupported(Operation &opInst)
static llvm::IRBuilderBase::InsertPoint createDeviceArgumentAccessor(MapInfoData &mapData, llvm::Argument &arg, llvm::Value *input, llvm::Value *&retVal, llvm::IRBuilderBase &builder, llvm::OpenMPIRBuilder &ompBuilder, LLVM::ModuleTranslation &moduleTranslation, llvm::IRBuilderBase::InsertPoint allocaIP, llvm::IRBuilderBase::InsertPoint codeGenIP)
static llvm::omp::ProcBindKind getProcBindKind(omp::ClauseProcBindKind kind)
Convert ProcBindKind from MLIR-generated enum to LLVM enum.
static LogicalResult convertTargetOpsInNest(Operation *op, llvm::IRBuilderBase &builder, LLVM::ModuleTranslation &moduleTranslation)
static void processMapMembersWithParent(LLVM::ModuleTranslation &moduleTranslation, llvm::IRBuilderBase &builder, llvm::OpenMPIRBuilder &ompBuilder, DataLayout &dl, llvm::OpenMPIRBuilder::MapInfosTy &combinedInfo, MapInfoData &mapData, uint64_t mapDataIndex, llvm::omp::OpenMPOffloadMappingFlags memberOfFlag)
static LogicalResult convertOmpSingle(omp::SingleOp &singleOp, llvm::IRBuilderBase &builder, LLVM::ModuleTranslation &moduleTranslation)
Converts an OpenMP single construct into LLVM IR using OpenMPIRBuilder.
static void processMapWithMembersOf(LLVM::ModuleTranslation &moduleTranslation, llvm::IRBuilderBase &builder, llvm::OpenMPIRBuilder &ompBuilder, DataLayout &dl, llvm::OpenMPIRBuilder::MapInfosTy &combinedInfo, MapInfoData &mapData, uint64_t mapDataIndex, bool isTargetParams)
static LogicalResult convertOmpTarget(Operation &opInst, llvm::IRBuilderBase &builder, LLVM::ModuleTranslation &moduleTranslation)
static llvm::OffloadEntriesInfoManager::OMPTargetDeviceClauseKind convertToDeviceClauseKind(mlir::omp::DeclareTargetDeviceType deviceClause)
static void collectReductionDecls(T loop, SmallVectorImpl< omp::DeclareReductionOp > &reductions)
Populates reductions with reduction declarations used in the given loop.
static omp::DeclareReductionOp findReductionDecl(Operation &containerOp, omp::ReductionOp reduction)
Searches for a reduction in a provided region and the regions it is nested in.
static LogicalResult convertOmpAtomicRead(Operation &opInst, llvm::IRBuilderBase &builder, LLVM::ModuleTranslation &moduleTranslation)
Convert omp.atomic.read operation to LLVM IR.
static bool getTargetEntryUniqueInfo(llvm::TargetRegionEntryInfo &targetInfo, omp::TargetOp targetOp, llvm::StringRef parentName="")
static LogicalResult convertOmpTeams(omp::TeamsOp op, llvm::IRBuilderBase &builder, LLVM::ModuleTranslation &moduleTranslation)
static LogicalResult convertHostOrTargetOperation(Operation *op, llvm::IRBuilderBase &builder, LLVM::ModuleTranslation &moduleTranslation)
Given an OpenMP MLIR operation, create the corresponding LLVM IR (including OpenMP runtime calls).
static void allocByValReductionVars(T loop, llvm::IRBuilderBase &builder, LLVM::ModuleTranslation &moduleTranslation, llvm::OpenMPIRBuilder::InsertPointTy &allocaIP, SmallVectorImpl< omp::DeclareReductionOp > &reductionDecls, SmallVectorImpl< llvm::Value * > &privateReductionVariables, DenseMap< Value, llvm::Value * > &reductionVariableMap)
Allocate space for privatized reduction variables.
static bool checkIfPointerMap(mlir::omp::MapInfoOp mapOp)
static LogicalResult convertTargetDeviceOp(Operation *op, llvm::IRBuilderBase &builder, LLVM::ModuleTranslation &moduleTranslation)
static llvm::BasicBlock * convertOmpOpRegions(Region ®ion, StringRef blockName, llvm::IRBuilderBase &builder, LLVM::ModuleTranslation &moduleTranslation, LogicalResult &bodyGenStatus, SmallVectorImpl< llvm::PHINode * > *continuationBlockPHIs=nullptr)
Converts the given region that appears within an OpenMP dialect operation to LLVM IR,...
static LogicalResult convertOmpTaskgroupOp(omp::TaskgroupOp tgOp, llvm::IRBuilderBase &builder, LLVM::ModuleTranslation &moduleTranslation)
Converts an OpenMP taskgroup construct into LLVM IR using OpenMPIRBuilder.
llvm::Value * getSizeInBytes(DataLayout &dl, const mlir::Type &type, Operation *clauseOp, llvm::Value *basePointer, llvm::Type *baseType, llvm::IRBuilderBase &builder, LLVM::ModuleTranslation &moduleTranslation)
static bool isDeclareTargetLink(mlir::Value value)
#define MLIR_DEFINE_EXPLICIT_INTERNAL_INLINE_TYPE_ID(CLASS_NAME)
A RAII class that on construction replaces the region arguments of the parallel op (which correspond ...
~OmpParallelOpConversionManager()
OmpParallelOpConversionManager(omp::ParallelOp opInst)
Attributes are known-constant values of operations.
Block represents an ordered list of Operations.
BlockArgument getArgument(unsigned i)
unsigned getNumArguments()
Operation * getTerminator()
Get the terminator operation of this block.
The main mechanism for performing data layout queries.
llvm::TypeSize getTypeSizeInBits(Type t) const
Returns the size in bits of the given type in the current scope.
The DialectRegistry maps a dialect namespace to a constructor for the matching dialect.
void addExtension(std::unique_ptr< DialectExtensionBase > extension)
Add the given extension to the registry.
This class coordinates rewriting a piece of IR outside of a pattern rewrite, providing a way to keep ...
Base class for dialect interfaces providing translation to LLVM IR.
LLVMTranslationDialectInterface(Dialect *dialect)
Concrete CRTP base class for ModuleTranslation stack frames.
Implementation class for module translation.
llvm::Value * lookupValue(Value value) const
Finds an LLVM IR value corresponding to the given MLIR value.
LogicalResult convertBlock(Block &bb, bool ignoreArguments, llvm::IRBuilderBase &builder)
Translates the contents of the given block to LLVM IR using this translator.
SmallVector< llvm::Value * > lookupValues(ValueRange values)
Looks up remapped a list of remapped values.
llvm::BasicBlock * lookupBlock(Block *block) const
Finds an LLVM IR basic block that corresponds to the given MLIR block.
SymbolTableCollection & symbolTable()
llvm::Type * convertType(Type type)
Converts the type from MLIR LLVM dialect to LLVM.
llvm::OpenMPIRBuilder * getOpenMPBuilder()
Returns the OpenMP IR builder associated with the LLVM IR module being constructed.
llvm::LLVMContext & getLLVMContext() const
Returns the LLVM context in which the IR is being constructed.
llvm::GlobalValue * lookupGlobal(Operation *op)
Finds an LLVM IR global value that corresponds to the given MLIR operation defining a global value.
llvm::Module * getLLVMModule()
Returns the LLVM module in which the IR is being constructed.
llvm::Function * lookupFunction(StringRef name) const
Finds an LLVM IR function by its name.
void mapBlock(Block *mlir, llvm::BasicBlock *llvm)
Stores the mapping between an MLIR block and LLVM IR basic block.
WalkResult stackWalk(llvm::function_ref< WalkResult(const T &)> callback) const
Calls callback for every ModuleTranslation stack frame of type T starting from the top of the stack.
void forgetMapping(Region ®ion)
Removes the mapping for blocks contained in the region and values defined in these blocks.
MLIRContext & getContext()
Returns the MLIR context of the module being translated.
void mapValue(Value mlir, llvm::Value *llvm)
Stores the mapping between an MLIR value and its LLVM IR counterpart.
T findInstanceOf()
Return an instance of the given location type if one is nested under the current location.
MLIRContext is the top-level object for a collection of MLIR operations.
void appendDialectRegistry(const DialectRegistry ®istry)
Append the contents of the given dialect registry to the registry associated with this context.
NamedAttribute represents a combination of a name and an Attribute value.
StringAttr getName() const
Return the name of the attribute.
Attribute getValue() const
Return the value of the attribute.
void cloneRegionBefore(Region ®ion, Region &parent, Region::iterator before, IRMapping &mapping)
Clone the blocks that belong to "region" before the given position in another region "parent".
This class implements the operand iterators for the Operation class.
Operation is the basic unit of execution within MLIR.
Value getOperand(unsigned idx)
OpResult getResult(unsigned idx)
Get the 'idx'th result of this operation.
std::enable_if_t< llvm::function_traits< std::decay_t< FnT > >::num_args==1, RetT > walk(FnT &&callback)
Walk the operation by calling the callback for each nested operation (including this one),...
Location getLoc()
The source location the operation was defined or derived from.
unsigned getNumOperands()
Operation * getParentOp()
Returns the closest surrounding operation that contains this operation or nullptr if this is a top-le...
InFlightDiagnostic emitError(const Twine &message={})
Emit an error about fatal conditions with this operation, reporting up to any diagnostic handlers tha...
OpTy getParentOfType()
Return the closest surrounding parent operation that is of type 'OpTy'.
Region & getRegion(unsigned index)
Returns the region held by this operation at position 'index'.
OperationName getName()
The name of an operation is the key identifier for it.
operand_range getOperands()
Returns an iterator on the underlying Value's.
This class contains a list of basic blocks and a link to the parent operation it is attached to.
BlockListType & getBlocks()
BlockArgument getArgument(unsigned i)
void mergeBlocks(Block *source, Block *dest, ValueRange argValues=std::nullopt)
Inline the operations of block 'source' into the end of block 'dest'.
@ Private
The symbol is private and may only be referenced by SymbolRefAttrs local to the operations within the...
Instances of the Type class are uniqued, have an immutable identifier and an optional mutable compone...
This class represents an instance of an SSA value in the MLIR system, representing a computable value...
Type getType() const
Return the type of this value.
Operation * getDefiningOp() const
If this value is the result of an operation, return the operation that defines it.
A utility result that is used to signal how to proceed with an ongoing walk:
static WalkResult advance()
bool wasInterrupted() const
Returns true if the walk was interrupted.
static WalkResult interrupt()
Include the generated interface declarations.
void connectPHINodes(Region ®ion, const ModuleTranslation &state)
For all blocks in the region that were converted to LLVM IR using the given ModuleTranslation,...
llvm::Constant * createMappingInformation(Location loc, llvm::OpenMPIRBuilder &builder)
Create a constant string representing the mapping information extracted from the MLIR location inform...
constexpr void enumerate(std::tuple< Tys... > &tuple, CallbackT &&callback)
Runtime
Potential runtimes for AMD GPU kernels.
Include the generated interface declarations.
LogicalResult failure(bool isFailure=true)
Utility function to generate a LogicalResult.
void replaceAllUsesInRegionWith(Value orig, Value replacement, Region ®ion)
Replace all uses of orig within the given region with replacement.
SetVector< Block * > getTopologicallySortedBlocks(Region ®ion)
Get a topologically sorted list of blocks of the given region.
InFlightDiagnostic emitError(Location loc)
Utility method to emit an error message using this location.
LogicalResult success(bool isSuccess=true)
Utility function to generate a LogicalResult.
void registerOpenMPDialectTranslation(DialectRegistry ®istry)
Register the OpenMP dialect and the translation from it to the LLVM IR in the given registry;.
auto get(MLIRContext *context, Ts &&...params)
Helper method that injects context only if needed, this helps unify some of the attribute constructio...
bool failed(LogicalResult result)
Utility function that returns true if the provided LogicalResult corresponds to a failure value.
llvm::SmallVector< bool, 4 > IsAMember
llvm::SmallVector< llvm::Value *, 4 > OriginalValue
llvm::SmallVector< bool, 4 > IsDeclareTarget
llvm::SmallVector< llvm::Type *, 4 > BaseType
void append(MapInfoData &CurInfo)
Append arrays in CurInfo.
llvm::SmallVector< mlir::Operation *, 4 > MapClause
RAII object calling stackPush/stackPop on construction/destruction.
This class represents an efficient way to signal success or failure.