20 #include "llvm/ADT/SetVector.h" 21 #include "llvm/ADT/TypeSwitch.h" 22 #include "llvm/Frontend/OpenMP/OMPIRBuilder.h" 23 #include "llvm/IR/DebugInfoMetadata.h" 24 #include "llvm/IR/IRBuilder.h" 29 static llvm::omp::ScheduleKind
31 if (!schedKind.hasValue())
32 return llvm::omp::OMP_SCHEDULE_Default;
33 switch (schedKind.getValue()) {
34 case omp::ClauseScheduleKind::Static:
35 return llvm::omp::OMP_SCHEDULE_Static;
36 case omp::ClauseScheduleKind::Dynamic:
37 return llvm::omp::OMP_SCHEDULE_Dynamic;
38 case omp::ClauseScheduleKind::Guided:
39 return llvm::omp::OMP_SCHEDULE_Guided;
40 case omp::ClauseScheduleKind::Auto:
41 return llvm::omp::OMP_SCHEDULE_Auto;
43 return llvm::omp::OMP_SCHEDULE_Runtime;
45 llvm_unreachable(
"unhandled schedule clause argument");
50 class OpenMPAllocaStackFrame
55 explicit OpenMPAllocaStackFrame(llvm::OpenMPIRBuilder::InsertPointTy allocaIP)
56 : allocaInsertPoint(allocaIP) {}
57 llvm::OpenMPIRBuilder::InsertPointTy allocaInsertPoint;
62 class OpenMPVarMappingStackFrame
64 OpenMPVarMappingStackFrame> {
68 explicit OpenMPVarMappingStackFrame(
78 static llvm::OpenMPIRBuilder::InsertPointTy
84 llvm::OpenMPIRBuilder::InsertPointTy allocaInsertPoint;
86 [&](
const OpenMPAllocaStackFrame &frame) {
87 allocaInsertPoint = frame.allocaInsertPoint;
91 return allocaInsertPoint;
100 if (builder.GetInsertBlock() ==
101 &builder.GetInsertBlock()->getParent()->getEntryBlock()) {
102 assert(builder.GetInsertPoint() == builder.GetInsertBlock()->end() &&
103 "Assuming end of basic block");
104 llvm::BasicBlock *entryBB = llvm::BasicBlock::Create(
105 builder.getContext(),
"entry", builder.GetInsertBlock()->getParent(),
106 builder.GetInsertBlock()->getNextNode());
107 builder.CreateBr(entryBB);
108 builder.SetInsertPoint(entryBB);
111 llvm::BasicBlock &funcEntryBlock =
112 builder.GetInsertBlock()->getParent()->getEntryBlock();
113 return llvm::OpenMPIRBuilder::InsertPointTy(
114 &funcEntryBlock, funcEntryBlock.getFirstInsertionPt());
123 Region ®ion, StringRef blockName, llvm::IRBuilderBase &builder,
126 llvm::BasicBlock *continuationBlock =
127 splitBB(builder,
true,
"omp.region.cont");
128 llvm::BasicBlock *sourceBlock = builder.GetInsertBlock();
130 llvm::LLVMContext &llvmContext = builder.getContext();
131 for (
Block &bb : region) {
132 llvm::BasicBlock *llvmBB = llvm::BasicBlock::Create(
133 llvmContext, blockName, builder.GetInsertBlock()->getParent(),
134 builder.GetInsertBlock()->getNextNode());
135 moduleTranslation.
mapBlock(&bb, llvmBB);
138 llvm::Instruction *sourceTerminator = sourceBlock->getTerminator();
144 bool operandsProcessed =
false;
145 unsigned numYields = 0;
146 for (
Block &bb : region.getBlocks()) {
147 if (omp::YieldOp yield = dyn_cast<omp::YieldOp>(bb.getTerminator())) {
148 if (!operandsProcessed) {
149 for (
unsigned i = 0, e = yield->getNumOperands(); i < e; ++i) {
150 continuationBlockPHITypes.push_back(
151 moduleTranslation.
convertType(yield->getOperand(i).getType()));
153 operandsProcessed =
true;
155 assert(continuationBlockPHITypes.size() == yield->getNumOperands() &&
156 "mismatching number of values yielded from the region");
157 for (
unsigned i = 0, e = yield->getNumOperands(); i < e; ++i) {
158 llvm::Type *operandType =
159 moduleTranslation.
convertType(yield->getOperand(i).getType());
161 assert(continuationBlockPHITypes[i] == operandType &&
162 "values of mismatching types yielded from the region");
171 if (!continuationBlockPHITypes.empty())
173 continuationBlockPHIs &&
174 "expected continuation block PHIs if converted regions yield values");
175 if (continuationBlockPHIs) {
176 llvm::IRBuilderBase::InsertPointGuard guard(builder);
177 continuationBlockPHIs->reserve(continuationBlockPHITypes.size());
178 builder.SetInsertPoint(continuationBlock, continuationBlock->begin());
179 for (llvm::Type *ty : continuationBlockPHITypes)
180 continuationBlockPHIs->push_back(builder.CreatePHI(ty, numYields));
187 for (
Block *bb : blocks) {
188 llvm::BasicBlock *llvmBB = moduleTranslation.
lookupBlock(bb);
191 if (bb->isEntryBlock()) {
192 assert(sourceTerminator->getNumSuccessors() == 1 &&
193 "provided entry block has multiple successors");
194 assert(sourceTerminator->getSuccessor(0) == continuationBlock &&
195 "ContinuationBlock is not the successor of the entry block");
196 sourceTerminator->setSuccessor(0, llvmBB);
199 llvm::IRBuilderBase::InsertPointGuard guard(builder);
201 moduleTranslation.
convertBlock(*bb, bb->isEntryBlock(), builder))) {
203 return continuationBlock;
213 Operation *terminator = bb->getTerminator();
214 if (isa<omp::TerminatorOp, omp::YieldOp>(terminator)) {
215 builder.CreateBr(continuationBlock);
217 for (
unsigned i = 0, e = terminator->
getNumOperands(); i < e; ++i)
218 (*continuationBlockPHIs)[i]->addIncoming(
232 return continuationBlock;
238 case omp::ClauseProcBindKind::Close:
239 return llvm::omp::ProcBindKind::OMP_PROC_BIND_close;
240 case omp::ClauseProcBindKind::Master:
241 return llvm::omp::ProcBindKind::OMP_PROC_BIND_master;
242 case omp::ClauseProcBindKind::Primary:
243 return llvm::omp::ProcBindKind::OMP_PROC_BIND_primary;
244 case omp::ClauseProcBindKind::Spread:
245 return llvm::omp::ProcBindKind::OMP_PROC_BIND_spread;
247 llvm_unreachable(
"Unknown ClauseProcBindKind kind");
254 using InsertPointTy = llvm::OpenMPIRBuilder::InsertPointTy;
259 auto bodyGenCB = [&](InsertPointTy allocaIP, InsertPointTy codeGenIP) {
263 moduleTranslation, allocaIP);
266 builder.restoreIP(codeGenIP);
268 moduleTranslation, bodyGenStatus);
274 auto privCB = [&](InsertPointTy allocaIP, InsertPointTy codeGenIP,
275 llvm::Value &, llvm::Value &vPtr,
276 llvm::Value *&replacementValue) -> InsertPointTy {
277 replacementValue = &vPtr;
284 auto finiCB = [&](InsertPointTy codeGenIP) {};
286 llvm::Value *ifCond =
nullptr;
287 if (
auto ifExprVar = opInst.if_expr_var())
289 llvm::Value *numThreads =
nullptr;
290 if (
auto numThreadsVar = opInst.num_threads_var())
291 numThreads = moduleTranslation.
lookupValue(numThreadsVar);
292 auto pbKind = llvm::omp::OMP_PROC_BIND_default;
293 if (
auto bind = opInst.proc_bind_val())
296 bool isCancellable =
false;
298 llvm::OpenMPIRBuilder::InsertPointTy allocaIP =
300 llvm::OpenMPIRBuilder::LocationDescription ompLoc(builder);
302 ompLoc, allocaIP, bodyGenCB, privCB, finiCB, ifCond, numThreads, pbKind,
305 return bodyGenStatus;
312 using InsertPointTy = llvm::OpenMPIRBuilder::InsertPointTy;
317 auto bodyGenCB = [&](InsertPointTy allocaIP, InsertPointTy codeGenIP) {
319 auto ®ion = cast<omp::MasterOp>(opInst).getRegion();
320 builder.restoreIP(codeGenIP);
327 auto finiCB = [&](InsertPointTy codeGenIP) {};
329 llvm::OpenMPIRBuilder::LocationDescription ompLoc(builder);
331 ompLoc, bodyGenCB, finiCB));
339 using InsertPointTy = llvm::OpenMPIRBuilder::InsertPointTy;
340 auto criticalOp = cast<omp::CriticalOp>(opInst);
345 auto bodyGenCB = [&](InsertPointTy allocaIP, InsertPointTy codeGenIP) {
347 auto ®ion = cast<omp::CriticalOp>(opInst).getRegion();
348 builder.restoreIP(codeGenIP);
350 moduleTranslation, bodyGenStatus);
355 auto finiCB = [&](InsertPointTy codeGenIP) {};
357 llvm::OpenMPIRBuilder::LocationDescription ompLoc(builder);
358 llvm::LLVMContext &llvmContext = moduleTranslation.
getLLVMContext();
362 if (criticalOp.nameAttr()) {
365 auto symbolRef = criticalOp.nameAttr().cast<SymbolRefAttr>();
366 auto criticalDeclareOp =
367 SymbolTable::lookupNearestSymbolFrom<omp::CriticalDeclareOp>(criticalOp,
370 llvm::ConstantInt::get(llvm::Type::getInt32Ty(llvmContext),
371 static_cast<int>(criticalDeclareOp.hint_val()));
374 ompLoc, bodyGenCB, finiCB, criticalOp.name().getValueOr(
""), hint));
382 omp::ReductionOp reduction) {
383 SymbolRefAttr reductionSymbol;
384 for (
unsigned i = 0, e = container.getNumReductionVars(); i < e; ++i) {
385 if (container.reduction_vars()[i] != reduction.accumulator())
387 reductionSymbol = (*container.reductions())[i].cast<SymbolRefAttr>();
390 assert(reductionSymbol &&
391 "reduction operation must be associated with a declaration");
393 return SymbolTable::lookupNearestSymbolFrom<omp::ReductionDeclareOp>(
394 container, reductionSymbol);
405 reductions.reserve(reductions.size() + loop.getNumReductionVars());
406 for (
auto symbolRef : attr->getAsRange<SymbolRefAttr>()) {
407 reductions.push_back(
408 SymbolTable::lookupNearestSymbolFrom<omp::ReductionDeclareOp>(
419 Region ®ion, StringRef blockName, llvm::IRBuilderBase &builder,
427 if (llvm::hasSingleElement(region)) {
428 moduleTranslation.
mapBlock(®ion.
front(), builder.GetInsertBlock());
430 region.
front(),
true, builder)))
434 if (continuationBlockArgs)
436 *continuationBlockArgs,
448 region, blockName, builder, moduleTranslation, bodyGenStatus, &phis);
449 if (
failed(bodyGenStatus))
451 if (continuationBlockArgs)
452 llvm::append_range(*continuationBlockArgs, phis);
453 builder.SetInsertPoint(continuationBlock,
454 continuationBlock->getFirstInsertionPt());
461 using OwningReductionGen = std::function<llvm::OpenMPIRBuilder::InsertPointTy(
462 llvm::OpenMPIRBuilder::InsertPointTy, llvm::Value *, llvm::Value *,
464 using OwningAtomicReductionGen =
465 std::function<llvm::OpenMPIRBuilder::InsertPointTy(
466 llvm::OpenMPIRBuilder::InsertPointTy, llvm::Type *, llvm::Value *,
473 static OwningReductionGen
479 OwningReductionGen gen =
480 [&, decl](llvm::OpenMPIRBuilder::InsertPointTy insertPoint,
481 llvm::Value *lhs, llvm::Value *rhs,
482 llvm::Value *&result)
mutable {
483 Region &reductionRegion = decl.reductionRegion();
486 builder.restoreIP(insertPoint);
489 "omp.reduction.nonatomic.body",
490 builder, moduleTranslation, &phis)))
491 return llvm::OpenMPIRBuilder::InsertPointTy();
492 assert(phis.size() == 1);
494 return builder.saveIP();
503 static OwningAtomicReductionGen
505 llvm::IRBuilderBase &builder,
507 if (decl.atomicReductionRegion().empty())
508 return OwningAtomicReductionGen();
513 OwningAtomicReductionGen atomicGen =
514 [&, decl](llvm::OpenMPIRBuilder::InsertPointTy insertPoint, llvm::Type *,
515 llvm::Value *lhs, llvm::Value *rhs)
mutable {
516 Region &atomicRegion = decl.atomicReductionRegion();
519 builder.restoreIP(insertPoint);
522 "omp.reduction.atomic.body", builder,
523 moduleTranslation, &phis)))
524 return llvm::OpenMPIRBuilder::InsertPointTy();
525 assert(phis.empty());
526 return builder.saveIP();
535 auto orderedOp = cast<omp::OrderedOp>(opInst);
537 omp::ClauseDepend dependType = *orderedOp.depend_type_val();
538 bool isDependSource = dependType == omp::ClauseDepend::dependsource;
539 unsigned numLoops = orderedOp.num_loops_val().getValue();
541 moduleTranslation.
lookupValues(orderedOp.depend_vec_vars());
543 size_t indexVecValues = 0;
544 while (indexVecValues < vecValues.size()) {
546 storeValues.reserve(numLoops);
547 for (
unsigned i = 0; i < numLoops; i++) {
548 storeValues.push_back(vecValues[indexVecValues]);
551 llvm::OpenMPIRBuilder::InsertPointTy allocaIP =
553 llvm::OpenMPIRBuilder::LocationDescription ompLoc(builder);
554 builder.restoreIP(moduleTranslation.
getOpenMPBuilder()->createOrderedDepend(
555 ompLoc, allocaIP, numLoops, storeValues,
".cnt.addr", isDependSource));
565 using InsertPointTy = llvm::OpenMPIRBuilder::InsertPointTy;
566 auto orderedRegionOp = cast<omp::OrderedRegionOp>(opInst);
569 if (orderedRegionOp.simd())
576 auto bodyGenCB = [&](InsertPointTy allocaIP, InsertPointTy codeGenIP) {
578 auto ®ion = cast<omp::OrderedRegionOp>(opInst).getRegion();
579 builder.restoreIP(codeGenIP);
581 moduleTranslation, bodyGenStatus);
586 auto finiCB = [&](InsertPointTy codeGenIP) {};
588 llvm::OpenMPIRBuilder::LocationDescription ompLoc(builder);
591 ompLoc, bodyGenCB, finiCB, !orderedRegionOp.simd()));
592 return bodyGenStatus;
598 using InsertPointTy = llvm::OpenMPIRBuilder::InsertPointTy;
599 using StorableBodyGenCallbackTy =
600 llvm::OpenMPIRBuilder::StorableBodyGenCallbackTy;
602 auto sectionsOp = cast<omp::SectionsOp>(opInst);
606 if (!sectionsOp.reduction_vars().empty() || sectionsOp.reductions() ||
607 !sectionsOp.allocate_vars().empty() ||
608 !sectionsOp.allocators_vars().empty())
610 <<
"reduction and allocate clauses are not supported for sections " 616 for (
Operation &op : *sectionsOp.region().begin()) {
617 auto sectionOp = dyn_cast<omp::SectionOp>(op);
621 Region ®ion = sectionOp.region();
622 auto sectionCB = [®ion, &builder, &moduleTranslation, &bodyGenStatus](
623 InsertPointTy allocaIP, InsertPointTy codeGenIP) {
624 builder.restoreIP(codeGenIP);
626 moduleTranslation, bodyGenStatus);
628 sectionCBs.push_back(sectionCB);
634 if (sectionCBs.empty())
637 assert(isa<omp::SectionOp>(*sectionsOp.region().op_begin()));
642 auto privCB = [&](InsertPointTy, InsertPointTy codeGenIP, llvm::Value &,
644 llvm::Value *&replacementValue) -> InsertPointTy {
645 replacementValue = &vPtr;
651 auto finiCB = [&](InsertPointTy codeGenIP) {};
653 llvm::OpenMPIRBuilder::InsertPointTy allocaIP =
655 llvm::OpenMPIRBuilder::LocationDescription ompLoc(builder);
657 ompLoc, allocaIP, sectionCBs, privCB, finiCB,
false,
658 sectionsOp.nowait()));
659 return bodyGenStatus;
666 using InsertPointTy = llvm::OpenMPIRBuilder::InsertPointTy;
667 llvm::OpenMPIRBuilder::LocationDescription ompLoc(builder);
669 auto bodyCB = [&](InsertPointTy allocaIP, InsertPointTy codegenIP) {
670 builder.restoreIP(codegenIP);
672 moduleTranslation, bodyGenStatus);
674 auto finiCB = [&](InsertPointTy codeGenIP) {};
676 ompLoc, bodyCB, finiCB, singleOp.nowait(),
nullptr));
677 return bodyGenStatus;
684 auto loop = cast<omp::WsLoopOp>(opInst);
686 if (loop.lowerBound().empty())
691 loop.schedule_val().getValueOr(omp::ClauseScheduleKind::Static);
694 llvm::Value *step = moduleTranslation.
lookupValue(loop.step()[0]);
695 llvm::Type *ivType = step->getType();
696 llvm::Value *chunk =
nullptr;
697 if (loop.schedule_chunk_var()) {
698 llvm::Value *chunkVar =
699 moduleTranslation.
lookupValue(loop.schedule_chunk_var());
700 llvm::Type *chunkVarType = chunkVar->getType();
701 assert(chunkVarType->isIntegerTy() &&
702 "chunk size must be one integer expression");
703 if (chunkVarType->getIntegerBitWidth() < ivType->getIntegerBitWidth())
704 chunk = builder.CreateSExt(chunkVar, ivType);
705 else if (chunkVarType->getIntegerBitWidth() > ivType->getIntegerBitWidth())
706 chunk = builder.CreateTrunc(chunkVar, ivType);
713 llvm::OpenMPIRBuilder::InsertPointTy allocaIP =
719 unsigned numReductions = loop.getNumReductionVars();
720 privateReductionVariables.reserve(numReductions);
721 if (numReductions != 0) {
722 llvm::IRBuilderBase::InsertPointGuard guard(builder);
723 builder.restoreIP(allocaIP);
724 for (
unsigned i = 0; i < numReductions; ++i) {
727 llvm::Value *var = builder.CreateAlloca(
728 moduleTranslation.
convertType(reductionType.getElementType()));
729 privateReductionVariables.push_back(var);
730 reductionVariableMap.try_emplace(loop.reduction_vars()[i], var);
738 moduleTranslation, reductionVariableMap);
743 for (
unsigned i = 0; i < numReductions; ++i) {
746 "omp.reduction.neutral", builder,
747 moduleTranslation, &phis)))
749 assert(phis.size() == 1 &&
"expected one value to be yielded from the " 750 "reduction neutral element declaration region");
751 builder.CreateStore(phis[0], privateReductionVariables[i]);
755 llvm::OpenMPIRBuilder::LocationDescription ompLoc(builder);
763 auto bodyGen = [&](llvm::OpenMPIRBuilder::InsertPointTy ip, llvm::Value *iv) {
766 loop.getRegion().front().getArgument(loopInfos.size()), iv);
771 bodyInsertPoints.push_back(ip);
773 if (loopInfos.size() != loop.getNumLoops() - 1)
777 builder.restoreIP(ip);
779 moduleTranslation, bodyGenStatus);
787 for (
unsigned i = 0, e = loop.getNumLoops(); i < e; ++i) {
788 llvm::Value *lowerBound =
789 moduleTranslation.
lookupValue(loop.lowerBound()[i]);
790 llvm::Value *upperBound =
791 moduleTranslation.
lookupValue(loop.upperBound()[i]);
792 llvm::Value *step = moduleTranslation.
lookupValue(loop.step()[i]);
797 llvm::OpenMPIRBuilder::LocationDescription loc = ompLoc;
798 llvm::OpenMPIRBuilder::InsertPointTy computeIP = ompLoc.IP;
800 loc = llvm::OpenMPIRBuilder::LocationDescription(bodyInsertPoints.back());
801 computeIP = loopInfos.front()->getPreheaderIP();
803 loopInfos.push_back(ompBuilder->createCanonicalLoop(
804 loc, bodyGen, lowerBound, upperBound, step,
805 true, loop.inclusive(), computeIP));
807 if (
failed(bodyGenStatus))
813 llvm::IRBuilderBase::InsertPoint afterIP = loopInfos.front()->getAfterIP();
814 llvm::CanonicalLoopInfo *loopInfo =
815 ompBuilder->collapseLoops(ompLoc.DL, loopInfos, {});
820 bool isOrdered = loop.ordered_val().hasValue();
822 bool isSimd = loop.simd_modifier();
824 ompBuilder->applyWorkshareLoop(
825 ompLoc.DL, loopInfo, allocaIP, !loop.nowait(),
826 convertToScheduleKind(schedule), chunk, isSimd,
827 scheduleModifier == omp::ScheduleModifier::monotonic,
828 scheduleModifier == omp::ScheduleModifier::nonmonotonic, isOrdered);
834 builder.restoreIP(afterIP);
837 if (numReductions == 0)
844 for (
unsigned i = 0; i < numReductions; ++i) {
845 owningReductionGens.push_back(
847 owningAtomicReductionGens.push_back(
853 reductionInfos.reserve(numReductions);
854 for (
unsigned i = 0; i < numReductions; ++i) {
855 llvm::OpenMPIRBuilder::AtomicReductionGenTy atomicGen =
nullptr;
856 if (owningAtomicReductionGens[i])
857 atomicGen = owningAtomicReductionGens[i];
860 llvm::Value *variable =
861 moduleTranslation.
lookupValue(loop.reduction_vars()[i]);
862 reductionInfos.push_back(
863 {moduleTranslation.
convertType(reductionType.getElementType()),
864 variable, privateReductionVariables[i], owningReductionGens[i],
871 llvm::UnreachableInst *tempTerminator = builder.CreateUnreachable();
872 builder.SetInsertPoint(tempTerminator);
873 llvm::OpenMPIRBuilder::InsertPointTy contInsertPoint =
874 ompBuilder->createReductions(builder.saveIP(), allocaIP, reductionInfos,
876 if (!contInsertPoint.getBlock())
877 return loop->emitOpError() <<
"failed to convert reductions";
878 auto nextInsertionPoint =
879 ompBuilder->createBarrier(contInsertPoint, llvm::omp::OMPD_for);
880 tempTerminator->eraseFromParent();
881 builder.restoreIP(nextInsertionPoint);
890 auto loop = cast<omp::SimdLoopOp>(opInst);
892 llvm::OpenMPIRBuilder::LocationDescription ompLoc(builder);
900 auto bodyGen = [&](llvm::OpenMPIRBuilder::InsertPointTy ip, llvm::Value *iv) {
903 loop.getRegion().front().getArgument(loopInfos.size()), iv);
908 bodyInsertPoints.push_back(ip);
910 if (loopInfos.size() != loop.getNumLoops() - 1)
914 builder.restoreIP(ip);
916 moduleTranslation, bodyGenStatus);
924 for (
unsigned i = 0, e = loop.getNumLoops(); i < e; ++i) {
925 llvm::Value *lowerBound =
926 moduleTranslation.
lookupValue(loop.lowerBound()[i]);
927 llvm::Value *upperBound =
928 moduleTranslation.
lookupValue(loop.upperBound()[i]);
929 llvm::Value *step = moduleTranslation.
lookupValue(loop.step()[i]);
934 llvm::OpenMPIRBuilder::LocationDescription loc = ompLoc;
935 llvm::OpenMPIRBuilder::InsertPointTy computeIP = ompLoc.IP;
937 loc = llvm::OpenMPIRBuilder::LocationDescription(bodyInsertPoints.back(),
939 computeIP = loopInfos.front()->getPreheaderIP();
941 loopInfos.push_back(ompBuilder->createCanonicalLoop(
942 loc, bodyGen, lowerBound, upperBound, step,
943 true,
true, computeIP));
945 if (
failed(bodyGenStatus))
950 llvm::IRBuilderBase::InsertPoint afterIP = loopInfos.front()->getAfterIP();
951 llvm::CanonicalLoopInfo *loopInfo =
952 ompBuilder->collapseLoops(ompLoc.DL, loopInfos, {});
954 ompBuilder->applySimd(ompLoc.DL, loopInfo);
956 builder.restoreIP(afterIP);
964 return llvm::AtomicOrdering::Monotonic;
967 case omp::ClauseMemoryOrderKind::Seq_cst:
968 return llvm::AtomicOrdering::SequentiallyConsistent;
969 case omp::ClauseMemoryOrderKind::Acq_rel:
970 return llvm::AtomicOrdering::AcquireRelease;
971 case omp::ClauseMemoryOrderKind::Acquire:
972 return llvm::AtomicOrdering::Acquire;
973 case omp::ClauseMemoryOrderKind::Release:
974 return llvm::AtomicOrdering::Release;
975 case omp::ClauseMemoryOrderKind::Relaxed:
976 return llvm::AtomicOrdering::Monotonic;
978 llvm_unreachable(
"Unknown ClauseMemoryOrderKind kind");
986 auto readOp = cast<omp::AtomicReadOp>(opInst);
989 llvm::OpenMPIRBuilder::LocationDescription ompLoc(builder);
992 llvm::Value *x = moduleTranslation.
lookupValue(readOp.x());
994 llvm::Value *v = moduleTranslation.
lookupValue(readOp.v());
996 llvm::OpenMPIRBuilder::AtomicOpValue V = {
997 v, moduleTranslation.
convertType(vTy),
false,
false};
998 llvm::OpenMPIRBuilder::AtomicOpValue X = {
999 x, moduleTranslation.
convertType(xTy),
false,
false};
1000 builder.restoreIP(ompBuilder->createAtomicRead(ompLoc, X, V, AO));
1008 auto writeOp = cast<omp::AtomicWriteOp>(opInst);
1011 llvm::OpenMPIRBuilder::LocationDescription ompLoc(builder);
1013 llvm::Value *expr = moduleTranslation.
lookupValue(writeOp.value());
1014 llvm::Value *dest = moduleTranslation.
lookupValue(writeOp.address());
1015 llvm::Type *ty = moduleTranslation.
convertType(writeOp.value().getType());
1016 llvm::OpenMPIRBuilder::AtomicOpValue x = {dest, ty,
false,
1018 builder.restoreIP(ompBuilder->createAtomicWrite(ompLoc, x, expr, ao));
1026 .Case([&](LLVM::AddOp) {
return llvm::AtomicRMWInst::BinOp::Add; })
1027 .Case([&](LLVM::SubOp) {
return llvm::AtomicRMWInst::BinOp::Sub; })
1028 .Case([&](LLVM::AndOp) {
return llvm::AtomicRMWInst::BinOp::And; })
1029 .Case([&](LLVM::OrOp) {
return llvm::AtomicRMWInst::BinOp::Or; })
1030 .Case([&](LLVM::XOrOp) {
return llvm::AtomicRMWInst::BinOp::Xor; })
1031 .Case([&](LLVM::UMaxOp) {
return llvm::AtomicRMWInst::BinOp::UMax; })
1032 .Case([&](LLVM::UMinOp) {
return llvm::AtomicRMWInst::BinOp::UMin; })
1033 .Case([&](LLVM::FAddOp) {
return llvm::AtomicRMWInst::BinOp::FAdd; })
1034 .Case([&](LLVM::FSubOp) {
return llvm::AtomicRMWInst::BinOp::FSub; })
1035 .Default(llvm::AtomicRMWInst::BinOp::BAD_BINOP);
1041 llvm::IRBuilderBase &builder,
1046 auto &innerOpList = opInst.region().front().getOperations();
1047 if (innerOpList.size() != 2)
1048 return opInst.emitError(
"exactly two operations are allowed inside an " 1049 "atomic update region while lowering to LLVM IR");
1051 Operation &innerUpdateOp = innerOpList.front();
1055 opInst.getRegion().getArgument(0)))
1057 "the update operation inside the region must be a binary operation and " 1058 "that update operation must have the region argument as an operand");
1064 innerUpdateOp.
getOperand(0) == opInst.getRegion().getArgument(0);
1068 llvm::Value *llvmExpr = moduleTranslation.
lookupValue(mlirExpr);
1069 llvm::Value *llvmX = moduleTranslation.
lookupValue(opInst.x());
1072 llvm::Type *llvmXElementType =
1074 llvm::OpenMPIRBuilder::AtomicOpValue llvmAtomicX = {llvmX, llvmXElementType,
1078 llvm::AtomicOrdering atomicOrdering =
1083 auto updateFn = [&opInst, &moduleTranslation, &updateGenStatus](
1084 llvm::Value *atomicx,
1085 llvm::IRBuilder<> &builder) -> llvm::Value * {
1087 moduleTranslation.mapValue(*opInst.region().args_begin(), atomicx);
1088 moduleTranslation.mapBlock(&bb, builder.GetInsertBlock());
1089 if (
failed(moduleTranslation.convertBlock(bb,
true, builder))) {
1090 updateGenStatus = (opInst.emitError()
1091 <<
"unable to convert update operation to llvm IR");
1094 omp::YieldOp yieldop = dyn_cast<omp::YieldOp>(bb.
getTerminator());
1095 assert(yieldop && yieldop.results().size() == 1 &&
1096 "terminator must be omp.yield op and it must have exactly one " 1098 return moduleTranslation.lookupValue(yieldop.results()[0]);
1103 llvm::OpenMPIRBuilder::LocationDescription ompLoc(builder);
1104 builder.restoreIP(ompBuilder->createAtomicUpdate(
1105 ompLoc, allocaIP, llvmAtomicX, llvmExpr, atomicOrdering, binop, updateFn,
1107 return updateGenStatus;
1112 llvm::IRBuilderBase &builder,
1116 bool isXBinopExpr =
false, isPostfixUpdate =
false;
1117 llvm::AtomicRMWInst::BinOp binop = llvm::AtomicRMWInst::BinOp::BAD_BINOP;
1119 omp::AtomicUpdateOp atomicUpdateOp = atomicCaptureOp.getAtomicUpdateOp();
1120 omp::AtomicWriteOp atomicWriteOp = atomicCaptureOp.getAtomicWriteOp();
1122 assert((atomicUpdateOp || atomicWriteOp) &&
1123 "internal op must be an atomic.update or atomic.write op");
1125 if (atomicWriteOp) {
1126 isPostfixUpdate =
true;
1127 mlirExpr = atomicWriteOp.value();
1129 isPostfixUpdate = atomicCaptureOp.getSecondOp() ==
1130 atomicCaptureOp.getAtomicUpdateOp().getOperation();
1131 auto &innerOpList = atomicUpdateOp.region().front().getOperations();
1132 if (innerOpList.size() != 2)
1133 return atomicUpdateOp.emitError(
1134 "exactly two operations are allowed inside an " 1135 "atomic update region while lowering to LLVM IR");
1136 Operation *innerUpdateOp = atomicUpdateOp.getFirstOp();
1139 atomicUpdateOp.getRegion().getArgument(0)))
1141 "the update operation inside the region must be a binary operation " 1142 "and that update operation must have the region argument as an " 1146 isXBinopExpr = innerUpdateOp->
getOperand(0) ==
1147 atomicUpdateOp.getRegion().getArgument(0);
1149 mlirExpr = (isXBinopExpr ? innerUpdateOp->
getOperand(1)
1153 llvm::Value *llvmExpr = moduleTranslation.
lookupValue(mlirExpr);
1154 llvm::Value *llvmX =
1155 moduleTranslation.
lookupValue(atomicCaptureOp.getAtomicReadOp().x());
1156 llvm::Value *llvmV =
1157 moduleTranslation.
lookupValue(atomicCaptureOp.getAtomicReadOp().v());
1158 auto mlirXType = atomicCaptureOp.getAtomicReadOp()
1162 llvm::Type *llvmXElementType =
1163 moduleTranslation.
convertType(mlirXType.getElementType());
1164 llvm::OpenMPIRBuilder::AtomicOpValue llvmAtomicX = {llvmX, llvmXElementType,
1167 llvm::OpenMPIRBuilder::AtomicOpValue llvmAtomicV = {llvmV, llvmXElementType,
1171 llvm::AtomicOrdering atomicOrdering =
1175 auto updateFn = [&](llvm::Value *atomicx,
1176 llvm::IRBuilder<> &builder) -> llvm::Value * {
1178 return moduleTranslation.
lookupValue(atomicWriteOp.value());
1179 Block &bb = *atomicUpdateOp.region().
begin();
1180 moduleTranslation.
mapValue(*atomicUpdateOp.region().args_begin(), atomicx);
1181 moduleTranslation.
mapBlock(&bb, builder.GetInsertBlock());
1183 updateGenStatus = (atomicUpdateOp.emitError()
1184 <<
"unable to convert update operation to llvm IR");
1187 omp::YieldOp yieldop = dyn_cast<omp::YieldOp>(bb.
getTerminator());
1188 assert(yieldop && yieldop.results().size() == 1 &&
1189 "terminator must be omp.yield op and it must have exactly one " 1191 return moduleTranslation.
lookupValue(yieldop.results()[0]);
1196 llvm::OpenMPIRBuilder::LocationDescription ompLoc(builder);
1197 builder.restoreIP(ompBuilder->createAtomicCapture(
1198 ompLoc, allocaIP, llvmAtomicX, llvmAtomicV, llvmExpr, atomicOrdering,
1199 binop, updateFn, atomicUpdateOp, isPostfixUpdate, isXBinopExpr));
1200 return updateGenStatus;
1209 llvm::IRBuilderBase &builder,
1212 auto reductionContainer = reductionOp->getParentOfType<omp::WsLoopOp>();
1213 omp::ReductionDeclareOp declaration =
1215 assert(declaration &&
"could not find reduction declaration");
1220 moduleTranslation.
stackWalk<OpenMPVarMappingStackFrame>(
1221 [&](
const OpenMPVarMappingStackFrame &frame) {
1222 reductionVariableMap = &frame.mapping;
1225 assert(reductionVariableMap &&
"couldn't find private reduction variables");
1229 Region &reductionRegion = declaration.reductionRegion();
1230 llvm::Value *privateReductionVar =
1231 reductionVariableMap->lookup(reductionOp.accumulator());
1232 llvm::Value *reductionVal = builder.CreateLoad(
1233 moduleTranslation.
convertType(reductionOp.operand().getType()),
1234 privateReductionVar);
1240 moduleTranslation.
lookupValue(reductionOp.operand()));
1244 builder, moduleTranslation, &phis)))
1246 assert(phis.size() == 1 &&
"expected one value to be yielded from " 1247 "the reduction body declaration region");
1248 builder.CreateStore(phis[0], privateReductionVar);
1257 llvm::OpenMPIRBuilder::LocationDescription ompLoc(builder);
1258 auto threadprivateOp = cast<omp::ThreadprivateOp>(opInst);
1260 Value symAddr = threadprivateOp.sym_addr();
1262 if (!isa<LLVM::AddressOfOp>(symOp))
1263 return opInst.
emitError(
"Addressing symbol not found");
1264 LLVM::AddressOfOp addressOfOp = dyn_cast<LLVM::AddressOfOp>(symOp);
1266 LLVM::GlobalOp global = addressOfOp.getGlobal();
1267 llvm::GlobalValue *globalValue = moduleTranslation.
lookupGlobal(global);
1269 builder.CreateBitCast(globalValue, builder.getInt8PtrTy());
1270 llvm::Type *type = globalValue->getValueType();
1271 llvm::TypeSize typeSize =
1272 builder.GetInsertBlock()->getModule()->getDataLayout().getTypeStoreSize(
1274 llvm::ConstantInt *size = builder.getInt64(typeSize.getFixedSize());
1275 llvm::StringRef suffix = llvm::StringRef(
".cache", 6);
1276 std::string cacheName = (Twine(global.getSymName()).
concat(suffix)).str();
1278 llvm::Value *callInst =
1280 ompLoc, data, size, cacheName);
1281 llvm::Value *result = builder.CreateBitCast(callInst, globalValue->getType());
1290 class OpenMPDialectLLVMIRTranslationInterface
1298 convertOperation(
Operation *op, llvm::IRBuilderBase &builder,
1306 LogicalResult OpenMPDialectLLVMIRTranslationInterface::convertOperation(
1307 Operation *op, llvm::IRBuilderBase &builder,
1313 .Case([&](omp::BarrierOp) {
1314 ompBuilder->createBarrier(builder.saveIP(), llvm::omp::OMPD_barrier);
1317 .Case([&](omp::TaskwaitOp) {
1318 ompBuilder->createTaskwait(builder.saveIP());
1321 .Case([&](omp::TaskyieldOp) {
1322 ompBuilder->createTaskyield(builder.saveIP());
1325 .Case([&](omp::FlushOp) {
1334 ompBuilder->createFlush(builder.saveIP());
1337 .Case([&](omp::ParallelOp op) {
1340 .Case([&](omp::ReductionOp reductionOp) {
1343 .Case([&](omp::MasterOp) {
1346 .Case([&](omp::CriticalOp) {
1349 .Case([&](omp::OrderedRegionOp) {
1352 .Case([&](omp::OrderedOp) {
1355 .Case([&](omp::WsLoopOp) {
1358 .Case([&](omp::SimdLoopOp) {
1361 .Case([&](omp::AtomicReadOp) {
1364 .Case([&](omp::AtomicWriteOp) {
1367 .Case([&](omp::AtomicUpdateOp op) {
1370 .Case([&](omp::AtomicCaptureOp op) {
1373 .Case([&](omp::SectionsOp) {
1376 .Case([&](omp::SingleOp op) {
1379 .Case<omp::YieldOp, omp::TerminatorOp, omp::ReductionDeclareOp,
1380 omp::CriticalDeclareOp>([](
auto op) {
1391 .Case([&](omp::ThreadprivateOp) {
1395 return inst->
emitError(
"unsupported OpenMP operation: ")
1401 registry.
insert<omp::OpenMPDialect>();
1403 dialect->addInterfaces<OpenMPDialectLLVMIRTranslationInterface>();
static LogicalResult convertOmpSections(Operation &opInst, llvm::IRBuilderBase &builder, LLVM::ModuleTranslation &moduleTranslation)
Include the generated interface declarations.
This class contains a list of basic blocks and a link to the parent operation it is attached to...
static LogicalResult convertOmpThreadprivate(Operation &opInst, llvm::IRBuilderBase &builder, LLVM::ModuleTranslation &moduleTranslation)
Converts an OpenMP Threadprivate operation into LLVM IR using OpenMPIRBuilder.
static omp::ReductionDeclareOp findReductionDecl(omp::WsLoopOp container, omp::ReductionOp reduction)
Returns a reduction declaration that corresponds to the given reduction operation in the given contai...
Operation is a basic unit of execution within MLIR.
operand_range getOperands()
Returns an iterator on the underlying Value's.
static llvm::omp::ProcBindKind getProcBindKind(omp::ClauseProcBindKind kind)
Convert ProcBindKind from MLIR-generated enum to LLVM enum.
Block represents an ordered list of Operations.
llvm::Type * convertType(Type type)
Converts the type from MLIR LLVM dialect to LLVM.
static LogicalResult convertOmpAtomicRead(Operation &opInst, llvm::IRBuilderBase &builder, LLVM::ModuleTranslation &moduleTranslation)
Convert omp.atomic.read operation to LLVM IR.
bool wasInterrupted() const
Returns true if the walk was interrupted.
Value getOperand(unsigned idx)
bool failed(LogicalResult result)
Utility function that returns true if the provided LogicalResult corresponds to a failure value...
static LogicalResult convertOmpWsLoop(Operation &opInst, llvm::IRBuilderBase &builder, LLVM::ModuleTranslation &moduleTranslation)
Converts an OpenMP workshare loop into LLVM IR using OpenMPIRBuilder.
static LogicalResult convertOmpOrdered(Operation &opInst, llvm::IRBuilderBase &builder, LLVM::ModuleTranslation &moduleTranslation)
Converts an OpenMP 'ordered' operation into LLVM IR using OpenMPIRBuilder.
unsigned getNumOperands()
#define MLIR_DEFINE_EXPLICIT_INTERNAL_INLINE_TYPE_ID(CLASS_NAME)
static Type getElementType(Type type, ArrayRef< int32_t > indices, function_ref< InFlightDiagnostic(StringRef)> emitErrorFn)
Walks the given type hierarchy with the given indices, potentially down to component granularity...
void appendDialectRegistry(const DialectRegistry ®istry)
Append the contents of the given dialect registry to the registry associated with this context...
llvm::LLVMContext & getLLVMContext() const
Returns the LLVM context in which the IR is being constructed.
Concrete CRTP base class for ModuleTranslation stack frames.
static LogicalResult convertOmpOrderedRegion(Operation &opInst, llvm::IRBuilderBase &builder, LLVM::ModuleTranslation &moduleTranslation)
Converts an OpenMP 'ordered_region' operation into LLVM IR using OpenMPIRBuilder. ...
llvm::AtomicOrdering convertAtomicOrdering(Optional< omp::ClauseMemoryOrderKind > ao)
Convert an Atomic Ordering attribute to llvm::AtomicOrdering.
SmallVector< llvm::Value * > lookupValues(ValueRange values)
Looks up remapped a list of remapped values.
LogicalResult convertBlock(Block &bb, bool ignoreArguments, llvm::IRBuilderBase &builder)
Translates the contents of the given block to LLVM IR using this translator.
llvm::BasicBlock * lookupBlock(Block *block) const
Finds an LLVM IR basic block that corresponds to the given MLIR block.
BlockArgument getArgument(unsigned i)
static LogicalResult inlineConvertOmpRegions(Region ®ion, StringRef blockName, llvm::IRBuilderBase &builder, LLVM::ModuleTranslation &moduleTranslation, SmallVectorImpl< llvm::Value *> *continuationBlockArgs=nullptr)
Translates the blocks contained in the given region and appends them to at the current insertion poin...
void forgetMapping(Region ®ion)
Removes the mapping for blocks contained in the region and values defined in these blocks...
void mapBlock(Block *mlir, llvm::BasicBlock *llvm)
Stores the mapping between an MLIR block and LLVM IR basic block.
Implementation class for module translation.
llvm::OpenMPIRBuilder * getOpenMPBuilder()
Returns the OpenMP IR builder associated with the LLVM IR module being constructed.
static LogicalResult convertOmpReductionOp(omp::ReductionOp reductionOp, llvm::IRBuilderBase &builder, LLVM::ModuleTranslation &moduleTranslation)
Converts an OpenMP reduction operation using OpenMPIRBuilder.
LogicalResult success(bool isSuccess=true)
Utility function to generate a LogicalResult.
void addExtension(std::unique_ptr< DialectExtensionBase > extension)
Add the given extension to the registry.
This class represents an efficient way to signal success or failure.
LogicalResult failure(bool isFailure=true)
Utility function to generate a LogicalResult.
void mapValue(Value mlir, llvm::Value *llvm)
Stores the mapping between an MLIR value and its LLVM IR counterpart.
static LogicalResult convertOmpSingle(omp::SingleOp &singleOp, llvm::IRBuilderBase &builder, LLVM::ModuleTranslation &moduleTranslation)
Converts an OpenMP single construct into LLVM IR using OpenMPIRBuilder.
static LogicalResult convertOmpAtomicCapture(omp::AtomicCaptureOp atomicCaptureOp, llvm::IRBuilderBase &builder, LLVM::ModuleTranslation &moduleTranslation)
static llvm::BasicBlock * convertOmpOpRegions(Region ®ion, StringRef blockName, llvm::IRBuilderBase &builder, LLVM::ModuleTranslation &moduleTranslation, LogicalResult &bodyGenStatus, SmallVectorImpl< llvm::PHINode *> *continuationBlockPHIs=nullptr)
Converts the given region that appears within an OpenMP dialect operation to LLVM IR...
Base class for dialect interfaces providing translation to LLVM IR.
RAII object calling stackPush/stackPop on construction/destruction.
OpResult getResult(unsigned idx)
Get the 'idx'th result of this operation.
static LogicalResult convertOmpCritical(Operation &opInst, llvm::IRBuilderBase &builder, LLVM::ModuleTranslation &moduleTranslation)
Converts an OpenMP 'critical' operation into LLVM IR using OpenMPIRBuilder.
static WalkResult interrupt()
A utility result that is used to signal how to proceed with an ongoing walk:
void registerOpenMPDialectTranslation(DialectRegistry ®istry)
Register the OpenMP dialect and the translation from it to the LLVM IR in the given registry;...
static LogicalResult convertOmpAtomicUpdate(omp::AtomicUpdateOp &opInst, llvm::IRBuilderBase &builder, LLVM::ModuleTranslation &moduleTranslation)
Converts an OpenMP atomic update operation using OpenMPIRBuilder.
llvm::Value * lookupValue(Value value) const
Finds an LLVM IR value corresponding to the given MLIR value.
static void collectReductionDecls(omp::WsLoopOp loop, SmallVectorImpl< omp::ReductionDeclareOp > &reductions)
Populates reductions with reduction declarations used in the given loop.
Instances of the Type class are uniqued, have an immutable identifier and an optional mutable compone...
WalkResult stackWalk(llvm::function_ref< WalkResult(const T &)> callback) const
Calls callback for every ModuleTranslation stack frame of type T starting from the top of the stack...
Runtime
Potential runtimes for AMD GPU kernels.
void connectPHINodes(Region ®ion, const ModuleTranslation &state)
For all blocks in the region that were converted to LLVM IR using the given ModuleTranslation, connect the PHI nodes of the corresponding LLVM IR blocks to the results of preceding blocks.
This class represents an instance of an SSA value in the MLIR system, representing a computable value...
static llvm::OpenMPIRBuilder::InsertPointTy findAllocaInsertPoint(llvm::IRBuilderBase &builder, const LLVM::ModuleTranslation &moduleTranslation)
Find the insertion point for allocas given the current insertion point for normal operations in the b...
Operation * getTerminator()
Get the terminator operation of this block.
static LogicalResult convertOmpParallel(omp::ParallelOp opInst, llvm::IRBuilderBase &builder, LLVM::ModuleTranslation &moduleTranslation)
Converts the OpenMP parallel operation to LLVM IR.
InFlightDiagnostic emitError(Location loc)
Utility method to emit an error message using this location.
SetVector< Block * > getTopologicallySortedBlocks(Region ®ion)
Get a topologically sorted list of blocks of the given region.
The DialectRegistry maps a dialect namespace to a constructor for the matching dialect.
llvm::GlobalValue * lookupGlobal(Operation *op)
Finds an LLVM IR global value that corresponds to the given MLIR operation defining a global value...
llvm::AtomicRMWInst::BinOp convertBinOpToAtomic(Operation &op)
Converts an LLVM dialect binary operation to the corresponding enum value for atomicrmw supported bin...
Operation * getDefiningOp() const
If this value is the result of an operation, return the operation that defines it.
MLIRContext is the top-level object for a collection of MLIR operations.
static OwningAtomicReductionGen makeAtomicReductionGen(omp::ReductionDeclareOp decl, llvm::IRBuilderBase &builder, LLVM::ModuleTranslation &moduleTranslation)
Create an OpenMPIRBuilder-compatible atomic reduction generator for the given reduction declaration...
LLVM dialect pointer type.
static LogicalResult convertOmpAtomicWrite(Operation &opInst, llvm::IRBuilderBase &builder, LLVM::ModuleTranslation &moduleTranslation)
Converts an omp.atomic.write operation to LLVM IR.
LLVMTranslationDialectInterface(Dialect *dialect)
static LogicalResult convertOmpMaster(Operation &opInst, llvm::IRBuilderBase &builder, LLVM::ModuleTranslation &moduleTranslation)
Converts an OpenMP 'master' operation into LLVM IR using OpenMPIRBuilder.
OperationName getName()
The name of an operation is the key identifier for it.
InFlightDiagnostic emitError(const Twine &message={})
Emit an error about fatal conditions with this operation, reporting up to any diagnostic handlers tha...
Type getElementType() const
Returns the pointed-to type. It may be null if the pointer is opaque.
static LogicalResult convertOmpSimdLoop(Operation &opInst, llvm::IRBuilderBase &builder, LLVM::ModuleTranslation &moduleTranslation)
Converts an OpenMP simd loop into LLVM IR using OpenMPIRBuilder.
static OwningReductionGen makeReductionGen(omp::ReductionDeclareOp decl, llvm::IRBuilderBase &builder, LLVM::ModuleTranslation &moduleTranslation)
Create an OpenMPIRBuilder-compatible reduction generator for the given reduction declaration.
SmallVector< AffineExpr, 4 > concat(ArrayRef< AffineExpr > a, ArrayRef< AffineExpr > b)
Return the vector that is the concatenation of a and b.